From d646c2f12f984c71b0b2f33a490327092fbc1c1f Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 14 Aug 2026 14:55:38 +0900 Subject: [PATCH] =?UTF-8?q?feat(messaging):=20=EB=B8=8C=EB=A1=9C=EC=BB=A4?= =?UTF-8?q?=20=EC=A4=91=EB=A6=BD=20=EB=A9=94=EC=8B=9C=EC=A7=95=20=ED=94=8C?= =?UTF-8?q?=EB=9E=AB=ED=8F=BC=2024=EA=B0=9C=20leaf=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit messaging-superpowers-package 설계서/계획서 기반 구현. registry를 19 → 43 leaf로 확장하고 src/messaging 아래 24개 leaf를 등록. - core-api: M1 publish/consume + M2 batch·delayed·pause-resume - policy/transport-spi: 재시도 결정, DLQ orchestration, admission control, lifecycle - kafka·rabbit(Stable): contiguous commit, confirm/return 상관, 배치, 보안 설정 - pulsar·nats(Experimental): 기본 비활성, live 인증 없음을 코드로 기록 - outbox/inbox/claim-check: 트랜잭션 결합, lease, 무결성 검증 - admin: plan → approve → execute를 타입으로 강제 - 문서 9종, infra compose 7종, JMH 벤치마크 3종 검증: 아키텍처 게이트 3종 통과, 24개 leaf 전부 check 통과, messaging 테스트 604개 통과/0 실패. 미완: 계획서가 요구한 실 브로커 IT 40개 중 7개만 작성. Rabbit 13 / Outbox 6 / Inbox 4 / NATS·Pulsar·Share 5 / testkit 2 / starter·admin 3, 그리고 TLS·ACL 2개가 남음. --- docs/messaging/configuration-reference.md | 156 +++++++ docs/messaging/delivery-guarantees.md | 75 ++++ docs/messaging/experimental-policy.md | 97 ++++ docs/messaging/migration-guide.md | 113 +++++ docs/messaging/operations.md | 113 +++++ docs/messaging/outbox-inbox.md | 95 ++++ docs/messaging/retry-dlq-redrive.md | 103 +++++ docs/messaging/security.md | 92 ++++ docs/messaging/support-matrix.md | 116 +++++ infra/messaging/kafka/docker-compose.yml | 33 ++ infra/messaging/nats/docker-compose.yml | 21 + infra/messaging/postgres/docker-compose.yml | 27 ++ infra/messaging/pulsar/docker-compose.yml | 17 + infra/messaging/rabbitmq/docker-compose.yml | 22 + infra/messaging/tls/README.md | 35 ++ infra/messaging/toxiproxy/docker-compose.yml | 19 + src/build.gradle | 50 ++- src/config/architecture/modules.json | 279 ++++++++++++ src/config/spotbugs/exclude.xml | 20 + .../messaging-admin-api/build.gradle | 6 + .../messaging-admin-api/gradle.lockfile | 83 ++++ .../messaging/admin/AdminApproval.java | 45 ++ .../messaging/admin/ApprovedRedrivePlan.java | 56 +++ .../messaging/admin/ApprovedReplayPlan.java | 49 ++ .../messaging/admin/DestinationTopology.java | 46 ++ .../messaging/admin/DestructiveOperation.java | 25 ++ .../admin/DestructiveOperationGuard.java | 73 +++ .../messaging/admin/RedrivePlan.java | 64 +++ .../messaging/admin/RedriveRequest.java | 37 ++ .../messaging/admin/RedriveResult.java | 51 +++ .../messaging/admin/ReplayPlan.java | 55 +++ .../messaging/admin/ReplayRequest.java | 36 ++ .../messaging/admin/ReplayResult.java | 49 ++ .../messaging/admin/TopologyIssue.java | 82 ++++ .../admin/TopologyManagementMode.java | 37 ++ .../messaging/admin/TopologyManifest.java | 76 ++++ .../admin/TopologyValidationReport.java | 83 ++++ .../admin/DestructiveOperationGuardTest.java | 116 +++++ .../messaging-admin-runtime/build.gradle | 10 + .../messaging-admin-runtime/gradle.lockfile | 88 ++++ .../AdminOperationIdempotencyStore.java | 78 ++++ .../runtime/BrokerTopologyInspector.java | 33 ++ .../runtime/CompositeTopologyValidator.java | 52 +++ .../runtime/DefaultMessagingAdminService.java | 199 +++++++++ .../runtime/DestructiveMessagingAdmin.java | 82 ++++ .../admin/runtime/MessagingAdminService.java | 65 +++ .../admin/runtime/RedriveReport.java | 18 + .../admin/runtime/RedriveService.java | 152 +++++++ .../messaging/admin/runtime/ReplayReport.java | 21 + .../admin/runtime/ReplayService.java | 99 ++++ .../runtime/TopologyValidationRuntime.java | 88 ++++ .../admin/runtime/TopologyValidator.java | 91 ++++ .../runtime/ApprovedPlanExecutionTest.java | 190 ++++++++ .../TopologyValidationRuntimeTest.java | 65 +++ .../admin/runtime/TopologyValidatorTest.java | 172 +++++++ .../messaging-claim-check/build.gradle | 6 + .../messaging-claim-check/gradle.lockfile | 83 ++++ .../ClaimCheckIntegrityException.java | 44 ++ .../claimcheck/ClaimCheckIntegrityGuard.java | 71 +++ .../claimcheck/ClaimCheckPolicy.java | 89 ++++ .../claimcheck/ClaimCheckPublisher.java | 91 ++++ .../claimcheck/ClaimCheckResolver.java | 91 ++++ .../messaging/claimcheck/ClaimCheckStore.java | 32 ++ .../ClaimCheckIntegrityGuardTest.java | 86 ++++ .../claimcheck/ClaimCheckResolverTest.java | 168 +++++++ .../ClaimCheckRetentionValidatorTest.java | 79 ++++ .../messaging-cloudevents/build.gradle | 9 + .../messaging-cloudevents/gradle.lockfile | 85 ++++ .../cloudevents/CloudEventExtensions.java | 24 + .../cloudevents/CloudEventMapper.java | 33 ++ .../cloudevents/DefaultCloudEventMapper.java | 171 +++++++ .../cloudevents/CloudEventMappingTest.java | 160 +++++++ src/messaging/messaging-core-api/build.gradle | 4 + .../messaging-core-api/gradle.lockfile | 83 ++++ .../caskeleton/messaging/api/CausationId.java | 18 + .../caskeleton/messaging/api/ContentType.java | 35 ++ .../messaging/api/CorrelationId.java | 17 + .../messaging/api/MessageEnvelope.java | 146 ++++++ .../caskeleton/messaging/api/MessageId.java | 29 ++ .../caskeleton/messaging/api/MessageType.java | 20 + .../caskeleton/messaging/api/ProducerId.java | 20 + .../messaging/api/SchemaVersion.java | 15 + .../messaging/api/TenantContext.java | 24 + .../messaging/api/TraceContext.java | 44 ++ .../dev/caskeleton/messaging/api/UuidV7.java | 62 +++ .../api/delivery/BatchDeliveryMetadata.java | 46 ++ .../api/delivery/BatchMessageDelivery.java | 29 ++ .../api/delivery/BatchMessageHandler.java | 24 + .../api/delivery/DeliveryContext.java | 36 ++ .../api/delivery/DeliveryGuarantee.java | 18 + .../api/delivery/DeliveryMetadata.java | 46 ++ .../delivery/ExternalSideEffectGuarantee.java | 14 + .../messaging/api/delivery/HandleResult.java | 60 +++ .../api/delivery/MessageDelivery.java | 22 + .../api/delivery/MessageHandler.java | 23 + .../messaging/api/delivery/OrderingScope.java | 23 + .../api/delivery/PauseResumeController.java | 33 ++ .../api/delivery/ProcessingGuarantee.java | 11 + .../api/destination/CapabilityRegistry.java | 15 + .../destination/ConfirmationRequirement.java | 20 + .../destination/DestinationCapabilities.java | 26 ++ .../api/destination/DestinationKind.java | 32 ++ .../api/destination/DestinationName.java | 24 + .../api/destination/MessageDestination.java | 26 ++ .../destination/MessagingCapabilities.java | 47 ++ .../messaging/api/error/FailureCategory.java | 40 ++ .../api/error/FailureDescriptor.java | 80 ++++ .../error/MessageAuthenticationException.java | 49 ++ .../error/MessageAuthorizationException.java | 49 ++ .../error/MessageBackpressureException.java | 40 ++ .../MessageBrokerUnavailableException.java | 49 ++ .../api/error/MessageConsumerException.java | 49 ++ .../api/error/MessageDeadLetterException.java | 54 +++ .../error/MessageHandlerTimeoutException.java | 54 +++ .../error/MessageHeaderRejectedException.java | 49 ++ .../MessagePublishAmbiguousException.java | 54 +++ .../MessagePublishRejectedException.java | 53 +++ .../error/MessagePublishTimeoutException.java | 54 +++ .../api/error/MessageRedriveException.java | 49 ++ .../error/MessageRetryExhaustedException.java | 49 ++ .../api/error/MessageRoutingException.java | 49 ++ .../MessageSchemaIncompatibleException.java | 49 ++ .../error/MessageSerializationException.java | 49 ++ .../api/error/MessageSettlementException.java | 49 ++ .../MessageSettlementUnknownException.java | 53 +++ .../api/error/MessageTooLargeException.java | 54 +++ .../api/error/MessageTopologyException.java | 49 ++ .../api/error/MessageValidationException.java | 49 ++ ...ssagingCapabilityUnavailableException.java | 56 +++ .../MessagingConfigurationException.java | 55 +++ .../api/error/MessagingException.java | 66 +++ .../messaging/api/header/HeaderName.java | 36 ++ .../messaging/api/header/HeaderValue.java | 23 + .../messaging/api/header/MessageHeaders.java | 164 +++++++ .../messaging/api/header/ReservedHeaders.java | 126 ++++++ .../api/publish/BatchMessagePublisher.java | 25 ++ .../api/publish/BatchPublishItemResult.java | 27 ++ .../api/publish/BatchPublishOptions.java | 42 ++ .../api/publish/BatchPublishResult.java | 44 ++ .../api/publish/BlockingMessagePublisher.java | 30 ++ .../messaging/api/publish/BrokerPosition.java | 27 ++ .../api/publish/ConfirmationLevel.java | 20 + .../api/publish/DelayedMessagePublisher.java | 28 ++ .../api/publish/MessagePublisher.java | 26 ++ .../api/publish/PublishCompletion.java | 20 + .../api/publish/PublishDeduplication.java | 27 ++ .../api/publish/PublishEvidence.java | 66 +++ .../messaging/api/publish/PublishOptions.java | 61 +++ .../messaging/api/publish/PublishRequest.java | 40 ++ .../messaging/api/publish/PublishResult.java | 88 ++++ .../messaging/api/publish/RoutingOutcome.java | 23 + .../api/publish/TransmissionEvidence.java | 14 + .../api/settlement/ManualMessageHandler.java | 25 ++ .../api/settlement/SettlementCompletion.java | 14 + .../api/settlement/SettlementController.java | 50 +++ .../api/settlement/SettlementEvidence.java | 66 +++ .../api/settlement/SettlementResult.java | 47 ++ .../messaging/api/CoreValueTypesTest.java | 60 +++ .../messaging/api/MessageEnvelopeTest.java | 141 ++++++ .../messaging/api/ModuleSmokeTest.java | 13 + .../api/delivery/ConsumerContractTest.java | 129 ++++++ .../DestinationCapabilityTest.java | 60 +++ .../api/publish/PublishResultTest.java | 166 +++++++ .../messaging-inbox-jpa/build.gradle | 16 + .../messaging-inbox-jpa/gradle.lockfile | 105 +++++ .../messaging/inbox/IdempotentConsumer.java | 77 ++++ .../messaging/inbox/InboxCleanupJob.java | 65 +++ .../messaging/inbox/InboxOutcome.java | 34 ++ .../messaging/inbox/InboxRetentionPolicy.java | 78 ++++ .../messaging/inbox/JdbcInboxRepository.java | 116 +++++ .../inbox/TransactionalInboxHandler.java | 104 +++++ .../messaging/V2__messaging_inbox.sql | 18 + .../inbox/IdempotentConsumerTest.java | 140 ++++++ .../messaging/inbox/InboxOperationsTest.java | 153 +++++++ .../messaging/inbox/InboxPostgresIT.java | 181 ++++++++ .../build.gradle | 10 + .../gradle.lockfile | 101 +++++ .../kafka/share/KafkaShareGroupRegistrar.java | 88 ++++ .../kafka/share/KafkaShareProfile.java | 33 ++ .../share/KafkaShareProfileValidator.java | 42 ++ .../share/KafkaShareWorkQueueCapability.java | 27 ++ .../share/KafkaShareProfileValidatorTest.java | 112 +++++ src/messaging/messaging-kafka/build.gradle | 29 ++ src/messaging/messaging-kafka/gradle.lockfile | 120 +++++ .../kafka/KafkaPublishBenchmark.java | 155 +++++++ .../ContiguousPartitionOffsetTracker.java | 105 +++++ .../kafka/KafkaBatchConsumerRegistrar.java | 146 ++++++ .../messaging/kafka/KafkaBrokerProfile.java | 56 +++ .../kafka/KafkaConsumerRegistrar.java | 357 +++++++++++++++ .../kafka/KafkaDeadLetterPublisher.java | 49 ++ .../messaging/kafka/KafkaDeliveryMapper.java | 167 +++++++ .../messaging/kafka/KafkaHeaderMapper.java | 110 +++++ .../kafka/KafkaMessagingTransport.java | 196 ++++++++ .../kafka/KafkaOffsetResetExecutor.java | 63 +++ .../kafka/KafkaPartitionRetryScheduler.java | 90 ++++ .../messaging/kafka/KafkaPosition.java | 38 ++ .../kafka/KafkaProfileValidator.java | 60 +++ .../kafka/KafkaPublishFailureClassifier.java | 116 +++++ .../messaging/kafka/KafkaPublishMapper.java | 111 +++++ .../kafka/KafkaReplayCapability.java | 82 ++++ .../messaging/kafka/KafkaReplayPlan.java | 36 ++ .../messaging/kafka/KafkaReplayPlanner.java | 58 +++ .../messaging/kafka/KafkaRetryExecutor.java | 126 ++++++ .../kafka/KafkaRetryMetadataMapper.java | 80 ++++ .../messaging/kafka/KafkaRetryOutcome.java | 40 ++ .../kafka/KafkaRetryTopicPublisher.java | 75 ++++ .../kafka/KafkaSecurityConfigurer.java | 135 ++++++ .../kafka/KafkaSettlementCommand.java | 39 ++ .../messaging/kafka/KafkaSettlementQueue.java | 52 +++ .../kafka/KafkaTopologyInspector.java | 45 ++ .../KafkaTransactionProfileValidator.java | 52 +++ .../kafka/KafkaTransactionalDelivery.java | 22 + .../kafka/KafkaTransactionalOutput.java | 21 + .../kafka/KafkaTransactionalProcessor.java | 29 ++ .../kafka/KafkaTransactionalPublisher.java | 117 +++++ .../kafka/PartitionOffsetTracker.java | 48 ++ .../kafka/PartitionWorkCoordinator.java | 128 ++++++ .../SpringKafkaTransactionalProcessor.java | 61 +++ .../ContiguousPartitionOffsetTrackerTest.java | 94 ++++ .../kafka/KafkaAmbiguityChaosIT.java | 168 +++++++ .../messaging/kafka/KafkaBrokerIT.java | 284 ++++++++++++ .../kafka/KafkaConsumerRegistrarTest.java | 261 +++++++++++ .../kafka/KafkaConsumerSettlementIT.java | 216 +++++++++ .../kafka/KafkaContainerFixture.java | 193 ++++++++ .../kafka/KafkaContainerSmokeTest.java | 66 +++ .../messaging/kafka/KafkaContractHarness.java | 370 +++++++++++++++ .../messaging/kafka/KafkaFixtureProfiles.java | 32 ++ .../kafka/KafkaHeaderMapperTest.java | 118 +++++ .../kafka/KafkaProducerContractTest.java | 24 + .../kafka/KafkaProfileValidatorTest.java | 167 +++++++ .../messaging/kafka/KafkaReadCommittedIT.java | 155 +++++++ .../kafka/KafkaReplayPlannerTest.java | 88 ++++ .../kafka/KafkaTopologyValidationIT.java | 144 ++++++ .../kafka/KafkaTransactionFencingIT.java | 150 +++++++ .../kafka/KafkaTransactionFixtures.java | 78 ++++ .../messaging/kafka/KafkaTransactionIT.java | 168 +++++++ .../kafka/PartitionWorkCoordinatorTest.java | 77 ++++ .../messaging-nats-experimental/build.gradle | 13 + .../gradle.lockfile | 90 ++++ .../messaging/nats/NatsAckMode.java | 57 +++ .../messaging/nats/NatsJetStreamProfile.java | 103 +++++ .../nats/NatsJetStreamProfileValidator.java | 75 ++++ .../nats/NatsJetStreamTransport.java | 268 +++++++++++ .../nats/NatsMaxDeliverParkingWorkflow.java | 85 ++++ .../messaging/nats/NatsStreamPosition.java | 75 ++++ .../nats/NatsAdapterContractTest.java | 263 +++++++++++ .../nats/NatsMaxDeliverParkingTest.java | 123 +++++ .../messaging-observability/build.gradle | 7 + .../messaging-observability/gradle.lockfile | 88 ++++ .../observation/CardinalityGuard.java | 89 ++++ ...DefaultMessagingObservationConvention.java | 123 +++++ .../observation/MessagingAuditEvent.java | 45 ++ .../observation/MessagingAuditSink.java | 56 +++ .../observation/MessagingMetrics.java | 158 +++++++ .../observation/MessagingObservation.java | 56 +++ .../observation/MessagingRedactor.java | 105 +++++ .../messaging/observation/MessagingTags.java | 71 +++ .../observation/MessagingTracer.java | 96 ++++ .../MessagingMetricCardinalityTest.java | 110 +++++ .../observation/MessagingRedactorTest.java | 82 ++++ .../observation/MessagingSecretLeakTest.java | 119 +++++ .../observation/MessagingTraceLinkTest.java | 88 ++++ .../observation/SecretLeakStaticScanTest.java | 205 +++++++++ .../messaging-outbox-jpa/build.gradle | 18 + .../messaging-outbox-jpa/gradle.lockfile | 110 +++++ .../outbox/DebeziumMappedRecord.java | 54 +++ .../outbox/DebeziumOutboxEventRouter.java | 76 ++++ .../outbox/DebeziumOutboxProfile.java | 67 +++ .../outbox/DebeziumOutboxRecordMapper.java | 71 +++ .../outbox/JdbcOutboxRepository.java | 332 ++++++++++++++ .../messaging/outbox/OutboxCleanupJob.java | 58 +++ .../outbox/OutboxEnvelopeFactory.java | 68 +++ .../messaging/outbox/OutboxProperties.java | 81 ++++ .../messaging/outbox/OutboxRelay.java | 114 +++++ .../messaging/outbox/OutboxRelayReport.java | 22 + .../outbox/OutboxRetryScheduler.java | 103 +++++ .../messaging/V1__messaging_outbox.sql | 41 ++ .../debezium/outbox-event-router.properties | 43 ++ .../DebeziumOutboxRecordMapperTest.java | 134 ++++++ .../outbox/OutboxOperationsTest.java | 172 +++++++ .../messaging/outbox/OutboxPostgresIT.java | 223 +++++++++ .../messaging/outbox/OutboxRelayTest.java | 312 +++++++++++++ src/messaging/messaging-policy/build.gradle | 6 + .../messaging-policy/gradle.lockfile | 83 ++++ .../messaging/policy/BackoffCalculator.java | 58 +++ .../messaging/policy/CapabilityTier.java | 21 + .../messaging/policy/ConsumerPolicy.java | 55 +++ .../policy/DeadLetterEnvelopeFactory.java | 57 +++ .../messaging/policy/DeadLetterMetadata.java | 42 ++ .../policy/DeadLetterOrchestrator.java | 121 +++++ .../messaging/policy/DeadLetterPolicy.java | 45 ++ .../messaging/policy/DeadLetterResult.java | 17 + .../policy/DefaultRetryDecisionEngine.java | 103 +++++ .../messaging/policy/DestinationProfile.java | 83 ++++ .../policy/DestinationProfileValidator.java | 181 ++++++++ .../policy/FailureDescriptorDefaults.java | 14 + .../messaging/policy/InFlightLimiter.java | 82 ++++ .../policy/MessagingAdmissionController.java | 101 +++++ .../messaging/policy/OrderingImpact.java | 11 + .../messaging/policy/PayloadLimitGuard.java | 86 ++++ .../messaging/policy/PayloadPolicy.java | 39 ++ .../messaging/policy/PhysicalDestination.java | 125 ++++++ .../messaging/policy/ProducerPolicy.java | 38 ++ .../messaging/policy/RetryContext.java | 34 ++ .../messaging/policy/RetryDecision.java | 76 ++++ .../messaging/policy/RetryDecisionEngine.java | 13 + .../messaging/policy/RetryMode.java | 29 ++ .../messaging/policy/RetryPolicy.java | 136 ++++++ .../messaging/policy/SchemaPolicy.java | 31 ++ .../messaging/policy/SourceSettlement.java | 21 + .../policy/DeadLetterOrchestratorTest.java | 312 +++++++++++++ .../DestinationProfileValidatorTest.java | 422 ++++++++++++++++++ .../MessagingAdmissionControllerTest.java | 122 +++++ .../policy/RetryDecisionEngineTest.java | 259 +++++++++++ .../build.gradle | 13 + .../gradle.lockfile | 102 +++++ .../pulsar/PulsarMessagePosition.java | 49 ++ .../pulsar/PulsarMessagingTransport.java | 249 +++++++++++ .../messaging/pulsar/PulsarProfile.java | 80 ++++ .../pulsar/PulsarProfileValidator.java | 66 +++ .../pulsar/PulsarSubscriptionMode.java | 62 +++ .../pulsar/PulsarSubscriptionType.java | 17 + .../pulsar/PulsarTransactionCapability.java | 49 ++ .../pulsar/PulsarAdapterContractTest.java | 234 ++++++++++ .../pulsar/PulsarSubscriptionGuardTest.java | 125 ++++++ src/messaging/messaging-rabbit/build.gradle | 23 + .../messaging-rabbit/gradle.lockfile | 127 ++++++ .../rabbit/RabbitPublishBenchmark.java | 156 +++++++ .../rabbit/RabbitBatchConsumerRegistrar.java | 165 +++++++ .../messaging/rabbit/RabbitBrokerProfile.java | 50 +++ .../rabbit/RabbitChannelPublisher.java | 33 ++ .../rabbit/RabbitConfirmCoordinator.java | 190 ++++++++ .../rabbit/RabbitConsumerRegistrar.java | 198 ++++++++ .../rabbit/RabbitDeadLetterPublisher.java | 130 ++++++ .../rabbit/RabbitDeliveryMapper.java | 185 ++++++++ .../messaging/rabbit/RabbitHeaderMapper.java | 127 ++++++ .../rabbit/RabbitMessagingTransport.java | 218 +++++++++ .../RabbitNativeDeadLetterCapability.java | 70 +++ .../rabbit/RabbitProfileValidator.java | 83 ++++ .../RabbitPublishFailureClassifier.java | 117 +++++ .../messaging/rabbit/RabbitPublishMapper.java | 72 +++ .../rabbit/RabbitPublishReference.java | 37 ++ .../messaging/rabbit/RabbitRequestReply.java | 32 ++ .../rabbit/RabbitRetryQueueTopology.java | 57 +++ .../rabbit/RabbitSecurityConfigurer.java | 179 ++++++++ .../rabbit/RabbitSettlementController.java | 84 ++++ .../rabbit/RabbitSettlementOperations.java | 50 +++ .../rabbit/RabbitTopologyProfile.java | 137 ++++++ .../rabbit/RabbitAdapterContractTest.java | 24 + .../messaging/rabbit/RabbitBrokerIT.java | 232 ++++++++++ .../rabbit/RabbitConfirmCoordinatorTest.java | 116 +++++ .../rabbit/RabbitContractHarness.java | 304 +++++++++++++ .../rabbit/RabbitFixtureProfiles.java | 32 ++ .../rabbit/RabbitProfileValidatorTest.java | 184 ++++++++ .../messaging/rabbit/RabbitRuntimeTest.java | 263 +++++++++++ .../RabbitSettlementControllerTest.java | 105 +++++ .../rabbit/RabbitTopologyAndBatchTest.java | 218 +++++++++ .../messaging-reliability-api/build.gradle | 5 + .../messaging-reliability-api/gradle.lockfile | 83 ++++ .../reliability/ClaimCheckReference.java | 50 +++ .../reliability/IdempotentMessageHandler.java | 33 ++ .../messaging/reliability/InboxRecord.java | 27 ++ .../reliability/InboxRepository.java | 46 ++ .../messaging/reliability/InboxResult.java | 45 ++ .../messaging/reliability/OutboxRecord.java | 131 ++++++ .../reliability/OutboxRepository.java | 88 ++++ .../messaging/reliability/OutboxStatus.java | 27 ++ .../reliability/ReliableMessagePublisher.java | 27 ++ .../TransactionalMessageAction.java | 29 ++ .../messaging-schema-api/build.gradle | 5 + .../messaging-schema-api/gradle.lockfile | 83 ++++ .../messaging/schema/EncodedMessage.java | 66 +++ .../messaging/schema/MessageCodec.java | 59 +++ .../schema/MessageCodecRegistry.java | 26 ++ .../schema/RawBytesMessageCodec.java | 76 ++++ .../messaging/schema/SchemaCompatibility.java | 32 ++ .../schema/SchemaCompatibilityValidator.java | 113 +++++ .../messaging/schema/SchemaReference.java | 39 ++ .../messaging/schema/SchemaRegistry.java | 56 +++ .../schema/RawBytesMessageCodecTest.java | 67 +++ .../SchemaCompatibilityValidatorTest.java | 109 +++++ .../messaging-schema-avro/build.gradle | 8 + .../messaging-schema-avro/gradle.lockfile | 91 ++++ .../schema/avro/AvroCompatibilityGate.java | 73 +++ .../schema/avro/AvroMessageCodec.java | 200 +++++++++ .../schema/avro/AvroCompatibilityTest.java | 178 ++++++++ .../resources/schemas/order.created/v1.avsc | 11 + .../messaging-schema-json/build.gradle | 8 + .../messaging-schema-json/gradle.lockfile | 87 ++++ .../schema/json/JacksonMessageCodec.java | 176 ++++++++ .../schema/json/JacksonMessageCodecTest.java | 124 +++++ .../json/PlatformOverheadPerformanceTest.java | 135 ++++++ .../messaging-schema-protobuf/build.gradle | 8 + .../messaging-schema-protobuf/gradle.lockfile | 84 ++++ .../schema/protobuf/ProtobufMessageCodec.java | 124 +++++ .../protobuf/ProtobufCompatibilityTest.java | 186 ++++++++ .../src/test/proto/order_created_v1.proto | 23 + src/messaging/messaging-security/build.gradle | 5 + .../messaging-security/gradle.lockfile | 83 ++++ .../messaging/security/BrokerAclManifest.java | 141 ++++++ .../security/BrokerCredentialProfile.java | 78 ++++ .../security/BrokerSecurityProfile.java | 41 ++ .../messaging/security/BrokerTlsPolicy.java | 98 ++++ .../messaging/security/CredentialIds.java | 37 ++ .../security/CredentialProvider.java | 29 ++ .../security/CredentialRotationPlan.java | 54 +++ .../messaging/security/CredentialRuntime.java | 143 ++++++ .../security/CredentialRuntimeRegistry.java | 122 +++++ .../security/DestinationAccessPolicy.java | 70 +++ .../security/DestinationAccessValidator.java | 65 +++ .../security/MessageSecurityValidator.java | 50 +++ .../CredentialRuntimeRegistryTest.java | 218 +++++++++ .../MessageSecurityValidatorTest.java | 148 ++++++ .../build.gradle | 31 ++ .../gradle.lockfile | 131 ++++++ .../DefaultBatchMessagePublisher.java | 118 +++++ .../DefaultBlockingMessagePublisher.java | 65 +++ .../DefaultReactiveMessagePublisher.java | 39 ++ .../KafkaMessagingAutoConfiguration.java | 76 ++++ .../MessagingAdminAutoConfiguration.java | 65 +++ .../MessagingCoreAutoConfiguration.java | 247 ++++++++++ .../autoconfigure/MessagingEndpoint.java | 74 +++ .../autoconfigure/MessagingProperties.java | 210 +++++++++ ...MessagingReliabilityAutoConfiguration.java | 113 +++++ .../autoconfigure/PublishResults.java | 71 +++ .../RabbitMessagingAutoConfiguration.java | 62 +++ .../ReactiveMessagePublisher.java | 32 ++ .../ValidatedDestinationRegistry.java | 59 +++ ...ot.autoconfigure.AutoConfiguration.imports | 5 + .../autoconfigure/BatchPublisherTest.java | 227 ++++++++++ .../autoconfigure/BlockingFacadeTest.java | 149 +++++++ .../MessagingAutoConfigurationTest.java | 238 ++++++++++ .../autoconfigure/MessagingEndpointTest.java | 126 ++++++ .../autoconfigure/ReactiveFacadeTest.java | 134 ++++++ .../application-invalid-ordering.yml | 11 + .../src/test/resources/application-valid.yml | 24 + .../build.gradle | 9 + .../gradle.lockfile | 91 ++++ .../streambridge/BindingCapabilityReport.java | 88 ++++ .../streambridge/BindingProfileValidator.java | 97 ++++ .../streambridge/MessagingBindingBridge.java | 34 ++ .../SpringCloudStreamConsumerBridge.java | 87 ++++ .../SpringCloudStreamPublisherBridge.java | 149 +++++++ .../streambridge/StreamBridgePolicyGuard.java | 52 +++ .../BindingProfileValidatorTest.java | 164 +++++++ .../BridgePublishEvidenceTest.java | 131 ++++++ src/messaging/messaging-testkit/build.gradle | 11 + .../messaging-testkit/gradle.lockfile | 87 ++++ .../testkit/EnvelopeCodecBenchmark.java | 126 ++++++ .../testkit/BrokerFailureMatrix.java | 131 ++++++ .../testkit/CompatibilityMatrix.java | 115 +++++ .../messaging/testkit/ContractAssertions.java | 51 +++ .../messaging/testkit/ContractMessage.java | 87 ++++ .../messaging/testkit/DockerAvailability.java | 35 ++ .../messaging/testkit/FaultController.java | 26 ++ .../messaging/testkit/HandleOutcome.java | 14 + .../testkit/MessagingAdapterContract.java | 139 ++++++ .../testkit/MessagingAdapterHarness.java | 73 +++ .../testkit/NetworkFaultScenario.java | 127 ++++++ .../messaging/testkit/ObservedDelivery.java | 23 + .../testkit/CompatibilityMatrixTest.java | 99 ++++ .../testkit/CrossBrokerContractSuite.java | 111 +++++ .../testkit/InMemoryHarnessContractTest.java | 22 + .../testkit/InMemoryMessagingHarness.java | 260 +++++++++++ .../MessagingDocumentationContractTest.java | 145 ++++++ .../messaging-transport-spi/build.gradle | 7 + .../messaging-transport-spi/gradle.lockfile | 83 ++++ .../transport/BackpressureController.java | 114 +++++ .../DefaultMessagingRuntimeRegistry.java | 189 ++++++++ .../GracefulShutdownCoordinator.java | 133 ++++++ .../transport/MessagingLifecycle.java | 67 +++ .../messaging/transport/MessagingRuntime.java | 43 ++ .../transport/MessagingRuntimeLease.java | 22 + .../transport/MessagingRuntimeRegistry.java | 20 + .../transport/MessagingTransport.java | 57 +++ .../TransportConsumerRegistration.java | 40 ++ .../transport/TransportConsumerSpec.java | 21 + .../transport/TransportDelivery.java | 29 ++ .../transport/TransportPublishRequest.java | 28 ++ .../transport/TransportPublishResult.java | 20 + .../transport/TransportSettlement.java | 36 ++ .../BackpressureAndShutdownTest.java | 108 +++++ .../transport/MessagingLifecycleTest.java | 59 +++ .../MessagingRuntimeRegistryTest.java | 188 ++++++++ .../transport/ResourceLeakGateTest.java | 166 +++++++ src/settings.gradle | 2 +- 486 files changed, 40271 insertions(+), 2 deletions(-) create mode 100644 docs/messaging/configuration-reference.md create mode 100644 docs/messaging/delivery-guarantees.md create mode 100644 docs/messaging/experimental-policy.md create mode 100644 docs/messaging/migration-guide.md create mode 100644 docs/messaging/operations.md create mode 100644 docs/messaging/outbox-inbox.md create mode 100644 docs/messaging/retry-dlq-redrive.md create mode 100644 docs/messaging/security.md create mode 100644 docs/messaging/support-matrix.md create mode 100644 infra/messaging/kafka/docker-compose.yml create mode 100644 infra/messaging/nats/docker-compose.yml create mode 100644 infra/messaging/postgres/docker-compose.yml create mode 100644 infra/messaging/pulsar/docker-compose.yml create mode 100644 infra/messaging/rabbitmq/docker-compose.yml create mode 100644 infra/messaging/tls/README.md create mode 100644 infra/messaging/toxiproxy/docker-compose.yml create mode 100644 src/messaging/messaging-admin-api/build.gradle create mode 100644 src/messaging/messaging-admin-api/gradle.lockfile create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/AdminApproval.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ApprovedRedrivePlan.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ApprovedReplayPlan.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/DestinationTopology.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/DestructiveOperation.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/DestructiveOperationGuard.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/RedrivePlan.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/RedriveRequest.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/RedriveResult.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ReplayPlan.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ReplayRequest.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ReplayResult.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyIssue.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyManagementMode.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyManifest.java create mode 100644 src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyValidationReport.java create mode 100644 src/messaging/messaging-admin-api/src/test/java/dev/caskeleton/messaging/admin/DestructiveOperationGuardTest.java create mode 100644 src/messaging/messaging-admin-runtime/build.gradle create mode 100644 src/messaging/messaging-admin-runtime/gradle.lockfile create mode 100644 src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/AdminOperationIdempotencyStore.java create mode 100644 src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/BrokerTopologyInspector.java create mode 100644 src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/CompositeTopologyValidator.java create mode 100644 src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/DefaultMessagingAdminService.java create mode 100644 src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/DestructiveMessagingAdmin.java create mode 100644 src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/MessagingAdminService.java create mode 100644 src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/RedriveReport.java create mode 100644 src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/RedriveService.java create mode 100644 src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/ReplayReport.java create mode 100644 src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/ReplayService.java create mode 100644 src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/TopologyValidationRuntime.java create mode 100644 src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/TopologyValidator.java create mode 100644 src/messaging/messaging-admin-runtime/src/test/java/dev/caskeleton/messaging/admin/runtime/ApprovedPlanExecutionTest.java create mode 100644 src/messaging/messaging-admin-runtime/src/test/java/dev/caskeleton/messaging/admin/runtime/TopologyValidationRuntimeTest.java create mode 100644 src/messaging/messaging-admin-runtime/src/test/java/dev/caskeleton/messaging/admin/runtime/TopologyValidatorTest.java create mode 100644 src/messaging/messaging-claim-check/build.gradle create mode 100644 src/messaging/messaging-claim-check/gradle.lockfile create mode 100644 src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckIntegrityException.java create mode 100644 src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckIntegrityGuard.java create mode 100644 src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckPolicy.java create mode 100644 src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckPublisher.java create mode 100644 src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckResolver.java create mode 100644 src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckStore.java create mode 100644 src/messaging/messaging-claim-check/src/test/java/dev/caskeleton/messaging/claimcheck/ClaimCheckIntegrityGuardTest.java create mode 100644 src/messaging/messaging-claim-check/src/test/java/dev/caskeleton/messaging/claimcheck/ClaimCheckResolverTest.java create mode 100644 src/messaging/messaging-claim-check/src/test/java/dev/caskeleton/messaging/claimcheck/ClaimCheckRetentionValidatorTest.java create mode 100644 src/messaging/messaging-cloudevents/build.gradle create mode 100644 src/messaging/messaging-cloudevents/gradle.lockfile create mode 100644 src/messaging/messaging-cloudevents/src/main/java/dev/caskeleton/messaging/cloudevents/CloudEventExtensions.java create mode 100644 src/messaging/messaging-cloudevents/src/main/java/dev/caskeleton/messaging/cloudevents/CloudEventMapper.java create mode 100644 src/messaging/messaging-cloudevents/src/main/java/dev/caskeleton/messaging/cloudevents/DefaultCloudEventMapper.java create mode 100644 src/messaging/messaging-cloudevents/src/test/java/dev/caskeleton/messaging/cloudevents/CloudEventMappingTest.java create mode 100644 src/messaging/messaging-core-api/build.gradle create mode 100644 src/messaging/messaging-core-api/gradle.lockfile create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/CausationId.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/ContentType.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/CorrelationId.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageEnvelope.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageId.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageType.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/ProducerId.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/SchemaVersion.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/TenantContext.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/TraceContext.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/UuidV7.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/BatchDeliveryMetadata.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/BatchMessageDelivery.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/BatchMessageHandler.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/DeliveryContext.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/DeliveryGuarantee.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/DeliveryMetadata.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/ExternalSideEffectGuarantee.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/HandleResult.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/MessageDelivery.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/MessageHandler.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/OrderingScope.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/PauseResumeController.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/ProcessingGuarantee.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/CapabilityRegistry.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/ConfirmationRequirement.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/DestinationCapabilities.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/DestinationKind.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/DestinationName.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/MessageDestination.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/MessagingCapabilities.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/FailureCategory.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/FailureDescriptor.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageAuthenticationException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageAuthorizationException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageBackpressureException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageBrokerUnavailableException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageConsumerException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageDeadLetterException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageHandlerTimeoutException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageHeaderRejectedException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagePublishAmbiguousException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagePublishRejectedException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagePublishTimeoutException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageRedriveException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageRetryExhaustedException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageRoutingException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSchemaIncompatibleException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSerializationException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSettlementException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSettlementUnknownException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageTooLargeException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageTopologyException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageValidationException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagingCapabilityUnavailableException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagingConfigurationException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagingException.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/HeaderName.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/HeaderValue.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/MessageHeaders.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/ReservedHeaders.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchMessagePublisher.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchPublishItemResult.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchPublishOptions.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchPublishResult.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BlockingMessagePublisher.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BrokerPosition.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/ConfirmationLevel.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/DelayedMessagePublisher.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/MessagePublisher.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishCompletion.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishDeduplication.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishEvidence.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishOptions.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishRequest.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishResult.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/RoutingOutcome.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/TransmissionEvidence.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/ManualMessageHandler.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementCompletion.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementController.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementEvidence.java create mode 100644 src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementResult.java create mode 100644 src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/CoreValueTypesTest.java create mode 100644 src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/MessageEnvelopeTest.java create mode 100644 src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/ModuleSmokeTest.java create mode 100644 src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/delivery/ConsumerContractTest.java create mode 100644 src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/destination/DestinationCapabilityTest.java create mode 100644 src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/publish/PublishResultTest.java create mode 100644 src/messaging/messaging-inbox-jpa/build.gradle create mode 100644 src/messaging/messaging-inbox-jpa/gradle.lockfile create mode 100644 src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/IdempotentConsumer.java create mode 100644 src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/InboxCleanupJob.java create mode 100644 src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/InboxOutcome.java create mode 100644 src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/InboxRetentionPolicy.java create mode 100644 src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/JdbcInboxRepository.java create mode 100644 src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/TransactionalInboxHandler.java create mode 100644 src/messaging/messaging-inbox-jpa/src/main/resources/db/migration/messaging/V2__messaging_inbox.sql create mode 100644 src/messaging/messaging-inbox-jpa/src/test/java/dev/caskeleton/messaging/inbox/IdempotentConsumerTest.java create mode 100644 src/messaging/messaging-inbox-jpa/src/test/java/dev/caskeleton/messaging/inbox/InboxOperationsTest.java create mode 100644 src/messaging/messaging-inbox-jpa/src/test/java/dev/caskeleton/messaging/inbox/InboxPostgresIT.java create mode 100644 src/messaging/messaging-kafka-share-experimental/build.gradle create mode 100644 src/messaging/messaging-kafka-share-experimental/gradle.lockfile create mode 100644 src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareGroupRegistrar.java create mode 100644 src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareProfile.java create mode 100644 src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareProfileValidator.java create mode 100644 src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareWorkQueueCapability.java create mode 100644 src/messaging/messaging-kafka-share-experimental/src/test/java/dev/caskeleton/messaging/kafka/share/KafkaShareProfileValidatorTest.java create mode 100644 src/messaging/messaging-kafka/build.gradle create mode 100644 src/messaging/messaging-kafka/gradle.lockfile create mode 100644 src/messaging/messaging-kafka/src/jmh/java/dev/caskeleton/messaging/kafka/KafkaPublishBenchmark.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/ContiguousPartitionOffsetTracker.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaBatchConsumerRegistrar.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaBrokerProfile.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrar.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaDeadLetterPublisher.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaDeliveryMapper.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaHeaderMapper.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaMessagingTransport.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaOffsetResetExecutor.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPartitionRetryScheduler.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPosition.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaProfileValidator.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifier.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishMapper.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaReplayCapability.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaReplayPlan.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaReplayPlanner.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryExecutor.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryMetadataMapper.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryOutcome.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryTopicPublisher.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaSecurityConfigurer.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaSettlementCommand.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaSettlementQueue.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTopologyInspector.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionProfileValidator.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalDelivery.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalOutput.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalProcessor.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalPublisher.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/PartitionOffsetTracker.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/PartitionWorkCoordinator.java create mode 100644 src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/SpringKafkaTransactionalProcessor.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/ContiguousPartitionOffsetTrackerTest.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaAmbiguityChaosIT.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaBrokerIT.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrarTest.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaConsumerSettlementIT.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaContainerFixture.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaContainerSmokeTest.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaContractHarness.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaFixtureProfiles.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaHeaderMapperTest.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaProducerContractTest.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaProfileValidatorTest.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaReadCommittedIT.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaReplayPlannerTest.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTopologyValidationIT.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTransactionFencingIT.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTransactionFixtures.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTransactionIT.java create mode 100644 src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/PartitionWorkCoordinatorTest.java create mode 100644 src/messaging/messaging-nats-experimental/build.gradle create mode 100644 src/messaging/messaging-nats-experimental/gradle.lockfile create mode 100644 src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsAckMode.java create mode 100644 src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsJetStreamProfile.java create mode 100644 src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsJetStreamProfileValidator.java create mode 100644 src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsJetStreamTransport.java create mode 100644 src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsMaxDeliverParkingWorkflow.java create mode 100644 src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsStreamPosition.java create mode 100644 src/messaging/messaging-nats-experimental/src/test/java/dev/caskeleton/messaging/nats/NatsAdapterContractTest.java create mode 100644 src/messaging/messaging-nats-experimental/src/test/java/dev/caskeleton/messaging/nats/NatsMaxDeliverParkingTest.java create mode 100644 src/messaging/messaging-observability/build.gradle create mode 100644 src/messaging/messaging-observability/gradle.lockfile create mode 100644 src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/CardinalityGuard.java create mode 100644 src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/DefaultMessagingObservationConvention.java create mode 100644 src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingAuditEvent.java create mode 100644 src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingAuditSink.java create mode 100644 src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingMetrics.java create mode 100644 src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingObservation.java create mode 100644 src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingRedactor.java create mode 100644 src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingTags.java create mode 100644 src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingTracer.java create mode 100644 src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingMetricCardinalityTest.java create mode 100644 src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingRedactorTest.java create mode 100644 src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingSecretLeakTest.java create mode 100644 src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingTraceLinkTest.java create mode 100644 src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/SecretLeakStaticScanTest.java create mode 100644 src/messaging/messaging-outbox-jpa/build.gradle create mode 100644 src/messaging/messaging-outbox-jpa/gradle.lockfile create mode 100644 src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumMappedRecord.java create mode 100644 src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxEventRouter.java create mode 100644 src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxProfile.java create mode 100644 src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapper.java create mode 100644 src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/JdbcOutboxRepository.java create mode 100644 src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxCleanupJob.java create mode 100644 src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxEnvelopeFactory.java create mode 100644 src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxProperties.java create mode 100644 src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelay.java create mode 100644 src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelayReport.java create mode 100644 src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxRetryScheduler.java create mode 100644 src/messaging/messaging-outbox-jpa/src/main/resources/db/migration/messaging/V1__messaging_outbox.sql create mode 100644 src/messaging/messaging-outbox-jpa/src/main/resources/debezium/outbox-event-router.properties create mode 100644 src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapperTest.java create mode 100644 src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/OutboxOperationsTest.java create mode 100644 src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/OutboxPostgresIT.java create mode 100644 src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/OutboxRelayTest.java create mode 100644 src/messaging/messaging-policy/build.gradle create mode 100644 src/messaging/messaging-policy/gradle.lockfile create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/BackoffCalculator.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/CapabilityTier.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/ConsumerPolicy.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterEnvelopeFactory.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterMetadata.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterOrchestrator.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterPolicy.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterResult.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DefaultRetryDecisionEngine.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DestinationProfile.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DestinationProfileValidator.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/FailureDescriptorDefaults.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/InFlightLimiter.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/MessagingAdmissionController.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/OrderingImpact.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/PayloadLimitGuard.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/PayloadPolicy.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/PhysicalDestination.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/ProducerPolicy.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryContext.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryDecision.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryDecisionEngine.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryMode.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryPolicy.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/SchemaPolicy.java create mode 100644 src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/SourceSettlement.java create mode 100644 src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/DeadLetterOrchestratorTest.java create mode 100644 src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/DestinationProfileValidatorTest.java create mode 100644 src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/MessagingAdmissionControllerTest.java create mode 100644 src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/RetryDecisionEngineTest.java create mode 100644 src/messaging/messaging-pulsar-experimental/build.gradle create mode 100644 src/messaging/messaging-pulsar-experimental/gradle.lockfile create mode 100644 src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarMessagePosition.java create mode 100644 src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarMessagingTransport.java create mode 100644 src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarProfile.java create mode 100644 src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarProfileValidator.java create mode 100644 src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarSubscriptionMode.java create mode 100644 src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarSubscriptionType.java create mode 100644 src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarTransactionCapability.java create mode 100644 src/messaging/messaging-pulsar-experimental/src/test/java/dev/caskeleton/messaging/pulsar/PulsarAdapterContractTest.java create mode 100644 src/messaging/messaging-pulsar-experimental/src/test/java/dev/caskeleton/messaging/pulsar/PulsarSubscriptionGuardTest.java create mode 100644 src/messaging/messaging-rabbit/build.gradle create mode 100644 src/messaging/messaging-rabbit/gradle.lockfile create mode 100644 src/messaging/messaging-rabbit/src/jmh/java/dev/caskeleton/messaging/rabbit/RabbitPublishBenchmark.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitBatchConsumerRegistrar.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitBrokerProfile.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitChannelPublisher.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitConfirmCoordinator.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitConsumerRegistrar.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitDeadLetterPublisher.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitDeliveryMapper.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitHeaderMapper.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitMessagingTransport.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitNativeDeadLetterCapability.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitProfileValidator.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitPublishFailureClassifier.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitPublishMapper.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitPublishReference.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitRequestReply.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitRetryQueueTopology.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitSecurityConfigurer.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitSettlementController.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitSettlementOperations.java create mode 100644 src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitTopologyProfile.java create mode 100644 src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitAdapterContractTest.java create mode 100644 src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitBrokerIT.java create mode 100644 src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitConfirmCoordinatorTest.java create mode 100644 src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitContractHarness.java create mode 100644 src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitFixtureProfiles.java create mode 100644 src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitProfileValidatorTest.java create mode 100644 src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitRuntimeTest.java create mode 100644 src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitSettlementControllerTest.java create mode 100644 src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitTopologyAndBatchTest.java create mode 100644 src/messaging/messaging-reliability-api/build.gradle create mode 100644 src/messaging/messaging-reliability-api/gradle.lockfile create mode 100644 src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/ClaimCheckReference.java create mode 100644 src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/IdempotentMessageHandler.java create mode 100644 src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/InboxRecord.java create mode 100644 src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/InboxRepository.java create mode 100644 src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/InboxResult.java create mode 100644 src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxRecord.java create mode 100644 src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxRepository.java create mode 100644 src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxStatus.java create mode 100644 src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/ReliableMessagePublisher.java create mode 100644 src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/TransactionalMessageAction.java create mode 100644 src/messaging/messaging-schema-api/build.gradle create mode 100644 src/messaging/messaging-schema-api/gradle.lockfile create mode 100644 src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/EncodedMessage.java create mode 100644 src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/MessageCodec.java create mode 100644 src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/MessageCodecRegistry.java create mode 100644 src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/RawBytesMessageCodec.java create mode 100644 src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaCompatibility.java create mode 100644 src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaCompatibilityValidator.java create mode 100644 src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaReference.java create mode 100644 src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaRegistry.java create mode 100644 src/messaging/messaging-schema-api/src/test/java/dev/caskeleton/messaging/schema/RawBytesMessageCodecTest.java create mode 100644 src/messaging/messaging-schema-api/src/test/java/dev/caskeleton/messaging/schema/SchemaCompatibilityValidatorTest.java create mode 100644 src/messaging/messaging-schema-avro/build.gradle create mode 100644 src/messaging/messaging-schema-avro/gradle.lockfile create mode 100644 src/messaging/messaging-schema-avro/src/main/java/dev/caskeleton/messaging/schema/avro/AvroCompatibilityGate.java create mode 100644 src/messaging/messaging-schema-avro/src/main/java/dev/caskeleton/messaging/schema/avro/AvroMessageCodec.java create mode 100644 src/messaging/messaging-schema-avro/src/test/java/dev/caskeleton/messaging/schema/avro/AvroCompatibilityTest.java create mode 100644 src/messaging/messaging-schema-avro/src/test/resources/schemas/order.created/v1.avsc create mode 100644 src/messaging/messaging-schema-json/build.gradle create mode 100644 src/messaging/messaging-schema-json/gradle.lockfile create mode 100644 src/messaging/messaging-schema-json/src/main/java/dev/caskeleton/messaging/schema/json/JacksonMessageCodec.java create mode 100644 src/messaging/messaging-schema-json/src/test/java/dev/caskeleton/messaging/schema/json/JacksonMessageCodecTest.java create mode 100644 src/messaging/messaging-schema-json/src/test/java/dev/caskeleton/messaging/schema/json/PlatformOverheadPerformanceTest.java create mode 100644 src/messaging/messaging-schema-protobuf/build.gradle create mode 100644 src/messaging/messaging-schema-protobuf/gradle.lockfile create mode 100644 src/messaging/messaging-schema-protobuf/src/main/java/dev/caskeleton/messaging/schema/protobuf/ProtobufMessageCodec.java create mode 100644 src/messaging/messaging-schema-protobuf/src/test/java/dev/caskeleton/messaging/schema/protobuf/ProtobufCompatibilityTest.java create mode 100644 src/messaging/messaging-schema-protobuf/src/test/proto/order_created_v1.proto create mode 100644 src/messaging/messaging-security/build.gradle create mode 100644 src/messaging/messaging-security/gradle.lockfile create mode 100644 src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerAclManifest.java create mode 100644 src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerCredentialProfile.java create mode 100644 src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerSecurityProfile.java create mode 100644 src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerTlsPolicy.java create mode 100644 src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialIds.java create mode 100644 src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialProvider.java create mode 100644 src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialRotationPlan.java create mode 100644 src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialRuntime.java create mode 100644 src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialRuntimeRegistry.java create mode 100644 src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/DestinationAccessPolicy.java create mode 100644 src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/DestinationAccessValidator.java create mode 100644 src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/MessageSecurityValidator.java create mode 100644 src/messaging/messaging-security/src/test/java/dev/caskeleton/messaging/security/CredentialRuntimeRegistryTest.java create mode 100644 src/messaging/messaging-security/src/test/java/dev/caskeleton/messaging/security/MessageSecurityValidatorTest.java create mode 100644 src/messaging/messaging-spring-boot-starter/build.gradle create mode 100644 src/messaging/messaging-spring-boot-starter/gradle.lockfile create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DefaultBatchMessagePublisher.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DefaultBlockingMessagePublisher.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DefaultReactiveMessagePublisher.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/KafkaMessagingAutoConfiguration.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingAdminAutoConfiguration.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingCoreAutoConfiguration.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpoint.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingProperties.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingReliabilityAutoConfiguration.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/PublishResults.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/RabbitMessagingAutoConfiguration.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/ReactiveMessagePublisher.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/ValidatedDestinationRegistry.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports create mode 100644 src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/BatchPublisherTest.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/BlockingFacadeTest.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingAutoConfigurationTest.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpointTest.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/ReactiveFacadeTest.java create mode 100644 src/messaging/messaging-spring-boot-starter/src/test/resources/application-invalid-ordering.yml create mode 100644 src/messaging/messaging-spring-boot-starter/src/test/resources/application-valid.yml create mode 100644 src/messaging/messaging-spring-cloud-stream-bridge/build.gradle create mode 100644 src/messaging/messaging-spring-cloud-stream-bridge/gradle.lockfile create mode 100644 src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/BindingCapabilityReport.java create mode 100644 src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/BindingProfileValidator.java create mode 100644 src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/MessagingBindingBridge.java create mode 100644 src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/SpringCloudStreamConsumerBridge.java create mode 100644 src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/SpringCloudStreamPublisherBridge.java create mode 100644 src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/StreamBridgePolicyGuard.java create mode 100644 src/messaging/messaging-spring-cloud-stream-bridge/src/test/java/dev/caskeleton/messaging/streambridge/BindingProfileValidatorTest.java create mode 100644 src/messaging/messaging-spring-cloud-stream-bridge/src/test/java/dev/caskeleton/messaging/streambridge/BridgePublishEvidenceTest.java create mode 100644 src/messaging/messaging-testkit/build.gradle create mode 100644 src/messaging/messaging-testkit/gradle.lockfile create mode 100644 src/messaging/messaging-testkit/src/jmh/java/dev/caskeleton/messaging/testkit/EnvelopeCodecBenchmark.java create mode 100644 src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/BrokerFailureMatrix.java create mode 100644 src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/CompatibilityMatrix.java create mode 100644 src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/ContractAssertions.java create mode 100644 src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/ContractMessage.java create mode 100644 src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/DockerAvailability.java create mode 100644 src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/FaultController.java create mode 100644 src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/HandleOutcome.java create mode 100644 src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/MessagingAdapterContract.java create mode 100644 src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/MessagingAdapterHarness.java create mode 100644 src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/NetworkFaultScenario.java create mode 100644 src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/ObservedDelivery.java create mode 100644 src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CompatibilityMatrixTest.java create mode 100644 src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CrossBrokerContractSuite.java create mode 100644 src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/InMemoryHarnessContractTest.java create mode 100644 src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/InMemoryMessagingHarness.java create mode 100644 src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/MessagingDocumentationContractTest.java create mode 100644 src/messaging/messaging-transport-spi/build.gradle create mode 100644 src/messaging/messaging-transport-spi/gradle.lockfile create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/BackpressureController.java create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/DefaultMessagingRuntimeRegistry.java create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/GracefulShutdownCoordinator.java create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingLifecycle.java create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingRuntime.java create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingRuntimeLease.java create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingRuntimeRegistry.java create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingTransport.java create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportConsumerRegistration.java create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportConsumerSpec.java create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportDelivery.java create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportPublishRequest.java create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportPublishResult.java create mode 100644 src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportSettlement.java create mode 100644 src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/BackpressureAndShutdownTest.java create mode 100644 src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/MessagingLifecycleTest.java create mode 100644 src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/MessagingRuntimeRegistryTest.java create mode 100644 src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/ResourceLeakGateTest.java diff --git a/docs/messaging/configuration-reference.md b/docs/messaging/configuration-reference.md new file mode 100644 index 00000000..06f345e9 --- /dev/null +++ b/docs/messaging/configuration-reference.md @@ -0,0 +1,156 @@ +# 설정 레퍼런스 + +## Destination profile + +```yaml +messaging: + destinations: + order-events: + broker: kafka-primary + kind: EVENT_STREAM # ASYNC_COMMAND | DOMAIN_EVENT | INTEGRATION_EVENT + # | WORK_QUEUE | PUBLISH_SUBSCRIBE | EVENT_STREAM | REQUEST_REPLY + tier: M1 # M1 | M2 | M3 + physical: + topic: order.events.v1 + schema: + codec: application/json + compatibility: BACKWARD_TRANSITIVE + message-types: [order.created] + guarantees: + delivery: AT_LEAST_ONCE # AT_MOST_ONCE | AT_LEAST_ONCE + ordering: KEY # NONE | DESTINATION | PARTITION | KEY + external-side-effect: INBOX_TRANSACTIONAL + producer: + confirmation: REPLICATION_OR_PERSISTENCE_ACK + timeout: 5s + mandatory-routing: true + idempotent: true + consumer: + group: order-projection + concurrency: 6 + max-in-flight-per-ordering-unit: 1 + prefetch: 16 + handler-timeout: 30s + manual-settlement: false + retry: + mode: PAUSE_PARTITION # NONE | INLINE | BLOCKING | PAUSE_PARTITION + # | RETRY_DESTINATION | BROKER_DELAYED + max-attempts: 3 + initial-delay: 200ms + max-delay: 2s + multiplier: 2.0 + jitter: true + ordering-impact: PRESERVE # PRESERVE | ALLOW_REORDER + dlq: + destination: order-events-dlq + max-redrive-count: 1 + payload: + max-bytes: 1048576 + claim-check-threshold-bytes: 1048576 + key-resolver-configured: true + production: true + topology-auto-create: false +``` + +## 기본값 + +| 설정 | 기본값 | 근거 | +|---|---:|---| +| logical payload 최대 | 1,048,576 bytes | portability. 초과는 Claim Check | +| global hard 최대 | 8,388,608 bytes | 어떤 destination도 넘을 수 없는 상한 | +| header 총 크기 | 32,768 bytes | | +| header 개수 | 64 | | +| header key | 128 bytes | metric tag 안전 | +| header value | 4,096 bytes | | +| publish timeout | 5s | | +| handler timeout | 30s | | +| graceful shutdown drain | 30s | | +| 일반 destination retry | 0회 | 자동 retry는 opt-in | +| DLQ redrive batch | 100 | 한 번의 작업이 source를 덮치지 않게 | +| Outbox relay batch | 100 | | +| Outbox lease | 30s | | +| Outbox polling | 500ms | | +| metric dimension 상한 | 200 | cardinality 폭발 방지 | + +## Broker profile + +### Kafka + +```yaml +messaging: + brokers: + kafka-primary: + type: kafka + stable: true + production: true + bootstrap-servers: [broker-1:9093, broker-2:9093] + enable-idempotence: true # stable에서 필수 + acks: all # stable에서 필수 + max-in-flight-requests-per-connection: 5 # 최대 5 + delivery-timeout: 30s + enable-auto-commit: false # 항상 금지 + tls-enabled: true # production 필수 + authentication-enabled: true # production 필수 +``` + +### RabbitMQ + +```yaml +messaging: + brokers: + rabbit-primary: + type: rabbitmq + stable: true + production: true + addresses: [rabbit-1:5671] + publisher-confirms: true # stable에서 필수 + publisher-returns: true # stable에서 필수 + mandatory: true # stable에서 필수 + confirm-timeout: 5s + auto-ack: false # 항상 금지 + prefetch: 16 + quorum-queues: true # durable work queue 필수 + tls-enabled: true + authentication-enabled: true +``` + +## 보안 + +```yaml +messaging: + security: + kafka-primary: + producer: { type: SASL_SCRAM, credential-id: kafka-producer } + consumer: { type: SASL_SCRAM, credential-id: kafka-consumer } + # admin은 application runtime에 설정하지 않는다 + hostname-verification: true + access: + publishable: [order-events] + consumable: [] + administrable: [] +``` + +## Experimental / Optional + +기본값은 전부 `false`다. + +```yaml +messaging: + experimental: + kafka-share: false + pulsar: false + nats: false + bridge: + spring-cloud-stream: false +``` + +## Backpressure + +```yaml +messaging: + backpressure: + global-limit: 512 + per-destination-limit: 64 # global-limit 이하여야 한다 +``` + +`per-destination-limit > global-limit`이면 global limit이 limit이 아니게 되므로 부팅에 실패한다. diff --git a/docs/messaging/delivery-guarantees.md b/docs/messaging/delivery-guarantees.md new file mode 100644 index 00000000..518aa3e2 --- /dev/null +++ b/docs/messaging/delivery-guarantees.md @@ -0,0 +1,75 @@ +# 전달 보장 + +## 왜 `EXACTLY_ONCE`가 없는가 + +어떤 브로커도 **외부 side effect를 포함한** exactly-once를 제공하지 않는다. +실제로 존재하는 것은 at-least-once 전달 + 멱등하거나 transactional한 consumer의 조합이다. + +플랫폼이 지킬 수 없는 이름을 enum에 두면 그 책임이 눈에 보이지 않는 곳으로 밀려난다. +그래서 `DeliveryGuarantee`는 증거가 끝나는 지점에서 멈춘다. + +```java +public enum DeliveryGuarantee { AT_MOST_ONCE, AT_LEAST_ONCE } +``` + +## Publish 결과는 boolean이 아니다 + +```java +public enum PublishCompletion { CONFIRMED, REJECTED, AMBIGUOUS } +``` + +`REJECTED`와 `AMBIGUOUS`를 하나의 "실패"로 합치면 중복 주문이 만들어진다. +전자는 broker가 저장하지 않았음이 **확정**되어 포기해도 안전하고, 후자는 그렇지 않다. + +| 상황 | 결과 | +|---|---| +| 로컬 validation 실패 | `REJECTED`, `NOT_TRANSMITTED` | +| broker 명시적 reject / nack | `REJECTED` | +| confirm 수신 | `CONFIRMED` | +| Rabbit confirm + unroutable return | `REJECTED`, `UNROUTABLE` | +| bytes 전송 후 connection loss | `AMBIGUOUS` | +| confirm timeout | `AMBIGUOUS` | +| adapter가 판정 불가 | 보수적으로 `AMBIGUOUS` | + +`PublishResult` 생성자가 이 규칙을 강제한다. `CONFIRMED`인데 broker acceptance가 없거나, +`AMBIGUOUS`인데 confirmation level을 주장하면 **객체 생성 자체가 실패**한다. + +## Ordering + +```java +public enum OrderingScope { NONE, DESTINATION, PARTITION, KEY } +``` + +순서는 partition·key·단일 consumer의 성질이지 destination 전체의 성질이 아니다. +`GLOBAL`이 없는 이유가 이것이다. + +`DestinationProfileValidator`가 다음을 거부한다. + +- `ordering=KEY`인데 key resolver 없음 +- ordered destination인데 `ALLOW_REORDER` retry +- `orderingImpact=PRESERVE`인데 재발행형 retry(`RETRY_DESTINATION`, `BROKER_DELAYED`) +- `ordering=DESTINATION`인데 concurrency > 1 +- ordered destination인데 ordering unit당 in-flight > 1 + +## External side effect + +```java +public enum ExternalSideEffectGuarantee { NONE, IDEMPOTENCY_REQUIRED, INBOX_TRANSACTIONAL } +``` + +`INBOX_TRANSACTIONAL`만이 "DB side effect와 중복 차단이 같은 transaction에서 commit된다"를 의미한다. +Kafka transaction은 **Kafka 안에서만** 원자적이므로 이 값과 함께 설정하면 +`KafkaTransactionProfileValidator`가 거부한다. 두 개의 독립적인 commit을 하나로 착각하게 두지 않기 위해서다. + +## Consumer settlement 순서 + +```text +RECEIVED → DECODING → PROCESSING → HANDLER_SUCCEEDED → SETTLEMENT_SENDING + ├→ SETTLED + └→ SETTLEMENT_UNKNOWN +``` + +- handler는 broker ACK API를 호출하지 않는다. +- `Success` 이후에만 source settlement한다. +- `SETTLEMENT_UNKNOWN`은 성공이 아니다. redelivery 가능성을 의미한다. +- `SettlementResult` 생성자가 `SETTLED`인데 `redeliveryPossible=true`인 조합을 거부한다. diff --git a/docs/messaging/experimental-policy.md b/docs/messaging/experimental-policy.md new file mode 100644 index 00000000..e2052c4b --- /dev/null +++ b/docs/messaging/experimental-policy.md @@ -0,0 +1,97 @@ +# Experimental 정책 + +## Stable과 Experimental의 차이 + +**Stable**은 공통 Contract Suite(`MessagingAdapterContract`)를 변경 없이 통과한 어댑터다. +컴파일되는 어댑터가 아니라, 아래 7가지를 실제로 증명한 어댑터다. + +```text +publishesAndConfirms +returnsAmbiguousWhenConfirmIsLost +redeliversWhenSettlementIsLost +preservesMessageIdAcrossRetryAndDlq +keepsSourceUnsettledWhenDlqPublishFails +rejectsOversizedPayloadBeforeTransport +stopsAcceptingNewWorkDuringShutdown +``` + +**Experimental**은 아직 그 증명이 끝나지 않은 어댑터다. + +## 규칙 + +### 1. 기본 비활성 + +```yaml +messaging.experimental.kafka-share: false +messaging.experimental.pulsar: false +messaging.experimental.nats: false +``` + +활성화하지 않으면 validator가 `MessagingCapabilityUnavailableException`을 던진다. +Contract Suite가 아직 증명 중인 어댑터가 누군가의 기본 설정 때문에 load-bearing이 되어서는 안 된다. + +### 2. Stable 모듈이 Experimental 모듈에 의존하지 않는다 + +Gradle 의존 그래프로 강제된다. `messaging-spring-boot-starter`의 `allowed_dependencies`에 +`messaging-kafka-share-experimental`, `messaging-pulsar-experimental`, +`messaging-nats-experimental`, `messaging-spring-cloud-stream-bridge`가 **없다**. + +`verifyCleanArchitectureDependencies`가 위반을 빌드 실패로 만든다. + +### 3. Core 계약을 바꾸지 않는다 + +Experimental 어댑터는 브로커의 차이를 `MessagingCapabilities`로 표현할 뿐, +`messaging-core-api`의 타입을 바꾸지 않는다. + +### 4. 없는 기능을 광고하지 않는다 + +| 어댑터 | 광고하지 않는 것 | 이유 | +|---|---|---| +| Kafka Share Group | orderedStream, keyedOrdering, replay, brokerTransaction | 경쟁 소비자 + 개별 ack는 partition 순서를 유지할 수 없다 | +| Pulsar | brokerTransaction | Pulsar에 있지만 플랫폼 Contract Suite로 증명되지 않았다 | +| Pulsar (Shared) | keyedOrdering | round-robin 분배 | +| NATS JetStream | nativeDeadLetter | delivery limit 초과 시 terminate할 뿐 라우팅하지 않는다 | +| NATS JetStream | keyedOrdering | subject 기반 모델에 per-key 순서가 없다 | + +`false`인 capability를 요구하는 profile은 startup에서 실패한다. +조용히 약화되지 않는다. + +### 5. 명시적 거부 + +| 조합 | 결과 | +|---|---| +| Kafka Share Group + ordering != NONE | 거부 | +| Kafka Share Group + pause/resume | `MessagingCapabilityUnavailableException` | +| Pulsar Shared + ordering=KEY | 거부 (Key_Shared 필요) | +| Pulsar + ordering=DESTINATION | 거부 | +| NATS Core + AT_LEAST_ONCE | 거부 (JetStream 필요) | +| NATS ordered consumer + 경쟁 워커 > 1 | 거부 | +| NATS + ordering=KEY | 거부 | + +## Spring Cloud Stream bridge + +Experimental이 아니라 **Optional**이다. 위험이 다르다. + +Stream은 자체 binder 설정을 소유하므로, binding이 destination profile이 모르는 +serializer·error handling·acknowledgement mode를 조용히 획득할 수 있다. + +따라서 브리지는 **플랫폼 보장에 의존하지 않는 destination에만** 허용한다. + +```text +ordering scope 선언 → 거부 +retry policy 선언 → 거부 +dead letter 선언 → 거부 +``` + +이 셋 중 하나라도 필요하면 native adapter를 쓴다. 거기서만 실제로 강제되기 때문이다. + +## 승격 조건 + +Experimental → Stable로 올리려면 전부 필요하다. + +1. `MessagingAdapterContract` 7개 테스트를 변경 없이 통과 +2. 장애 주입(연결 끊김, confirm 유실, settlement 유실) 하에서 통과 +3. 지원 브로커 버전 범위 명시 및 CI 검증 +4. `support-matrix.md`의 capability 표 갱신 +5. ADR 작성 +6. 기본 활성화 여부에 대한 별도 결정 diff --git a/docs/messaging/migration-guide.md b/docs/messaging/migration-guide.md new file mode 100644 index 00000000..1b849a35 --- /dev/null +++ b/docs/messaging/migration-guide.md @@ -0,0 +1,113 @@ +# 마이그레이션 가이드 + +## 기존 Spring Kafka / Spring AMQP 코드에서 + +### 1. topic 이름을 코드에서 제거한다 + +```java +// before +kafkaTemplate.send("order.events.v1", key, payload); + +// after +publisher.publish(orderEvents, envelope, PublishOptions.defaults()); +``` + +`MessageDestination`은 logical name만 가진다. 물리 매핑은 destination profile이 소유한다. +`DestinationName`의 패턴이 `topic://orders` 같은 값을 거부하므로 우회할 수 없다. + +### 2. boolean 성공 판정을 없앤다 + +```java +// before +try { template.send(...).get(); success(); } +catch (Exception e) { fail(); } // REJECTED와 AMBIGUOUS를 구분하지 못한다 + +// after +PublishResult result = ...; +switch (result.completion()) { + case CONFIRMED -> success(); + case REJECTED -> abandon(); // broker가 저장하지 않음이 확정 + case AMBIGUOUS -> retrySameMessageId(result); // broker가 가지고 있을 수 있음 +} +``` + +이 구분이 없으면 confirm 유실 한 번이 중복 주문 하나가 된다. + +### 3. auto-commit / auto-ack를 끈다 + +```yaml +# Kafka +enable.auto.commit: false +# RabbitMQ +auto-ack: false +``` + +둘 다 validator가 강제로 거부한다. 타이머 기반 commit은 handler가 실행되기도 전에 +메시지를 처리 완료로 표시한다. + +### 4. handler에서 ack 호출을 제거한다 + +```java +// before +@KafkaListener(...) +void handle(ConsumerRecord record, Acknowledgment ack) { + process(record); + ack.acknowledge(); // 실패 시 순서가 애매해진다 +} + +// after +CompletionStage handle(MessageDelivery delivery) { + process(delivery.message().payload()); + return completedFuture(HandleResult.success()); +} +``` + +settlement는 플랫폼이 수행한다. "성공한 뒤에만 ack"가 각 handler의 기억이 아니라 +플랫폼 불변식이 된다. + +### 5. 중복을 정상 상황으로 다룬다 + +at-least-once는 중복을 전제한다. 세 가지 중 하나를 고른다. + +| 방식 | 언제 | +|---|---| +| handler 자체 멱등 | 자연 멱등 연산 (upsert 등) | +| Inbox | DB side effect가 있는 경우 | +| Kafka transaction | Kafka → Kafka 파이프라인만 | + +`ExternalSideEffectGuarantee`에 선언한다. `INBOX_TRANSACTIONAL`과 Kafka transaction을 +동시에 설정하면 거부된다. Kafka transaction은 DB를 포함하지 않는다. + +### 6. 큰 payload는 Claim Check로 + +broker frame 크기를 키우지 않는다. broker 메모리, replication latency, +consumer recovery가 동시에 나빠지고, 유계·검증 가능한 실패가 무계 실패로 바뀐다. + +1 MiB 초과는 외부 저장소로 offload하고 digest를 포함한 참조만 발행한다. + +## DB 마이그레이션 + +```text +V1__messaging_outbox.sql +V2__messaging_inbox.sql +``` + +Outbox row는 business transaction과 같은 transaction에서 쓴다. +Inbox reservation은 handler side effect와 같은 transaction에서 쓴다. +별도 transaction이면 각 패턴이 닫으려던 창이 그대로 열려 있다. + +## 단계적 전환 + +1. **publish만 전환** — 기존 consumer는 그대로. wire format은 reserved header가 추가될 뿐이다. +2. **Outbox 도입** — publish 유실 창을 닫는다. +3. **consume 전환** — handler를 `MessageHandler`로 옮기고 ack 호출을 제거한다. +4. **Inbox 도입** — 중복 side effect를 닫는다. +5. **retry·DLQ 정책 선언** — 이 시점까지 자동 retry는 0회다. + +각 단계는 독립적으로 배포 가능하고, 되돌릴 수 있다. + +## 되돌릴 수 없는 것 + +- 한 번 발행된 message type의 wire contract +- 이미 retention 안에 있는 메시지의 schema +- redrive된 메시지의 `messageId` (바뀌지 않는다 — 이것이 의도다) diff --git a/docs/messaging/operations.md b/docs/messaging/operations.md new file mode 100644 index 00000000..e5d5bf61 --- /dev/null +++ b/docs/messaging/operations.md @@ -0,0 +1,113 @@ +# 운영 Runbook + +## 배포 전 체크 + +```bash +./gradlew verifyCleanArchitectureDependencies --console=plain +./gradlew verifyRuntimeModuleMembership --console=plain +./gradlew verifyOneTypePerFile --console=plain +``` + +destination profile은 startup에서 검증된다. 아래는 **부팅 실패**다. + +- ordered destination + reorder 가능 retry +- `ordering=KEY` + key resolver 없음 +- payload 상한 > 8,388,608 bytes +- DLQ 자기 참조 / retry 자기 참조 +- retry·DLQ 그래프 cycle +- 미등록 retry·DLQ destination +- M1 destination + manual settlement +- `AT_LEAST_ONCE` + confirmation `NONE` +- production profile + topology auto-create +- broker topology가 manifest와 불일치 + +## 증상별 대응 + +### publish가 AMBIGUOUS로 쏟아진다 + +broker confirm 경로 문제다. 실패가 아니다. + +1. `PublishEvidence.transmission`이 `MAY_HAVE_BEEN_TRANSMITTED`인지 확인 +2. Kafka: `delivery.timeout.ms`, ISR 상태, leader election 확인 +3. Rabbit: confirm timeout, channel 상태 확인 +4. Outbox를 쓰고 있다면 `status='AMBIGUOUS'` row가 같은 messageId로 재시도 중이다. **정상이다.** +5. consumer 쪽 Inbox가 중복을 흡수하는지 확인 + +`AMBIGUOUS`를 실패로 취급해 새 messageId로 재발행하지 말 것. 중복이 복구 불가능해진다. + +### DLQ가 비어 있는데 메시지가 사라졌다 + +DLQ publish 실패 시 source는 settlement되지 않는다. 메시지는 source에 남아 재전달된다. + +1. `msg.failure-code`가 `DEAD_LETTER_*`인 로그 확인 +2. DLQ destination이 실제로 존재하는지 (topology validation) +3. DLQ credential에 publish 권한이 있는지 + +### consumer lag이 한 partition에서만 증가한다 + +`ContiguousPartitionOffsetTracker`가 gap에서 멈춘 것이다. 설계된 동작이다. + +commit은 **연속** 완료 offset까지만 전진한다. offset 11이 아직 실행 중이면 +10과 12가 끝나도 watermark는 10에 머문다. 12를 commit하면 consumer가 죽었을 때 11을 잃는다. + +1. 해당 partition의 in-flight를 확인 +2. 느린 handler를 찾는다 (`handlerTimeout` 초과 여부) +3. 필요하면 `PAUSE_PARTITION` retry가 걸려 있는지 확인 + +### 재시도 폭풍 + +`RetryPolicy.jitter=false`인지 확인한다. jitter 없이는 같은 초에 실패한 모든 consumer가 +같은 초에 재시도한다. + +### shutdown이 오래 걸린다 + +`GracefulShutdownCoordinator`가 in-flight를 기다리는 중이다. + +- `inFlight()`가 0이 되면 즉시 종료 +- drain deadline(기본 30초) 초과 시 남은 작업을 **unsettled로 포기**한다 → broker가 재전달 +- draining 시작 후 새 retry attempt는 만들지 않는다 + +## Destructive 작업 + +전부 `DestructiveOperationGuard`를 통과해야 한다. + +| 조건 | 요구 | +|---|---| +| admin credential | application runtime은 보유하지 않음 | +| `AdminApproval` | 유효기간 내 | +| dry-run | 항상 허용 | + +### Replay + +```text +기본: 격리된 consumer group (replay-) +기존 group 대상: 승인 티켓 필수 +``` + +기존 production group으로 replay하는 것은 "다시 읽기"가 아니라 **live consumer를 되감는 것**이다. +그 사이의 모든 것이 재처리된다. + +### Redrive + +```text +dry-run으로 후보 수 확인 +→ 승인 획득 +→ batch 100건 이하로 실행 +→ republish CONFIRMED 인 것만 DLQ에서 settlement +``` + +`redriveId`로 재구동 루프를 추적한다. 같은 메시지가 반복해서 redrive되면 +근본 원인이 해결되지 않은 것이다. + +### Offset reset + +`KafkaOffsetResetExecutor`는 승인 predicate를 **생성자 인자**로 받는다. +승인 소스 없이 조립된 runtime은 물리적으로 reset을 수행할 수 없다. + +## Topology + +production topology는 IaC가 만들고 애플리케이션은 **검증만** 한다. + +`TopologyValidationRuntime`은 모든 불일치를 한 번에 보고하고 startup을 실패시킨다. +partition 수가 다르면 destination이 광고하는 ordering 보장이 달라지고, +`min.insync.replicas`가 없으면 `acks=all`의 의미가 달라진다. diff --git a/docs/messaging/outbox-inbox.md b/docs/messaging/outbox-inbox.md new file mode 100644 index 00000000..a3813cab --- /dev/null +++ b/docs/messaging/outbox-inbox.md @@ -0,0 +1,95 @@ +# Outbox · Inbox + +## 두 패턴이 각각 무엇을 해결하는가 + +| 패턴 | 해결하는 문제 | 해결하지 않는 문제 | +|---|---|---| +| Transactional Outbox | DB commit과 publish 사이의 창(窓) | 중복 | +| Inbox | 중복 delivery의 side effect | 유실 | + +**둘 다 필요하다.** Outbox만으로는 exactly-once가 되지 않는다. + +## Outbox + +business transaction과 **같은 transaction**에서 row를 쓴다. 둘 다 commit되거나 둘 다 안 된다. + +```sql +BEGIN; + UPDATE orders SET status = 'PLACED' WHERE id = ?; + INSERT INTO messaging_outbox (message_id, destination, ...) VALUES (?, ?, ...); +COMMIT; +``` + +### relay + +```text +leaseBatch(100, 30s) -- lease로 다중 relay 인스턴스 안전 +→ publish (messageId 그대로) +→ CONFIRMED → markPublished +→ AMBIGUOUS → markAmbiguous (같은 messageId로 재시도 가능) +→ REJECTED → markFailed +``` + +### 핵심 규칙: ambiguous는 같은 messageId로 재시도 + +새 id를 발급하면 "전달됐을 수도 있는 메시지"가 "확실히 두 번째인 메시지"가 되어 +downstream의 어떤 중복 제거도 복구할 수 없다. +failed로 표시하면 broker가 이미 가지고 있을 수 있는 메시지를 잃는다. + +`message_id`를 primary key로 둔 것도 같은 이유다. 어떤 코드 경로도 실수로 새 id를 붙일 수 없다. + +### lease + +```text +status IN ('PENDING','AMBIGUOUS') AND (lease_expires_at IS NULL OR lease_expires_at <= now) +``` + +partial index `ix_messaging_outbox_claimable`이 이 쿼리를 backlog 크기에 비례하게 유지한다. +PUBLISHED row는 retention job이 지울 때까지 쌓이기 때문이다. + +## Inbox + +reservation과 side effect가 **같은 transaction**이어야 한다. + +```java +transactions.inTransaction(() -> { + if (!inbox.reserve(messageId, consumerId, now)) { + return InboxOutcome.duplicate(); // 이미 처리됨 + } + return InboxOutcome.processed(sideEffect.get()); +}); +``` + +별도 transaction으로 예약하면 Inbox가 닫으려던 바로 그 창이 다시 열린다. + +### 복합 키 + +`PRIMARY KEY (message_id, consumer_id)`. + +message_id만으로 중복 제거하면 같은 event를 소비하는 두 번째 consumer가 +첫 번째에 의해 억제된다. 각 consumer가 한 번씩 처리해야 한다. + +### retention + +broker의 최대 redelivery window보다 **길어야** 한다. +row를 먼저 지우면 늦게 도착한 redelivery가 두 번 처리된다. + +## Debezium CDC 대안 + +polling relay 대신 WAL을 읽는다. polling interval과 lease 경합이 사라지지만 +인프라와 그 자체의 실패 모드가 추가된다. + +wire contract는 동일하다. `DebeziumOutboxEventRouter`가 polling relay와 같은 reserved header를 +방출하므로 consumer는 어느 쪽이 발행했는지 구분할 수 없고, 전환은 배포 결정일 뿐 계약 변경이 아니다. + +## Claim Check + +1 MiB 초과 payload는 broker 프레임을 키우지 않고 외부 저장소로 offload한다. + +`ClaimCheckReference`는 digest를 **필수**로 가진다. claim check는 메시지를 서로 다른 retention과 +replication을 가진 두 시스템으로 쪼개므로, consumer는 producer가 저장한 바로 그 bytes를 받았음을 +증명할 수 있어야 한다. 그렇지 않으면 잘린 객체와 정상 객체를 구분할 수 없다. + +`ClaimCheckIntegrityGuard`는 fetch 전에 만료를, fetch 후에 크기와 digest를 검사한다. +digest 불일치는 `DESERIALIZATION`이 아니라 **validation** 실패로 분류한다. +bytes가 깨진 JSON인 게 아니라, 틀린 bytes이기 때문이다. diff --git a/docs/messaging/retry-dlq-redrive.md b/docs/messaging/retry-dlq-redrive.md new file mode 100644 index 00000000..39355e65 --- /dev/null +++ b/docs/messaging/retry-dlq-redrive.md @@ -0,0 +1,103 @@ +# Retry · DLQ · Redrive + +## 자동 retry는 opt-in이다 + +일반 destination의 기본값은 **retry 없음**이다. 순서를 깨거나, 멱등하지 않은 side effect를 +증폭시키거나, 이미 throttle된 downstream을 더 때리는 retry는 보이는 실패보다 나쁘다. + +## 결정 순서 + +`DefaultRetryDecisionEngine`은 아래 순서를 위에서 아래로 평가한다. + +```text +1. non-retryable category → parking(DeadLetter) 또는 Reject +2. attempt >= maxAttempts → DeadLetter +3. PRESERVE + ordered + orderedStream capability → PauseAndRetry +4. mode=PAUSE_PARTITION → PauseAndRetry +5. mode=RETRY_DESTINATION + ALLOW_REORDER → PublishToRetryDestination +6. mode=INLINE|BLOCKING → RetryInline +7. mode=BROKER_DELAYED + delayedDelivery capability → PublishToRetryDestination +8. 그 외 → DeadLetter +``` + +**retryability를 attempt 예산보다 먼저** 검사한다. deserialization 실패는 payload가 바뀌지 않으므로 +재시도가 3번 더 실패할 뿐이다. 첫 delivery에서 바로 park한다. + +**순서 보존 전략을 재발행 전략보다 먼저** 검사한다. 둘 다 설정되어 있어도 ordered destination이 +reorder 경로로 흘러내리지 않는다. + +## 기본 non-retryable + +`DESERIALIZATION`, `AUTHENTICATION`, `AUTHORIZATION`, `CONFIGURATION`은 자동 retry하지 않는다. +매 redelivery마다 동일하게 실패하므로 부하만 늘어난다. +destination profile의 `retryableCategories`로 명시적으로 뒤집을 수는 있다. + +## Backoff + +`min(maxDelay, initialDelay * multiplier^(attempt-1))`, 이후 full jitter. + +full jitter는 `[0, delay]` 균등 분포다. jitter가 없으면 같은 초에 실패한 모든 consumer가 +같은 초에 재시도하고, downstream의 회복이 재시도 폭풍으로 즉시 무효화된다. + +## Kafka: pause-and-seek vs retry topic + +| 전략 | 순서 | 언제 | +|---|---|---| +| `PAUSE_PARTITION` | 유지 | ordered destination | +| `RETRY_DESTINATION` | 깨짐 | work queue, `ALLOW_REORDER` 명시 | + +pause-and-seek는 메시지가 로그의 자기 자리를 떠나지 않는다. partition을 멈추고, 기다리고, +같은 offset으로 seek해 재전달한다. 뒤의 메시지도 함께 기다리며 이것이 의도된 동작이다. + +## RabbitMQ: delayed retry queue + +core broker에 per-message delay가 없으므로 **TTL + DLX**로 구현한다. +retry queue의 `x-message-ttl`이 만료되면 `x-dead-letter-exchange`를 통해 work queue로 되돌아간다. + +주의: TTL 만료는 큐 **head**에서 평가된다. 하나의 retry queue에 서로 다른 delay를 섞으면 +독립적으로 만료되지 않는다. + +`basic.nack(requeue=true)`는 사용하지 않는다. delay 없이 큐 head로 되돌리므로 hot loop가 된다. + +## DLQ: publish 확인 후 settlement + +이것이 dead lettering이 데이터 손실이 되지 않게 하는 **유일한** 불변식이다. + +```text +DLQ envelope 생성 (원래 messageId 유지) +→ DLQ publish +→ CONFIRMED 이면 source settlement +→ REJECTED / AMBIGUOUS 이면 source를 settlement하지 않음 +``` + +source를 먼저 ACK하면, DLQ publish가 실패했을 때 메시지의 사본이 **어디에도 남지 않는다**. +broker는 이미 해제했고 DLQ는 받지 못했다. + +AMBIGUOUS DLQ publish는 중복을 만든다. 이것이 의도된 trade다. DLQ는 사람이 읽는 곳이고 +중복은 알아볼 수 있지만, 손실은 복구할 수 없다. + +## DLQ envelope 내용 + +reserved header에만 기록한다. payload에 넣지 않는다. + +```text +msg.failure-category, msg.failure-code, msg.origin-destination, +msg.retry-attempt, msg.first-failure-at, msg.last-failure-at +``` + +stack trace, exception message, secret header, 실제 key는 **넣지 않는다**. +DLQ는 원본 topic보다 오래 보관되고 더 많은 사람이 읽는다. + +## Redrive + +M4 Admin 전용이다. `DestructiveOperationGuard`를 통과해야 한다. + +- admin credential 필요 (application runtime은 보유하지 않는다) +- 유효기간 내 `AdminApproval` 필요 +- dry-run은 항상 허용 (계획이 공짜여야 사람이 계획한다) +- batch 상한 100건 +- source == target 금지 +- `redriveId`는 `messageId`와 별개다. 재구동 루프를 식별하기 위해서다. + +redrive도 **publish → settlement** 순서다. republish가 confirm되지 않은 메시지는 +DLQ에 남는다. diff --git a/docs/messaging/security.md b/docs/messaging/security.md new file mode 100644 index 00000000..c6dc3923 --- /dev/null +++ b/docs/messaging/security.md @@ -0,0 +1,92 @@ +# Messaging 보안 + +## Credential 분리 + +producer / consumer / admin은 **서로 다른 credential**이다. +`MessageSecurityValidator`가 startup에서 강제한다. + +```text +producer credential == consumer credential → 실패 +admin credential == producer|consumer → 실패 +production 프로필에 admin credential 존재 → 실패 +``` + +마지막 규칙이 "애플리케이션은 topic을 purge할 수 없다"를 **구조적으로** 만든다. +runtime이 admin 자격 증명을 아예 보유하지 않으므로, 침해된 handler가 상승시킬 대상이 없다. + +## Production 필수 조건 + +- TLS 활성 +- TLS hostname verification 활성 +- broker authentication 활성 +- topology auto-create 비활성 + +Kafka는 추가로 `enable.idempotence=true`, `acks=all`, +`max.in.flight.requests.per.connection <= 5`, consumer auto-commit 금지. + +RabbitMQ는 추가로 publisher confirm, publisher return, `mandatory=true`, +durable work queue의 quorum queue, consumer auto-ack 금지. + +## Credential은 값이 아니라 참조다 + +`BrokerCredentialProfile`의 어떤 variant도 secret을 담지 않는다. +식별자만 보관하고 connect 시점에 `CredentialProvider`로 해석한다. +heap dump나 설정 출력에서 사용 가능한 credential이 나오지 않는다. + +`CredentialIds`는 `bearer `, `sk-`, `-----begin`, `eyJ` 같은 접두사를 거부한다. +참조가 들어갈 자리에 secret 자체를 붙여넣는 가장 흔한 사고를 막는다. + +## Rotation + +`CredentialRotationPlan.isDue()`는 만료 **전에** 참이 된다. +broker가 연결을 거부하기 시작한 시점에는 이미 publish가 실패하고 consumer가 멈춰 있다. + +rotation은 세대 교체다. `DefaultMessagingRuntimeRegistry.install()`이 새 세대를 원자적으로 +게시하고, 이전 세대는 마지막 lease가 닫힐 때까지 열려 있다가 닫힌다. +진행 중인 publish는 시작한 연결에서 confirm을 받는다. + +drain deadline이 이 대기를 제한한다. 없으면 lease 하나가 새면 폐기된 credential이 +무기한 열려 있고, rotation이 보안상 무의미해진다. + +## Header + +금지 header는 application·platform 양쪽에서 거부한다. + +```text +Authorization, Proxy-Authorization, Cookie, Set-Cookie, +access_token, refresh_token, api_key, password, client_secret +``` + +credential이 header에 들어가면 broker storage, DLQ dump, 운영 도구에 남는다. +downstream redaction으로는 되돌릴 수 없다. + +예약 header(`msg.*`, `traceparent`, `tracestate`, `baggage`)는 platform만 쓴다. +application이 `msg.id`를 설정할 수 있으면 Inbox 중복 제거와 DLQ 상관관계가 의존하는 +logical identity가 호출자 제어가 된다. + +## ACL + +`DestinationAccessValidator`가 broker ACL **이전에** 검사한다. +broker ACL 거부는 애플리케이션 컨텍스트가 없는 연결 수준 오류로 도착하므로 +"어느 모듈이 어디에 publish하려 했는가"가 조사 대상이 된다. + +## 관측성 누출 + +`MessagingRedactor`는 denylist다. + +- secret: authorization, cookie, token, password, secret, credential +- per-message identity: messageId, correlationId, causationId, partitionKey, key, offset, deliveryTag, sequence +- payload: payload, body, data +- 예외 상세: exceptionMessage, stackTrace + +identity를 지우는 이유는 두 가지다. bounded metric을 message당 하나의 series로 만들고, +support log를 재식별 표면으로 만들기 때문이다. + +`CardinalityGuard`는 dimension당 값 개수를 상한한다. +cardinality 사고는 점진적이지 않다. 테스트 10건에서는 멀쩡하고 운영에서 백엔드를 죽인다. + +## 감사 + +`MessagingAuditEvent`는 replay, redrive, offset reset, purge, delete를 기록한다. +subject(운영자 identity), approval ticket, 그리고 redactor를 통과한 details만 담는다. +누가 무엇을 했는지 증명하되 payload의 두 번째 사본이 되지 않는다. diff --git a/docs/messaging/support-matrix.md b/docs/messaging/support-matrix.md new file mode 100644 index 00000000..dc9356c3 --- /dev/null +++ b/docs/messaging/support-matrix.md @@ -0,0 +1,116 @@ +# Messaging 지원 매트릭스 + +플랫폼이 **무엇을 보장하는지**와 **무엇을 보장하지 않는지**를 브로커별로 고정한다. +여기 없는 조합은 지원되지 않는다. + +## 브로커 등급 + +| 브로커 | 등급 | 인증 기준 | Stable 기능 | 제한 | +|---|---|---|---|---| +| Kafka | Stable | 4.2+ / 4.3.x | producer idempotence, consumer group, batch, pause/resume, replay, transaction capability | Share Group은 Experimental | +| RabbitMQ | Stable | 4.3.x | exchange/routing, publisher confirm, mandatory return, manual ACK, quorum queue, retry queue, DLQ | stream 및 특수 plugin 미지원 | +| Pulsar | Experimental | 4.0 LTS + 4.2 | typed publish/consume, Shared, Key_Shared, schema | transaction 미승격, 기본 비활성 | +| NATS JetStream | Experimental | 2.14.x | stream, durable consumer, explicit ACK, dedupe, replay | native DLQ 없음(플랫폼이 대행), 기본 비활성 | +| Artemis/JMS | Extension | 범위 밖 | adapter SPI만 | 별도 ADR + Contract Suite 통과 필요 | + +## Capability 매트릭스 + +`MessagingCapabilities`가 런타임에 선언하는 값이다. `false`인 기능을 요구하는 destination profile은 +**startup에서 실패**하며, 조용히 약화되지 않는다. + +| Capability | Kafka | Kafka Share | RabbitMQ | Pulsar | NATS JS | +|---|---|---|---|---|---| +| brokerAcknowledgement | O | O | O | O | O | +| replicationOrPersistenceEvidence | O | O | O | O | O | +| perMessageSettlement | O | O | O | O | O | +| batchSettlement | O | X | X | O | O | +| orderedStream | O | **X** | X | X | O | +| keyedOrdering | O | **X** | X | Key_Shared만 | X | +| replay | O | X | X | O | O | +| delayedDelivery | X | X | retry queue로 대행 | O | X | +| brokerTransaction | O | X | X | 미승격 | X | +| deduplicatedPublish | O | X | X | X | O | +| nativeDeadLetter | X | X | O | O | **X** | +| topologyManagement | O | X | O | O | O | + +Kafka Share Group이 ordering 전부 `X`인 것은 설계 결정이다. share group은 개별 record를 +경쟁 소비자에게 나눠주고 개별 ack하므로 partition 순서를 유지할 수 없다. ordered destination을 +share group에 설정하면 `KafkaShareProfileValidator`가 거부한다. + +NATS JetStream의 `nativeDeadLetter=X`도 마찬가지다. JetStream은 delivery limit 초과 시 메시지를 +**terminate**할 뿐 어디로도 라우팅하지 않으므로, 플랫폼이 DLQ publish를 직접 수행한다. + +## 기능 등급 + +| 기능 | 등급 | +|---|---| +| Typed Publish·Consume | Stable M1 | +| At-least-once contract | Stable | +| Ambiguous publish 결과 | Stable | +| handler 성공 후 자동 settlement | Stable M1 | +| Batch / Manual settlement / Pause·Resume / Delayed / Replay 요청 | M2 | +| Broker transaction / partition / routing / subscription | M3 | +| Replay 실행 / Redrive / offset reset / purge / delete | M4 Admin | +| Kafka Share Group, Pulsar, NATS | Experimental | +| Spring Cloud Stream bridge | Optional | + +## 무엇이 "Stable"을 증명하는가 + +Stable 등급은 두 가지를 **모두** 통과해야 한다. `CompatibilityMatrixTest`가 이 규칙을 강제한다. + +### 1. 공유 Contract Suite (`MessagingAdapterContract`, 7개) + +Kafka와 RabbitMQ가 동일한 7개 테스트를 변경 없이 통과한다. 결정적 하네스를 쓰므로 +확인 유실·settlement 유실 같은 장애를 요청 시점에 재현할 수 있다. + +### 2. 실 브로커 인증 (Testcontainers) + +| 스위트 | 무엇을 증명하는가 | +|---|---| +| `KafkaBrokerIT` | `acks=all`이 실제 replication 증거를 만든다 / 잘못된 토픽은 `REJECTED` / 발행-소비 왕복에서 identity 보존 및 contiguous commit | +| `KafkaAmbiguityChaosIT` | 브로커를 `docker pause`로 멈춘 상태의 publish가 **`AMBIGUOUS`** 로 보고된다 (broker acceptance 없음, confirmation level `NONE`, 비-retryable) | +| `RabbitBrokerIT` | exchange가 confirm했는데 어떤 큐에도 바인딩되지 않은 publish가 **`REJECTED` + `UNROUTABLE`** 로 보고된다 | +| `OutboxPostgresIT` | 롤백된 트랜잭션은 발행 가능한 행을 남기지 않는다 / `SKIP LOCKED` lease가 두 relay를 분리한다 / ambiguous 행이 같은 `messageId`로 재클레임된다 | +| `InboxPostgresIT` | 재전달이 side effect를 두 번 적용하지 않는다 / 롤백은 예약도 되돌린다 | + +Docker가 없으면 `DockerAvailability` 가드로 skip되며, 이 표의 항목은 그때 **검증되지 않은 것**으로 취급한다. + +### 3. 장애 시나리오 커버리지 (`BrokerFailureMatrix`) + +`NetworkFaultScenario`가 5개 시나리오와 **각각의 기대 결과**를 코드로 고정한다. 기대 결과를 어댑터별로 +두지 않는 것이 핵심이다 — 어댑터마다 다른 답을 허용하면 공유 계약이 존재할 이유가 없다. + +| 시나리오 | 시점 | 기대 결과 | 이유 | +|---|---|---|---| +| `connection-refused` | 전송 전 | `REJECTED` | 바이트가 나가지 않았으므로 broker가 가질 수 없다 | +| `connection-cut-after-write` | 전송 후 | `AMBIGUOUS` | broker가 저장했고 confirm만 유실됐을 수 있다 | +| `confirm-timeout` | 전송 후 | `AMBIGUOUS` | timeout은 부재의 증거가 아니라 증거의 부재다 | +| `settlement-lost` | settlement 중 | `REDELIVERED` | 미settlement 메시지는 재전달이 설계다 | +| `high-latency` | 전송 후 | `AMBIGUOUS` | 판단 시점에는 confirm 유실과 구별할 수 없다 | + +`CrossBrokerContractSuite`가 릴리스 게이트로 이를 강제한다. Stable 어댑터는 5개 전부를 **실 브로커에서** +커버해야 하고, Experimental 어댑터는 `LIVE_BROKER` 커버리지를 주장할 수 없다. 커버리지는 *능력*이 아니라 +*무엇을 실제로 돌렸는지*의 기록이다. + +### 실 브로커가 실제로 잡아낸 결함 + +이 스위트들은 장식이 아니다. 작성 과정에서 결정적 테스트가 통과하는데 실 인프라에서 실패한 +결함을 두 건 잡았다. + +1. **Outbox `IN_FLIGHT` 고아 행** — lease 쿼리가 `PENDING`/`AMBIGUOUS`만 클레임 대상으로 봐서, + publish 도중 죽은 relay가 남긴 행이 lease 만료 후에도 영영 회수되지 않았다. +2. **Rabbit confirm 경합** — transport가 publish *후에* confirm을 등록해서, 연결 스레드에서 + confirm이 먼저 도착하면 유실되고 호출자가 무한 대기했다. + +둘 다 인메모리 double이 실제보다 관대해서 통과하고 있었다. + +## 명시적 비지원 + +- 공통 `EXACTLY_ONCE` 설정 — `DeliveryGuarantee`에 상수가 존재하지 않는다. +- 전역 순서 — `OrderingScope`에 `GLOBAL`이 존재하지 않는다. +- DB와 broker의 자동 원자 transaction, 기본 XA +- Java native serialization +- 무제한 payload·header, 무한 retry +- 운영 application에서의 topology 파괴 작업 +- 일반 애플리케이션에 raw broker client 반환 +- DLQ publish 확인 전 source ACK diff --git a/infra/messaging/kafka/docker-compose.yml b/infra/messaging/kafka/docker-compose.yml new file mode 100644 index 00000000..449ff900 --- /dev/null +++ b/infra/messaging/kafka/docker-compose.yml @@ -0,0 +1,33 @@ +# Kafka 4.3.x in KRaft mode. +# +# Single broker on purpose: this compose file exists to reproduce the platform's Stable profile +# locally, not to model a production cluster. The settings below are the ones the profile guard +# enforces, so a local run fails the same way a misconfigured deployment would. +services: + kafka: + image: apache/kafka:4.3.0 + container_name: messaging-kafka + ports: + - "9092:9092" + environment: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + # acks=all is only a durability guarantee when more than one replica must acknowledge. + # With a single broker the platform still requires acks=all; min.insync.replicas is 1 here + # and is expected to be 2 in any environment that claims replication evidence. + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_MIN_INSYNC_REPLICAS: 1 + # Topology is created by infrastructure, never by the application. + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + healthcheck: + test: ["CMD-SHELL", "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1"] + interval: 5s + timeout: 10s + retries: 20 diff --git a/infra/messaging/nats/docker-compose.yml b/infra/messaging/nats/docker-compose.yml new file mode 100644 index 00000000..08e19069 --- /dev/null +++ b/infra/messaging/nats/docker-compose.yml @@ -0,0 +1,21 @@ +# NATS 2.14.x with JetStream, Experimental tier. +# +# JetStream is mandatory: core NATS is fire-and-forget with no persistence and no acknowledgement, +# so an at-least-once destination configured against it would report success for messages that were +# never stored. The adapter's validator refuses that combination. +services: + nats: + image: nats:2.14-alpine + container_name: messaging-nats + ports: + - "4222:4222" + - "8222:8222" + command: + - "--jetstream" + - "--store_dir=/data" + - "--http_port=8222" + healthcheck: + test: ["CMD-SHELL", "wget -q -O- http://localhost:8222/healthz || exit 1"] + interval: 5s + timeout: 5s + retries: 20 diff --git a/infra/messaging/postgres/docker-compose.yml b/infra/messaging/postgres/docker-compose.yml new file mode 100644 index 00000000..1502cee6 --- /dev/null +++ b/infra/messaging/postgres/docker-compose.yml @@ -0,0 +1,27 @@ +# PostgreSQL 16 for the Outbox and Inbox. +# +# logical replication is enabled so the optional Debezium CDC relay can be exercised against the +# same database the polling relay uses; the two must produce an identical wire contract. +services: + postgres: + image: postgres:16-alpine + container_name: messaging-postgres + ports: + - "5432:5432" + environment: + POSTGRES_DB: messaging + POSTGRES_USER: messaging + POSTGRES_PASSWORD: messaging + command: + - "postgres" + - "-c" + - "wal_level=logical" + - "-c" + - "max_replication_slots=4" + - "-c" + - "max_wal_senders=4" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messaging -d messaging"] + interval: 5s + timeout: 5s + retries: 20 diff --git a/infra/messaging/pulsar/docker-compose.yml b/infra/messaging/pulsar/docker-compose.yml new file mode 100644 index 00000000..5e950cfc --- /dev/null +++ b/infra/messaging/pulsar/docker-compose.yml @@ -0,0 +1,17 @@ +# Pulsar 4.0 LTS, Experimental tier. +# +# Present so the Experimental adapter can be exercised, not because it is supported. The adapter +# stays disabled unless backend.messaging.experimental.pulsar=true. +services: + pulsar: + image: apachepulsar/pulsar:4.0.3 + container_name: messaging-pulsar + ports: + - "6650:6650" + - "8080:8080" + command: bin/pulsar standalone --no-functions-worker --no-stream-storage + healthcheck: + test: ["CMD", "bin/pulsar-admin", "brokers", "healthcheck"] + interval: 10s + timeout: 10s + retries: 20 diff --git a/infra/messaging/rabbitmq/docker-compose.yml b/infra/messaging/rabbitmq/docker-compose.yml new file mode 100644 index 00000000..4e6c354c --- /dev/null +++ b/infra/messaging/rabbitmq/docker-compose.yml @@ -0,0 +1,22 @@ +# RabbitMQ 4.3.x. +# +# Quorum queues are the default for durable work queues in this platform, so the classic mirroring +# policy is deliberately absent: classic mirrored queues can lose acknowledged messages during a +# partition, which is precisely the guarantee a durable work queue exists to provide. +services: + rabbitmq: + image: rabbitmq:4.3-management + container_name: messaging-rabbitmq + ports: + - "5672:5672" + - "15672:15672" + environment: + RABBITMQ_DEFAULT_USER: messaging + RABBITMQ_DEFAULT_PASS: messaging + # Publisher confirms and returns are client-side settings; the profile guard enforces them. + RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "-rabbit consumer_timeout 1800000" + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"] + interval: 5s + timeout: 10s + retries: 20 diff --git a/infra/messaging/tls/README.md b/infra/messaging/tls/README.md new file mode 100644 index 00000000..e57113c6 --- /dev/null +++ b/infra/messaging/tls/README.md @@ -0,0 +1,35 @@ +# TLS material + +Production profiles require TLS **and** hostname verification; `MessageSecurityValidator` fails +startup without either. + +No key material is committed here, and none should be. Certificates are issued by the deployment's +own PKI and mounted at runtime; a keystore in a repository is a credential in a repository, and +rotating it means a commit. + +## Local development + +The compose files in the sibling directories run plaintext listeners deliberately. They exist to +reproduce the *messaging* semantics locally, not the transport security, and running them with +`production: false` in the destination profile is what keeps the validator honest — a profile marked +`production: true` against a plaintext broker must fail, and that is a test, not an inconvenience. + +## Generating a local CA for TLS testing + +```bash +openssl req -x509 -newkey rsa:4096 -sha256 -days 30 -nodes \ + -keyout ca.key -out ca.crt -subj "/CN=messaging-local-ca" + +openssl req -newkey rsa:4096 -nodes -keyout broker.key -out broker.csr \ + -subj "/CN=localhost" + +openssl x509 -req -in broker.csr -CA ca.crt -CAkey ca.key -CAcreateserial \ + -out broker.crt -days 30 -sha256 \ + -extfile <(printf "subjectAltName=DNS:localhost,IP:127.0.0.1") +``` + +The `subjectAltName` is not optional. Hostname verification is required in production profiles, and +a certificate without a SAN fails it — which is the correct outcome, not something to work around by +disabling the check. + +Generated files are ignored by `.gitignore` in this directory. diff --git a/infra/messaging/toxiproxy/docker-compose.yml b/infra/messaging/toxiproxy/docker-compose.yml new file mode 100644 index 00000000..9aac0b49 --- /dev/null +++ b/infra/messaging/toxiproxy/docker-compose.yml @@ -0,0 +1,19 @@ +# Toxiproxy, for the failures that matter most. +# +# The platform's hardest guarantee is that a lost confirmation is reported as AMBIGUOUS rather than +# guessed. A healthy broker will not lose one on request, so the chaos suite puts a proxy in front of +# it and severs the connection after the record was accepted but before the acknowledgement arrives. +services: + toxiproxy: + image: ghcr.io/shopify/toxiproxy:2.12.0 + container_name: messaging-toxiproxy + ports: + - "8474:8474" # control API + - "19092:19092" # proxied Kafka + - "15673:15673" # proxied RabbitMQ + - "15433:15433" # proxied PostgreSQL + healthcheck: + test: ["CMD", "/toxiproxy-cli", "list"] + interval: 5s + timeout: 5s + retries: 20 diff --git a/src/build.gradle b/src/build.gradle index e90a58c1..3906de84 100644 --- a/src/build.gradle +++ b/src/build.gradle @@ -431,7 +431,12 @@ configure(subprojects.findAll { it.childProjects.isEmpty() }) { } dependencies { - if (project.path in [':domain-core', ':application-core', ':shared-contract']) { + // The messaging platform leaves own a broker-neutral public contract. Keeping their test + // classpath on plain JUnit + AssertJ is what makes "messaging-core-api has no Spring + // dependency" verifiable rather than aspirational; leaves that genuinely need a Spring + // test context add it in their own build file. + if (project.path in [':domain-core', ':application-core', ':shared-contract'] || + project.path.startsWith(':messaging:')) { testImplementation 'org.junit.jupiter:junit-jupiter' testImplementation 'org.assertj:assertj-core' } else { @@ -444,6 +449,49 @@ configure(subprojects.findAll { it.childProjects.isEmpty() }) { errorprone 'com.google.errorprone:error_prone_core:2.49.0' // D5 compile-time checker } + // The three messaging leaves that carry JMH benchmarks get a `jmh` source set. It is a source + // set rather than a plugin because the benchmarks are compiled and reviewed on every build but + // only *run* on demand: a benchmark that stops compiling is a defect, while a benchmark that + // runs in CI is a flaky test measuring the build agent. + if (project.path in [':messaging:messaging-kafka', + ':messaging:messaging-rabbit', + ':messaging:messaging-testkit']) { + sourceSets { + jmh { + compileClasspath += sourceSets.main.output + sourceSets.test.output + runtimeClasspath += sourceSets.main.output + sourceSets.test.output + } + } + configurations { + jmhImplementation.extendsFrom implementation, testImplementation + jmhRuntimeOnly.extendsFrom runtimeOnly, testRuntimeOnly + } + dependencies { + jmhImplementation 'org.openjdk.jmh:jmh-core:1.37' + jmhAnnotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess:1.37' + // ErrorProne's -Werror would reject JMH's generated sources, which the platform does + // not own and cannot fix. + jmhAnnotationProcessor 'com.google.errorprone:error_prone_core:2.49.0' + } + tasks.named('compileJmhJava') { + options.errorprone.enabled = false + options.compilerArgs.removeAll { it == '-Werror' } + } + // JMH's annotation processor emits the generated harness into this source set, and its + // generated code trips DLS_DEAD_LOCAL_STORE by design (the dead stores are how it defeats + // dead-code elimination). Analysing code the platform neither wrote nor can fix would make + // the gate unactionable, so the jmh source set is excluded from the bug and style checks. + // The benchmarks themselves are still compiled, which is what catches a real breakage. + tasks.named('spotbugsJmh') { enabled = false } + tasks.named('checkstyleJmh') { enabled = false } + tasks.register('jmh', JavaExec) { + group = 'verification' + description = 'Runs the JMH benchmarks in this leaf.' + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'org.openjdk.jmh.Main' + } + } + // feature-ci-quality-gates-contract §4 (D7) — the main release gate EXCLUDES the flaky // quarantine bucket so a quarantined test can never block merge. Quarantined tests carry // JUnit's built-in @Tag("quarantine"); they run separately via `quarantineTest` (non-blocking) diff --git a/src/config/architecture/modules.json b/src/config/architecture/modules.json index 0b2d8060..a7af7da5 100644 --- a/src/config/architecture/modules.json +++ b/src/config/architecture/modules.json @@ -252,6 +252,285 @@ "runtime_memberships": [ "sample-portfolio" ] + }, + { + "id": "messaging-core-api", + "gradle_path": ":messaging:messaging-core-api", + "source_path": "src/messaging/messaging-core-api", + "allowed_dependencies": [], + "runtime_memberships": [] + }, + { + "id": "messaging-schema-api", + "gradle_path": ":messaging:messaging-schema-api", + "source_path": "src/messaging/messaging-schema-api", + "allowed_dependencies": [ + "messaging-core-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-schema-json", + "gradle_path": ":messaging:messaging-schema-json", + "source_path": "src/messaging/messaging-schema-json", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-schema-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-schema-avro", + "gradle_path": ":messaging:messaging-schema-avro", + "source_path": "src/messaging/messaging-schema-avro", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-schema-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-schema-protobuf", + "gradle_path": ":messaging:messaging-schema-protobuf", + "source_path": "src/messaging/messaging-schema-protobuf", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-schema-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-cloudevents", + "gradle_path": ":messaging:messaging-cloudevents", + "source_path": "src/messaging/messaging-cloudevents", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-schema-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-policy", + "gradle_path": ":messaging:messaging-policy", + "source_path": "src/messaging/messaging-policy", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-schema-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-transport-spi", + "gradle_path": ":messaging:messaging-transport-spi", + "source_path": "src/messaging/messaging-transport-spi", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-schema-api", + "messaging-policy" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-observability", + "gradle_path": ":messaging:messaging-observability", + "source_path": "src/messaging/messaging-observability", + "allowed_dependencies": [ + "messaging-core-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-security", + "gradle_path": ":messaging:messaging-security", + "source_path": "src/messaging/messaging-security", + "allowed_dependencies": [ + "messaging-core-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-kafka", + "gradle_path": ":messaging:messaging-kafka", + "source_path": "src/messaging/messaging-kafka", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-schema-api", + "messaging-policy", + "messaging-transport-spi", + "messaging-observability", + "messaging-security", + "messaging-admin-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-kafka-share-experimental", + "gradle_path": ":messaging:messaging-kafka-share-experimental", + "source_path": "src/messaging/messaging-kafka-share-experimental", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-policy", + "messaging-transport-spi", + "messaging-kafka" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-rabbit", + "gradle_path": ":messaging:messaging-rabbit", + "source_path": "src/messaging/messaging-rabbit", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-schema-api", + "messaging-policy", + "messaging-transport-spi", + "messaging-observability", + "messaging-security", + "messaging-admin-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-reliability-api", + "gradle_path": ":messaging:messaging-reliability-api", + "source_path": "src/messaging/messaging-reliability-api", + "allowed_dependencies": [ + "messaging-core-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-outbox-jpa", + "gradle_path": ":messaging:messaging-outbox-jpa", + "source_path": "src/messaging/messaging-outbox-jpa", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-reliability-api", + "messaging-policy", + "messaging-observability" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-inbox-jpa", + "gradle_path": ":messaging:messaging-inbox-jpa", + "source_path": "src/messaging/messaging-inbox-jpa", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-reliability-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-claim-check", + "gradle_path": ":messaging:messaging-claim-check", + "source_path": "src/messaging/messaging-claim-check", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-reliability-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-admin-api", + "gradle_path": ":messaging:messaging-admin-api", + "source_path": "src/messaging/messaging-admin-api", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-policy" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-admin-runtime", + "gradle_path": ":messaging:messaging-admin-runtime", + "source_path": "src/messaging/messaging-admin-runtime", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-policy", + "messaging-admin-api", + "messaging-transport-spi", + "messaging-security", + "messaging-observability" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-pulsar-experimental", + "gradle_path": ":messaging:messaging-pulsar-experimental", + "source_path": "src/messaging/messaging-pulsar-experimental", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-schema-api", + "messaging-policy", + "messaging-transport-spi", + "messaging-observability", + "messaging-security", + "messaging-admin-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-nats-experimental", + "gradle_path": ":messaging:messaging-nats-experimental", + "source_path": "src/messaging/messaging-nats-experimental", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-schema-api", + "messaging-policy", + "messaging-transport-spi", + "messaging-observability", + "messaging-security", + "messaging-admin-api" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-spring-cloud-stream-bridge", + "gradle_path": ":messaging:messaging-spring-cloud-stream-bridge", + "source_path": "src/messaging/messaging-spring-cloud-stream-bridge", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-policy", + "messaging-transport-spi" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-spring-boot-starter", + "gradle_path": ":messaging:messaging-spring-boot-starter", + "source_path": "src/messaging/messaging-spring-boot-starter", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-schema-api", + "messaging-schema-json", + "messaging-cloudevents", + "messaging-policy", + "messaging-transport-spi", + "messaging-observability", + "messaging-security", + "messaging-kafka", + "messaging-rabbit", + "messaging-reliability-api", + "messaging-outbox-jpa", + "messaging-inbox-jpa", + "messaging-claim-check", + "messaging-admin-api", + "messaging-admin-runtime" + ], + "runtime_memberships": [] + }, + { + "id": "messaging-testkit", + "gradle_path": ":messaging:messaging-testkit", + "source_path": "src/messaging/messaging-testkit", + "allowed_dependencies": [ + "messaging-core-api", + "messaging-schema-api", + "messaging-policy", + "messaging-transport-spi" + ], + "runtime_memberships": [] } ] } diff --git a/src/config/spotbugs/exclude.xml b/src/config/spotbugs/exclude.xml index 928020f9..0cbae62a 100644 --- a/src/config/spotbugs/exclude.xml +++ b/src/config/spotbugs/exclude.xml @@ -43,4 +43,24 @@ + + + + + + + + + + + + diff --git a/src/messaging/messaging-admin-api/build.gradle b/src/messaging/messaging-admin-api/build.gradle new file mode 100644 index 00000000..33935341 --- /dev/null +++ b/src/messaging/messaging-admin-api/build.gradle @@ -0,0 +1,6 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-policy') +} diff --git a/src/messaging/messaging-admin-api/gradle.lockfile b/src/messaging/messaging-admin-api/gradle.lockfile new file mode 100644 index 00000000..599ff921 --- /dev/null +++ b/src/messaging/messaging-admin-api/gradle.lockfile @@ -0,0 +1,83 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty=compileClasspath,runtimeClasspath diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/AdminApproval.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/AdminApproval.java new file mode 100644 index 00000000..8013c77d --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/AdminApproval.java @@ -0,0 +1,45 @@ +package dev.caskeleton.messaging.admin; + +import java.time.Instant; +import java.util.Objects; + +/** + * The approval that authorises a destructive messaging operation. + * + *

Approvals expire. An open-ended approval becomes a standing permission, which is the same + * thing as no approval at all — the window is what keeps "we approved a redrive last quarter" from + * authorising one today. + * + * @param ticket the change reference + * @param approvedBy the approver's identity + * @param approvedAt when the approval was granted + * @param validUntil when the approval stops authorising anything + */ +public record AdminApproval( + String ticket, String approvedBy, Instant approvedAt, Instant validUntil) { + + public AdminApproval { + Objects.requireNonNull(approvedAt, "approvedAt must not be null"); + Objects.requireNonNull(validUntil, "validUntil must not be null"); + if (ticket == null || ticket.isBlank()) { + throw new IllegalArgumentException("ticket must not be blank"); + } + if (approvedBy == null || approvedBy.isBlank()) { + throw new IllegalArgumentException("approvedBy must not be blank"); + } + if (validUntil.isBefore(approvedAt)) { + throw new IllegalArgumentException("an approval cannot expire before it was granted"); + } + } + + /** + * Reports whether the approval still authorises an operation. + * + * @param now the current instant + * @return true while inside the window + */ + public boolean isValidAt(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + return !now.isBefore(approvedAt) && now.isBefore(validUntil); + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ApprovedRedrivePlan.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ApprovedRedrivePlan.java new file mode 100644 index 00000000..43552e48 --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ApprovedRedrivePlan.java @@ -0,0 +1,56 @@ +package dev.caskeleton.messaging.admin; + +import dev.caskeleton.messaging.api.error.MessageAuthorizationException; +import java.time.Instant; +import java.util.Objects; + +/** + * A redrive plan that a named human has approved. + * + *

Carries {@code loopAcknowledged} separately from the approval itself. Approving a redrive of + * 900 parked messages and approving a redrive that will re-fail 400 of them are different + * decisions, and the second one needs the approver to have seen the number — so the plan cannot + * execute on a loop-risking set unless that was acknowledged explicitly. + * + * @param plan the plan that was approved + * @param approval the approval authorising it + * @param loopAcknowledged whether the approver accepted the previously-redriven candidates + */ +public record ApprovedRedrivePlan( + RedrivePlan plan, AdminApproval approval, boolean loopAcknowledged) { + + public ApprovedRedrivePlan { + Objects.requireNonNull(plan, "plan must not be null"); + Objects.requireNonNull(approval, "approval must not be null"); + } + + /** + * Refuses execution when the approval, the topology, or the loop risk no longer permits it. + * + * @param now the current instant + * @param currentTopologyVersion the topology version at execution time + * @throws MessageAuthorizationException when the plan may not execute + */ + public void requireExecutable(Instant now, String currentTopologyVersion) { + Objects.requireNonNull(now, "now must not be null"); + Objects.requireNonNull(currentTopologyVersion, "currentTopologyVersion must not be null"); + + if (!approval.isValidAt(now)) { + throw new MessageAuthorizationException( + "APPROVAL_EXPIRED", "approval %s is not valid at %s".formatted(approval.ticket(), now)); + } + if (!plan.topologyVersion().equals(currentTopologyVersion)) { + throw new MessageAuthorizationException( + "TOPOLOGY_CHANGED_SINCE_APPROVAL", + "the plan was approved against topology %s but the broker is now at %s" + .formatted(plan.topologyVersion(), currentTopologyVersion)); + } + if (plan.risksALoop() && !loopAcknowledged) { + throw new MessageAuthorizationException( + "REDRIVE_LOOP_NOT_ACKNOWLEDGED", + "%d of the %d candidates already failed a previous redrive; re-running them without " + .formatted(plan.alreadyRedrivenCandidates(), plan.candidates()) + + "fixing the cause produces a loop that looks like progress"); + } + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ApprovedReplayPlan.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ApprovedReplayPlan.java new file mode 100644 index 00000000..d438e491 --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ApprovedReplayPlan.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.admin; + +import dev.caskeleton.messaging.api.error.MessageAuthorizationException; +import java.time.Instant; +import java.util.Objects; + +/** + * A replay plan that a named human has approved. + * + *

A distinct type from {@link ReplayPlan} rather than a boolean on it. The execute method takes + * this type, so an unapproved plan cannot reach it — the authorisation is enforced by the compiler + * instead of by a runtime check somebody can forget to write. + * + * @param plan the plan that was approved + * @param approval the approval authorising it + */ +public record ApprovedReplayPlan(ReplayPlan plan, AdminApproval approval) { + + public ApprovedReplayPlan { + Objects.requireNonNull(plan, "plan must not be null"); + Objects.requireNonNull(approval, "approval must not be null"); + } + + /** + * Refuses execution when the approval or the plan no longer applies. + * + * @param now the current instant + * @param currentTopologyVersion the topology version at execution time + * @throws MessageAuthorizationException when the approval has expired or the topology moved + */ + public void requireExecutable(Instant now, String currentTopologyVersion) { + Objects.requireNonNull(now, "now must not be null"); + Objects.requireNonNull(currentTopologyVersion, "currentTopologyVersion must not be null"); + + if (!approval.isValidAt(now)) { + throw new MessageAuthorizationException( + "APPROVAL_EXPIRED", "approval %s is not valid at %s".formatted(approval.ticket(), now)); + } + if (!plan.topologyVersion().equals(currentTopologyVersion)) { + // Every number in the plan was computed against the old topology, so the approver agreed to + // an impact estimate that no longer describes what would happen. + throw new MessageAuthorizationException( + "TOPOLOGY_CHANGED_SINCE_APPROVAL", + "the plan was approved against topology %s but the broker is now at %s; the estimated " + .formatted(plan.topologyVersion(), currentTopologyVersion) + + "impact no longer applies and the plan must be rebuilt"); + } + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/DestinationTopology.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/DestinationTopology.java new file mode 100644 index 00000000..cf639cf5 --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/DestinationTopology.java @@ -0,0 +1,46 @@ +package dev.caskeleton.messaging.admin; + +import java.util.Map; +import java.util.Objects; + +/** + * What the broker actually reports for one destination. + * + *

The counterpart to {@link TopologyManifest}: the manifest is what was declared, this is what + * exists. Kept as a separate type rather than reusing the manifest so that a comparison cannot + * accidentally compare a manifest with itself and report success. + * + * @param physicalName the broker-side name + * @param partitions the observed partition count + * @param replicationFactor the observed replication factor + * @param configuration the observed configuration entries + * @param exists whether the destination is present at all + */ +public record DestinationTopology( + String physicalName, + int partitions, + int replicationFactor, + Map configuration, + boolean exists) { + + public DestinationTopology { + Objects.requireNonNull(configuration, "configuration must not be null"); + if (physicalName == null || physicalName.isBlank()) { + throw new IllegalArgumentException("physicalName must not be blank"); + } + if (exists && partitions < 1) { + throw new IllegalArgumentException("an existing destination has at least one partition"); + } + configuration = Map.copyOf(configuration); + } + + /** + * Returns the topology for a destination the broker does not have. + * + * @param physicalName the broker-side name that was looked up + * @return the absent topology + */ + public static DestinationTopology absent(String physicalName) { + return new DestinationTopology(physicalName, 0, 0, Map.of(), false); + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/DestructiveOperation.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/DestructiveOperation.java new file mode 100644 index 00000000..b09bfb69 --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/DestructiveOperation.java @@ -0,0 +1,25 @@ +package dev.caskeleton.messaging.admin; + +/** + * The operations that cannot be undone. + * + *

Enumerated so the guard is exhaustive rather than a list of {@code if} statements that a new + * operation can quietly avoid. + */ +public enum DestructiveOperation { + + /** Re-read a destination from an earlier position. */ + REPLAY, + + /** Move messages from a dead letter destination back to their source. */ + REDRIVE, + + /** Move a consumer group's committed position. */ + OFFSET_RESET, + + /** Discard the contents of a destination. */ + PURGE, + + /** Remove a destination entirely. */ + DELETE_DESTINATION +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/DestructiveOperationGuard.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/DestructiveOperationGuard.java new file mode 100644 index 00000000..3acd854d --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/DestructiveOperationGuard.java @@ -0,0 +1,73 @@ +package dev.caskeleton.messaging.admin; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessageAuthorizationException; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * The single gate every destructive messaging operation passes through. + * + *

Three conditions, all required. The caller must hold the admin credential — an application + * runtime does not, by construction. The approval must be present and inside its validity window. + * And a dry run is always permitted, because the way to make operators plan before they act is to + * make planning free. + * + *

Centralised so that adding a new destructive operation means adding an enum constant, not + * remembering to re-implement the checks. + */ +public final class DestructiveOperationGuard { + + private final boolean adminCredentialPresent; + + /** + * Creates a guard for a runtime. + * + * @param adminCredentialPresent whether this runtime holds the admin credential + */ + public DestructiveOperationGuard(boolean adminCredentialPresent) { + this.adminCredentialPresent = adminCredentialPresent; + } + + /** + * Authorises one destructive operation. + * + * @param operation the operation + * @param destination the destination affected + * @param approval the approval, when one was supplied + * @param dryRun whether this is a plan-only run + * @param now the current instant + * @throws MessageAuthorizationException when the operation is not authorised + */ + public void authorize( + DestructiveOperation operation, + DestinationName destination, + Optional approval, + boolean dryRun, + Instant now) { + Objects.requireNonNull(operation, "operation must not be null"); + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(approval, "approval must not be null"); + Objects.requireNonNull(now, "now must not be null"); + + if (dryRun) { + return; + } + if (!adminCredentialPresent) { + throw new MessageAuthorizationException( + "ADMIN_CREDENTIAL_REQUIRED", + operation + " on " + destination.value() + " requires the admin credential"); + } + AdminApproval granted = + approval.orElseThrow( + () -> + new MessageAuthorizationException( + "APPROVAL_REQUIRED", + operation + " on " + destination.value() + " requires an approval")); + if (!granted.isValidAt(now)) { + throw new MessageAuthorizationException( + "APPROVAL_EXPIRED", "approval " + granted.ticket() + " is outside its validity window"); + } + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/RedrivePlan.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/RedrivePlan.java new file mode 100644 index 00000000..3f605bf0 --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/RedrivePlan.java @@ -0,0 +1,64 @@ +package dev.caskeleton.messaging.admin; + +import java.time.Instant; +import java.util.Objects; + +/** + * What a redrive would do, produced before anything is moved. + * + *

{@code alreadyRedrivenCandidates} is the number that matters most. A message with a non-zero + * redrive count has been sent back to its source before and failed again; redriving it a second + * time without fixing the cause produces a loop that looks like progress in every dashboard. + * Surfacing the count at plan time is what lets an operator notice before starting it. + * + * @param request the request this plan was built from + * @param candidates how many messages are eligible + * @param alreadyRedrivenCandidates how many of those have been redriven before + * @param plannedAt when the plan was produced + * @param topologyVersion the topology the estimate was computed against + */ +public record RedrivePlan( + RedriveRequest request, + int candidates, + int alreadyRedrivenCandidates, + Instant plannedAt, + String topologyVersion) { + + public RedrivePlan { + Objects.requireNonNull(request, "request must not be null"); + Objects.requireNonNull(plannedAt, "plannedAt must not be null"); + if (candidates < 0) { + throw new IllegalArgumentException("candidates must not be negative"); + } + if (alreadyRedrivenCandidates < 0 || alreadyRedrivenCandidates > candidates) { + throw new IllegalArgumentException( + "alreadyRedrivenCandidates must be between 0 and the candidate count"); + } + if (topologyVersion == null || topologyVersion.isBlank()) { + throw new IllegalArgumentException("topologyVersion must not be blank"); + } + } + + /** + * Reports whether this redrive would replay messages that already failed a redrive. + * + * @return true when any candidate has been redriven before + */ + public boolean risksALoop() { + return alreadyRedrivenCandidates > 0; + } + + /** + * Returns a one-line operator-facing summary of the impact. + * + * @return the sanitized summary + */ + public String describeImpact() { + String loop = + risksALoop() + ? ", %d of which already failed a previous redrive".formatted(alreadyRedrivenCandidates) + : ""; + return "redrive %d messages from %s to %s%s" + .formatted(candidates, request.source().value(), request.target().value(), loop); + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/RedriveRequest.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/RedriveRequest.java new file mode 100644 index 00000000..fd924af6 --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/RedriveRequest.java @@ -0,0 +1,37 @@ +package dev.caskeleton.messaging.admin; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.util.Objects; +import java.util.UUID; + +/** + * A request to move messages from a dead letter destination back to their source. + * + *

The redrive id is a separate identifier from the message id, and both travel with the message. + * Reusing the message id as the operation id would make "this message was redriven" and "this + * message is a different message" indistinguishable, and an operator could not tell a redrive loop + * from ordinary traffic. + * + * @param redriveId the operation identity + * @param source the dead letter destination to drain + * @param target the destination to publish back to + * @param batchSize how many messages to move per pass + * @param dryRun whether to plan without moving anything + */ +public record RedriveRequest( + UUID redriveId, DestinationName source, DestinationName target, int batchSize, boolean dryRun) { + + private static final int MAX_BATCH = 100; + + public RedriveRequest { + Objects.requireNonNull(redriveId, "redriveId must not be null"); + Objects.requireNonNull(source, "source must not be null"); + Objects.requireNonNull(target, "target must not be null"); + if (source.equals(target)) { + throw new IllegalArgumentException("a redrive cannot target its own source"); + } + if (batchSize < 1 || batchSize > MAX_BATCH) { + throw new IllegalArgumentException("redrive batch size must be between 1 and " + MAX_BATCH); + } + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/RedriveResult.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/RedriveResult.java new file mode 100644 index 00000000..e0896eb6 --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/RedriveResult.java @@ -0,0 +1,51 @@ +package dev.caskeleton.messaging.admin; + +import java.time.Duration; +import java.util.Objects; +import java.util.UUID; + +/** + * What a redrive actually did. + * + *

{@code stillParked} is not simply {@code candidates - moved}. A message stays parked when its + * republish did not confirm, and the redrive deliberately leaves it there rather than settling it — + * the DLQ-confirm-before-settle rule applies to a redrive exactly as it does to the original + * dead-lettering, because a redrive that settles an unconfirmed republish deletes the last copy. + * + * @param redriveId the operation identity + * @param candidates how many messages were eligible + * @param moved how many were republished and settled + * @param stillParked how many stayed on the dead letter destination + * @param elapsed how long the redrive took + * @param dryRun whether nothing was actually moved + */ +public record RedriveResult( + UUID redriveId, int candidates, int moved, int stillParked, Duration elapsed, boolean dryRun) { + + public RedriveResult { + Objects.requireNonNull(redriveId, "redriveId must not be null"); + Objects.requireNonNull(elapsed, "elapsed must not be null"); + if (candidates < 0 || moved < 0 || stillParked < 0) { + throw new IllegalArgumentException("redrive counters must not be negative"); + } + if (moved + stillParked > candidates) { + throw new IllegalArgumentException( + "a redrive cannot account for more messages than it had candidates"); + } + if (dryRun && moved > 0) { + throw new IllegalArgumentException("a dry run must not move anything"); + } + } + + /** + * Reports whether every candidate was accounted for. + * + *

An unaccounted message is a bug, not a partial success: it was neither republished nor left + * parked, which means the redrive lost track of it. + * + * @return true when moved plus still-parked covers every candidate + */ + public boolean isFullyAccounted() { + return moved + stillParked == candidates; + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ReplayPlan.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ReplayPlan.java new file mode 100644 index 00000000..de24065a --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ReplayPlan.java @@ -0,0 +1,55 @@ +package dev.caskeleton.messaging.admin; + +import java.time.Instant; +import java.util.Objects; + +/** + * What a replay would do, produced before anything is read. + * + *

The plan exists so the estimate can be reviewed. "Replay from yesterday" is a sentence; "this + * will re-deliver 4.2 million messages into the live consumer group" is a decision, and the only + * moment an operator can make it is before the replay starts. + * + *

{@code topologyVersion} is captured here and re-checked at execution. A plan approved against + * one topology and executed against another is estimating a different thing entirely — a partition + * count that changed in between invalidates every number in this record. + * + * @param request the request this plan was built from + * @param estimatedMessages how many messages the window covers + * @param plannedAt when the plan was produced + * @param topologyVersion the topology the estimate was computed against + * @param targetsLiveConsumerGroup whether the replay would feed the live group rather than an + * isolated one + */ +public record ReplayPlan( + ReplayRequest request, + long estimatedMessages, + Instant plannedAt, + String topologyVersion, + boolean targetsLiveConsumerGroup) { + + public ReplayPlan { + Objects.requireNonNull(request, "request must not be null"); + Objects.requireNonNull(plannedAt, "plannedAt must not be null"); + if (estimatedMessages < 0) { + throw new IllegalArgumentException("estimatedMessages must not be negative"); + } + if (topologyVersion == null || topologyVersion.isBlank()) { + throw new IllegalArgumentException("topologyVersion must not be blank"); + } + } + + /** + * Returns a one-line operator-facing summary of the impact. + * + * @return the sanitized summary + */ + public String describeImpact() { + return "replay %s from %s: about %d messages into %s" + .formatted( + request.destination().value(), + request.from(), + estimatedMessages, + targetsLiveConsumerGroup ? "the LIVE consumer group" : "an isolated group"); + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ReplayRequest.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ReplayRequest.java new file mode 100644 index 00000000..055bee1a --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ReplayRequest.java @@ -0,0 +1,36 @@ +package dev.caskeleton.messaging.admin; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; + +/** + * A request to re-read a destination from an earlier position. + * + * @param replayId the operation identity + * @param destination the destination to replay + * @param from the replay start point + * @param to the replay end point, when bounded + * @param isolatedConsumerGroup whether to replay into a throwaway group + * @param dryRun whether to plan without reading anything + */ +public record ReplayRequest( + UUID replayId, + DestinationName destination, + Instant from, + Optional to, + boolean isolatedConsumerGroup, + boolean dryRun) { + + public ReplayRequest { + Objects.requireNonNull(replayId, "replayId must not be null"); + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(from, "from must not be null"); + Objects.requireNonNull(to, "to must not be null"); + if (to.filter(end -> end.isBefore(from)).isPresent()) { + throw new IllegalArgumentException("replay window ends before it starts"); + } + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ReplayResult.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ReplayResult.java new file mode 100644 index 00000000..c2cf2086 --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/ReplayResult.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.admin; + +import java.time.Duration; +import java.util.Objects; +import java.util.UUID; + +/** + * What a replay actually did. + * + *

Reports the delivered count against the plan's estimate. They routinely differ — retention may + * have expired part of the window, or the stream may have grown while the plan was being approved — + * and the difference is the operator's signal that the replay covered something other than what was + * approved. + * + * @param replayId the operation identity + * @param estimatedMessages what the plan predicted + * @param deliveredMessages what was actually re-delivered + * @param elapsed how long the replay took + * @param completed whether the whole window was covered + * @param dryRun whether nothing was actually read + */ +public record ReplayResult( + UUID replayId, + long estimatedMessages, + long deliveredMessages, + Duration elapsed, + boolean completed, + boolean dryRun) { + + public ReplayResult { + Objects.requireNonNull(replayId, "replayId must not be null"); + Objects.requireNonNull(elapsed, "elapsed must not be null"); + if (estimatedMessages < 0 || deliveredMessages < 0) { + throw new IllegalArgumentException("replay counters must not be negative"); + } + if (dryRun && deliveredMessages > 0) { + throw new IllegalArgumentException("a dry run must not deliver anything"); + } + } + + /** + * Reports whether the replay covered materially less than was approved. + * + * @return true when fewer than 90% of the estimated messages were delivered + */ + public boolean fellShortOfTheEstimate() { + return completed && estimatedMessages > 0 && deliveredMessages * 10 < estimatedMessages * 9; + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyIssue.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyIssue.java new file mode 100644 index 00000000..fdd3e4c7 --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyIssue.java @@ -0,0 +1,82 @@ +package dev.caskeleton.messaging.admin; + +import java.util.Objects; + +/** + * One discrepancy between a declared topology and what the broker actually has. + * + *

Severity is part of the finding because the two kinds behave differently at startup. A {@link + * Severity#BLOCKING} issue means the destination cannot deliver its declared guarantee — + * replication factor 1 on a destination promising durability is not a warning, it is a promise the + * platform cannot keep — so the context refuses to start. A {@link Severity#ADVISORY} issue is a + * drift worth reporting that does not break a guarantee. + * + * @param destination the logical destination + * @param attribute the topology attribute that differs + * @param declared what the manifest declared + * @param actual what the broker reported + * @param severity how the platform should react + */ +public record TopologyIssue( + String destination, String attribute, String declared, String actual, Severity severity) { + + /** How the platform reacts to a topology discrepancy. */ + public enum Severity { + /** The destination cannot deliver a declared guarantee; startup must fail. */ + BLOCKING, + /** Drift worth reporting that does not break a guarantee. */ + ADVISORY + } + + public TopologyIssue { + Objects.requireNonNull(severity, "severity must not be null"); + requireText(destination, "destination"); + requireText(attribute, "attribute"); + requireText(declared, "declared"); + requireText(actual, "actual"); + } + + /** + * Creates a blocking issue. + * + * @param destination the logical destination + * @param attribute the differing attribute + * @param declared the declared value + * @param actual the observed value + * @return the issue + */ + public static TopologyIssue blocking( + String destination, String attribute, String declared, String actual) { + return new TopologyIssue(destination, attribute, declared, actual, Severity.BLOCKING); + } + + /** + * Creates an advisory issue. + * + * @param destination the logical destination + * @param attribute the differing attribute + * @param declared the declared value + * @param actual the observed value + * @return the issue + */ + public static TopologyIssue advisory( + String destination, String attribute, String declared, String actual) { + return new TopologyIssue(destination, attribute, declared, actual, Severity.ADVISORY); + } + + /** + * Returns a one-line operator-facing description. + * + * @return the sanitized description + */ + public String describe() { + return "%s: %s declared %s but the broker has %s" + .formatted(destination, attribute, declared, actual); + } + + private static void requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " must not be blank"); + } + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyManagementMode.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyManagementMode.java new file mode 100644 index 00000000..47d530cd --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyManagementMode.java @@ -0,0 +1,37 @@ +package dev.caskeleton.messaging.admin; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; + +/** + * Whether the application may create broker topology, or only check it. + * + *

{@link #VALIDATE_ONLY} in production, always. An application that auto-creates topology will + * auto-create it after a configuration typo too, and the topic it makes is indistinguishable from a + * real one — same broker, same client, same metrics — while carrying the broker's default partition + * count and replication factor instead of the ones the destination needs. The failure surfaces + * weeks later as data loss on a partition that was never replicated. + */ +public enum TopologyManagementMode { + + /** Compare the declared topology against the broker and fail on a mismatch. */ + VALIDATE_ONLY, + + /** Create missing topology. Permitted outside production only. */ + CREATE_IF_MISSING; + + /** + * Refuses auto-creation on a production runtime. + * + * @param production whether this runtime is production + * @throws MessagingConfigurationException when auto-creation is configured in production + */ + public void requireSafeFor(boolean production) { + if (production && this == CREATE_IF_MISSING) { + throw new MessagingConfigurationException( + "AUTO_CREATE_IN_PRODUCTION", + "topology auto-creation is not permitted in production: a mistyped destination would be " + + "created with the broker's default partition count and replication factor, and " + + "would look exactly like a correctly provisioned one"); + } + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyManifest.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyManifest.java new file mode 100644 index 00000000..7466a17d --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyManifest.java @@ -0,0 +1,76 @@ +package dev.caskeleton.messaging.admin; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * The declared shape of a destination's broker topology. + * + *

Production topology is created by infrastructure code and only validated by the + * application. An application that creates topology on startup will happily create it in the wrong + * place after a configuration mistake, and the resulting topic looks exactly like a real one. + * + * @param destination the logical destination + * @param physicalName the broker-side name + * @param partitions the expected partition count, where the broker has partitions + * @param replicationFactor the expected replication factor + * @param requiredConfiguration configuration entries that must match exactly + */ +public record TopologyManifest( + String destination, + String physicalName, + int partitions, + int replicationFactor, + Map requiredConfiguration) { + + public TopologyManifest { + Objects.requireNonNull(requiredConfiguration, "requiredConfiguration must not be null"); + if (destination == null || destination.isBlank()) { + throw new IllegalArgumentException("destination must not be blank"); + } + if (physicalName == null || physicalName.isBlank()) { + throw new IllegalArgumentException("physicalName must not be blank"); + } + if (partitions < 1) { + throw new IllegalArgumentException("partitions must be at least 1"); + } + if (replicationFactor < 1) { + throw new IllegalArgumentException("replicationFactor must be at least 1"); + } + requiredConfiguration = Map.copyOf(requiredConfiguration); + } + + /** + * Compares this manifest against what the broker actually reports. + * + * @param actualPartitions the observed partition count + * @param actualReplicationFactor the observed replication factor + * @param actualConfiguration the observed configuration + * @return the differences, empty when the topology matches + */ + public List differencesFrom( + int actualPartitions, int actualReplicationFactor, Map actualConfiguration) { + Objects.requireNonNull(actualConfiguration, "actualConfiguration must not be null"); + List differences = new java.util.ArrayList<>(); + + if (actualPartitions != partitions) { + differences.add("partitions expected " + partitions + " but found " + actualPartitions); + } + if (actualReplicationFactor != replicationFactor) { + differences.add( + "replicationFactor expected " + + replicationFactor + + " but found " + + actualReplicationFactor); + } + requiredConfiguration.forEach( + (key, expected) -> { + String actual = actualConfiguration.get(key); + if (!expected.equals(actual)) { + differences.add(key + " expected " + expected + " but found " + actual); + } + }); + return List.copyOf(differences); + } +} diff --git a/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyValidationReport.java b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyValidationReport.java new file mode 100644 index 00000000..44e0faa0 --- /dev/null +++ b/src/messaging/messaging-admin-api/src/main/java/dev/caskeleton/messaging/admin/TopologyValidationReport.java @@ -0,0 +1,83 @@ +package dev.caskeleton.messaging.admin; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.util.List; +import java.util.Objects; + +/** + * The outcome of comparing every declared topology against the broker. + * + *

Reports both severities together rather than failing on the first blocking issue. An operator + * fixing a topology wants the whole list — fixing one attribute, redeploying, and discovering the + * next one is how a ten-minute fix becomes an afternoon. + * + * @param issues every discrepancy found + * @param destinationsChecked how many destinations were compared + */ +public record TopologyValidationReport(List issues, int destinationsChecked) { + + public TopologyValidationReport { + Objects.requireNonNull(issues, "issues must not be null"); + if (destinationsChecked < 0) { + throw new IllegalArgumentException("destinationsChecked must not be negative"); + } + issues = List.copyOf(issues); + } + + /** + * Returns a report with nothing to fix. + * + * @param destinationsChecked how many destinations were compared + * @return the clean report + */ + public static TopologyValidationReport clean(int destinationsChecked) { + return new TopologyValidationReport(List.of(), destinationsChecked); + } + + /** + * Returns the issues that must fail startup. + * + * @return the blocking issues + */ + public List blocking() { + return issues.stream() + .filter(issue -> issue.severity() == TopologyIssue.Severity.BLOCKING) + .toList(); + } + + /** + * Returns the issues worth reporting that do not break a guarantee. + * + * @return the advisory issues + */ + public List advisory() { + return issues.stream() + .filter(issue -> issue.severity() == TopologyIssue.Severity.ADVISORY) + .toList(); + } + + /** + * Reports whether the topology may be used. + * + * @return true when nothing blocking was found + */ + public boolean isAcceptable() { + return blocking().isEmpty(); + } + + /** + * Fails startup when any blocking issue was found. + * + * @throws MessagingConfigurationException listing every blocking issue + */ + public void requireAcceptable() { + List blocking = blocking(); + if (blocking.isEmpty()) { + return; + } + throw new MessagingConfigurationException( + "TOPOLOGY_MISMATCH", + "the broker topology cannot deliver the declared guarantees: " + + String.join("; ", blocking.stream().map(TopologyIssue::describe).toList())); + } +} diff --git a/src/messaging/messaging-admin-api/src/test/java/dev/caskeleton/messaging/admin/DestructiveOperationGuardTest.java b/src/messaging/messaging-admin-api/src/test/java/dev/caskeleton/messaging/admin/DestructiveOperationGuardTest.java new file mode 100644 index 00000000..f957edec --- /dev/null +++ b/src/messaging/messaging-admin-api/src/test/java/dev/caskeleton/messaging/admin/DestructiveOperationGuardTest.java @@ -0,0 +1,116 @@ +package dev.caskeleton.messaging.admin; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessageAuthorizationException; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.Test; + +class DestructiveOperationGuardTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + private static final DestinationName ORDERS = new DestinationName("order-events"); + + private static final AdminApproval VALID = + new AdminApproval("CHG-1001", "operator", NOW.minusSeconds(60), NOW.plusSeconds(3600)); + + @Test + void anApplicationRuntimeCannotRedrive() { + DestructiveOperationGuard guard = new DestructiveOperationGuard(false); + + assertThatThrownBy( + () -> + guard.authorize( + DestructiveOperation.REDRIVE, ORDERS, Optional.of(VALID), false, NOW)) + .isInstanceOf(MessageAuthorizationException.class) + .hasMessageContaining("admin credential"); + } + + @Test + void anAdminRuntimeStillNeedsAnApproval() { + DestructiveOperationGuard guard = new DestructiveOperationGuard(true); + + assertThatThrownBy( + () -> guard.authorize(DestructiveOperation.PURGE, ORDERS, Optional.empty(), false, NOW)) + .isInstanceOf(MessageAuthorizationException.class) + .hasMessageContaining("approval"); + } + + @Test + void anExpiredApprovalDoesNotAuthorise() { + DestructiveOperationGuard guard = new DestructiveOperationGuard(true); + AdminApproval expired = + new AdminApproval("CHG-1000", "operator", NOW.minusSeconds(7200), NOW.minusSeconds(60)); + + assertThatThrownBy( + () -> + guard.authorize( + DestructiveOperation.OFFSET_RESET, ORDERS, Optional.of(expired), false, NOW)) + .isInstanceOf(MessageAuthorizationException.class) + .hasMessageContaining("validity window"); + } + + @Test + void aDryRunIsAlwaysPermitted() { + DestructiveOperationGuard guard = new DestructiveOperationGuard(false); + + assertThatCode( + () -> + guard.authorize( + DestructiveOperation.DELETE_DESTINATION, ORDERS, Optional.empty(), true, NOW)) + .doesNotThrowAnyException(); + } + + @Test + void anApprovedAdminOperationIsAuthorised() { + DestructiveOperationGuard guard = new DestructiveOperationGuard(true); + + assertThatCode( + () -> + guard.authorize( + DestructiveOperation.REPLAY, ORDERS, Optional.of(VALID), false, NOW)) + .doesNotThrowAnyException(); + } + + @Test + void aRedriveCannotTargetItsOwnSource() { + assertThatThrownBy(() -> new RedriveRequest(UUID.randomUUID(), ORDERS, ORDERS, 100, false)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void aRedriveBatchIsBoundedSoOneOperationCannotFloodTheSource() { + assertThatThrownBy( + () -> + new RedriveRequest( + UUID.randomUUID(), new DestinationName("order-events-dlq"), ORDERS, 101, false)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void aTopologyManifestReportsEveryDifference() { + TopologyManifest manifest = + new TopologyManifest( + "order-events", "order.events.v1", 6, 3, Map.of("min.insync.replicas", "2")); + + assertThat(manifest.differencesFrom(3, 3, Map.of("min.insync.replicas", "1"))) + .hasSize(2) + .anySatisfy(difference -> assertThat(difference).contains("partitions")) + .anySatisfy(difference -> assertThat(difference).contains("min.insync.replicas")); + } + + @Test + void aMatchingTopologyReportsNoDifferences() { + TopologyManifest manifest = + new TopologyManifest( + "order-events", "order.events.v1", 6, 3, Map.of("min.insync.replicas", "2")); + + assertThat(manifest.differencesFrom(6, 3, Map.of("min.insync.replicas", "2"))).isEmpty(); + } +} diff --git a/src/messaging/messaging-admin-runtime/build.gradle b/src/messaging/messaging-admin-runtime/build.gradle new file mode 100644 index 00000000..8c6ba468 --- /dev/null +++ b/src/messaging/messaging-admin-runtime/build.gradle @@ -0,0 +1,10 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-policy') + api project(':messaging:messaging-admin-api') + api project(':messaging:messaging-transport-spi') + api project(':messaging:messaging-security') + api project(':messaging:messaging-observability') +} diff --git a/src/messaging/messaging-admin-runtime/gradle.lockfile b/src/messaging/messaging-admin-runtime/gradle.lockfile new file mode 100644 index 00000000..ae1a5dc3 --- /dev/null +++ b/src/messaging/messaging-admin-runtime/gradle.lockfile @@ -0,0 +1,88 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +io.micrometer:micrometer-commons:1.16.0=runtimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-core:1.16.0=runtimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-observation:1.16.0=runtimeClasspath,testRuntimeClasspath +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.hdrhistogram:HdrHistogram:2.2.2=runtimeClasspath,testRuntimeClasspath +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.latencyutils:LatencyUtils:2.0.3=runtimeClasspath,testRuntimeClasspath +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty=compileClasspath diff --git a/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/AdminOperationIdempotencyStore.java b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/AdminOperationIdempotencyStore.java new file mode 100644 index 00000000..29410b34 --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/AdminOperationIdempotencyStore.java @@ -0,0 +1,78 @@ +package dev.caskeleton.messaging.admin.runtime; + +import java.time.Instant; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Remembers which approvals have already been executed. + * + *

An approval authorises one execution, not a standing permission. Without this, re-running the + * same approved redrive twice is a single command-history arrow-up away — and the second run + * republishes messages the first one already moved, which on a destination without an inbox is + * indistinguishable from a duplicate storm. + * + *

Claiming is atomic and returns the previous claim rather than a boolean, so a duplicate + * attempt can tell the operator when it ran and by which operation id instead of just + * refusing. + */ +public final class AdminOperationIdempotencyStore { + + private final Map claims = new ConcurrentHashMap<>(); + + /** + * One recorded execution of an approval. + * + * @param approvalTicket the approval that was executed + * @param operationId the operation identity that claimed it + * @param executedAt when it ran + */ + public record Claim(String approvalTicket, String operationId, Instant executedAt) { + + public Claim { + Objects.requireNonNull(executedAt, "executedAt must not be null"); + if (approvalTicket == null || approvalTicket.isBlank()) { + throw new IllegalArgumentException("approvalTicket must not be blank"); + } + if (operationId == null || operationId.isBlank()) { + throw new IllegalArgumentException("operationId must not be blank"); + } + } + } + + /** + * Claims an approval for execution. + * + * @param approvalTicket the approval to claim + * @param operationId the operation attempting it + * @param now the current instant + * @return empty when the claim succeeded; the existing claim when it was already executed + */ + public Optional claim(String approvalTicket, String operationId, Instant now) { + Objects.requireNonNull(now, "now must not be null"); + Claim candidate = new Claim(approvalTicket, operationId, now); + Claim existing = claims.putIfAbsent(approvalTicket, candidate); + return Optional.ofNullable(existing); + } + + /** + * Returns the recorded execution of an approval. + * + * @param approvalTicket the approval + * @return the claim, when the approval has been executed + */ + public Optional find(String approvalTicket) { + return Optional.ofNullable(claims.get(approvalTicket)); + } + + /** + * Returns how many approvals have been executed. + * + * @return the claim count + */ + public int size() { + return claims.size(); + } +} diff --git a/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/BrokerTopologyInspector.java b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/BrokerTopologyInspector.java new file mode 100644 index 00000000..27a86aa5 --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/BrokerTopologyInspector.java @@ -0,0 +1,33 @@ +package dev.caskeleton.messaging.admin.runtime; + +import dev.caskeleton.messaging.admin.DestinationTopology; +import java.util.Optional; + +/** + * Reads the broker's current topology. + * + *

Read-only by construction. The inspector is what the application's own credential uses, and + * that credential holds no destructive grant, so the type exposes no way to create, alter, or + * delete — an application cannot reach a destructive operation even by mistake because there is no + * method to reach. + */ +public interface BrokerTopologyInspector { + + /** + * Describes one destination. + * + * @param physicalName the broker-side name + * @return the observed topology, absent when the broker has no such destination + */ + Optional describe(String physicalName); + + /** + * Returns an opaque version for the broker's current topology. + * + *

Used to invalidate an approved plan whose impact estimate was computed against an earlier + * shape. Any value that changes when the topology changes is sufficient; it is never parsed. + * + * @return the topology version + */ + String topologyVersion(); +} diff --git a/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/CompositeTopologyValidator.java b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/CompositeTopologyValidator.java new file mode 100644 index 00000000..d12e912e --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/CompositeTopologyValidator.java @@ -0,0 +1,52 @@ +package dev.caskeleton.messaging.admin.runtime; + +import dev.caskeleton.messaging.admin.DestinationTopology; +import dev.caskeleton.messaging.admin.TopologyIssue; +import dev.caskeleton.messaging.admin.TopologyManifest; +import dev.caskeleton.messaging.admin.TopologyValidationReport; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Validates every declared topology in one pass and reports the whole result. + * + *

Collects all issues rather than stopping at the first blocking one. An operator fixing a + * topology mismatch wants the complete list — discovering the next problem only after a redeploy + * turns one fix into a sequence of them, and each redeploy is another restart of a production + * service. + */ +public final class CompositeTopologyValidator { + + private final BrokerTopologyInspector inspector; + private final TopologyValidator validator = new TopologyValidator(); + + /** + * Creates a validator over a broker inspector. + * + * @param inspector reads the broker's current topology + */ + public CompositeTopologyValidator(BrokerTopologyInspector inspector) { + this.inspector = Objects.requireNonNull(inspector, "inspector must not be null"); + } + + /** + * Compares every manifest against the broker. + * + * @param manifests the declared topologies + * @return the complete report + */ + public TopologyValidationReport validate(List manifests) { + Objects.requireNonNull(manifests, "manifests must not be null"); + + List issues = new ArrayList<>(); + for (TopologyManifest manifest : manifests) { + DestinationTopology observed = + inspector + .describe(manifest.physicalName()) + .orElseGet(() -> DestinationTopology.absent(manifest.physicalName())); + issues.addAll(validator.compare(manifest, observed)); + } + return new TopologyValidationReport(issues, manifests.size()); + } +} diff --git a/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/DefaultMessagingAdminService.java b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/DefaultMessagingAdminService.java new file mode 100644 index 00000000..2c6df02e --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/DefaultMessagingAdminService.java @@ -0,0 +1,199 @@ +package dev.caskeleton.messaging.admin.runtime; + +import dev.caskeleton.messaging.admin.ApprovedRedrivePlan; +import dev.caskeleton.messaging.admin.ApprovedReplayPlan; +import dev.caskeleton.messaging.admin.RedrivePlan; +import dev.caskeleton.messaging.admin.RedriveRequest; +import dev.caskeleton.messaging.admin.RedriveResult; +import dev.caskeleton.messaging.admin.ReplayPlan; +import dev.caskeleton.messaging.admin.ReplayRequest; +import dev.caskeleton.messaging.admin.ReplayResult; +import dev.caskeleton.messaging.admin.TopologyManifest; +import dev.caskeleton.messaging.admin.TopologyValidationReport; +import dev.caskeleton.messaging.api.error.MessageAuthorizationException; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Supplier; + +/** + * Wires plan, approval, and execution together for the non-destructive admin operations. + * + *

Execution runs four checks, in this order, and the order is the point. + * + *

    + *
  1. The approval is still inside its window. + *
  2. The topology has not changed since the plan was approved. + *
  3. The approval has not already been executed. + *
  4. Only then does anything move. + *
+ * + *

The idempotency claim comes before the work rather than after it. Claiming afterwards + * leaves a window where a second execution starts while the first is still running, which is + * precisely the double-redrive this store exists to prevent. + */ +public final class DefaultMessagingAdminService implements MessagingAdminService { + + private final CompositeTopologyValidator topologyValidator; + private final BrokerTopologyInspector inspector; + private final AdminOperationIdempotencyStore idempotency; + private final ReplayService replayService; + private final RedriveService redriveService; + private final Supplier> manifests; + private final ReplayEstimator replayEstimator; + private final RedriveEstimator redriveEstimator; + private final Supplier clock; + + /** + * Creates the admin service. + * + * @param topologyValidator compares declared topology against the broker + * @param inspector reads the broker's topology version + * @param idempotency records which approvals have been executed + * @param replayService performs replays + * @param redriveService performs redrives + * @param manifests supplies the declared topologies + * @param replayEstimator estimates a replay's message count + * @param redriveEstimator estimates a redrive's candidates + * @param clock supplies the current instant + */ + public DefaultMessagingAdminService( + CompositeTopologyValidator topologyValidator, + BrokerTopologyInspector inspector, + AdminOperationIdempotencyStore idempotency, + ReplayService replayService, + RedriveService redriveService, + Supplier> manifests, + ReplayEstimator replayEstimator, + RedriveEstimator redriveEstimator, + Supplier clock) { + this.topologyValidator = + Objects.requireNonNull(topologyValidator, "topologyValidator required"); + this.inspector = Objects.requireNonNull(inspector, "inspector must not be null"); + this.idempotency = Objects.requireNonNull(idempotency, "idempotency must not be null"); + this.replayService = Objects.requireNonNull(replayService, "replayService must not be null"); + this.redriveService = Objects.requireNonNull(redriveService, "redriveService must not be null"); + this.manifests = Objects.requireNonNull(manifests, "manifests must not be null"); + this.replayEstimator = Objects.requireNonNull(replayEstimator, "replayEstimator required"); + this.redriveEstimator = Objects.requireNonNull(redriveEstimator, "redriveEstimator required"); + this.clock = Objects.requireNonNull(clock, "clock must not be null"); + } + + @Override + public TopologyValidationReport validateTopology() { + return topologyValidator.validate(manifests.get()); + } + + @Override + public ReplayPlan planReplay(ReplayRequest request) { + Objects.requireNonNull(request, "request must not be null"); + return new ReplayPlan( + request, + replayEstimator.estimate(request), + clock.get(), + inspector.topologyVersion(), + !request.isolatedConsumerGroup()); + } + + @Override + public ReplayResult executeReplay(ApprovedReplayPlan plan) { + Objects.requireNonNull(plan, "plan must not be null"); + Instant now = clock.get(); + ReplayRequest request = plan.plan().request(); + + plan.requireExecutable(now, inspector.topologyVersion()); + claimOrRefuse(plan.approval().ticket(), request.replayId().toString(), now); + + Instant startedAt = clock.get(); + ReplayReport report = + replayService.replay( + request, Optional.of(plan.approval()), plan.approval().approvedBy(), now); + + return new ReplayResult( + request.replayId(), + plan.plan().estimatedMessages(), + report.replayed(), + Duration.between(startedAt, clock.get()), + true, + report.dryRun()); + } + + @Override + public RedrivePlan planRedrive(RedriveRequest request) { + Objects.requireNonNull(request, "request must not be null"); + RedriveEstimate estimate = redriveEstimator.estimate(request); + return new RedrivePlan( + request, + estimate.candidates(), + estimate.alreadyRedriven(), + clock.get(), + inspector.topologyVersion()); + } + + @Override + public RedriveResult executeRedrive(ApprovedRedrivePlan plan) { + Objects.requireNonNull(plan, "plan must not be null"); + Instant now = clock.get(); + RedriveRequest request = plan.plan().request(); + + plan.requireExecutable(now, inspector.topologyVersion()); + claimOrRefuse(plan.approval().ticket(), request.redriveId().toString(), now); + + Instant startedAt = clock.get(); + RedriveReport report = + redriveService.redrive( + request, Optional.of(plan.approval()), plan.approval().approvedBy(), now); + + return new RedriveResult( + request.redriveId(), + report.candidates(), + report.moved(), + report.failed(), + Duration.between(startedAt, clock.get()), + report.dryRun()); + } + + private void claimOrRefuse(String approvalTicket, String operationId, Instant now) { + idempotency + .claim(approvalTicket, operationId, now) + .ifPresent( + existing -> { + throw new MessageAuthorizationException( + "APPROVAL_ALREADY_EXECUTED", + "approval %s was already executed at %s by operation %s; an approval authorises " + .formatted(approvalTicket, existing.executedAt(), existing.operationId()) + + "one execution, not a standing permission"); + }); + } + + /** Estimates how many messages a replay would re-deliver. */ + @FunctionalInterface + public interface ReplayEstimator { + + /** + * Estimates a replay's message count without reading anything. + * + * @param request the replay request + * @return the estimated count + */ + long estimate(ReplayRequest request); + } + + /** How many messages a redrive would move, and how many already failed one. */ + record RedriveEstimate(int candidates, int alreadyRedriven) {} + + /** Estimates a redrive's candidates. */ + @FunctionalInterface + public interface RedriveEstimator { + + /** + * Estimates a redrive's candidates without moving anything. + * + * @param request the redrive request + * @return the estimate + */ + RedriveEstimate estimate(RedriveRequest request); + } +} diff --git a/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/DestructiveMessagingAdmin.java b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/DestructiveMessagingAdmin.java new file mode 100644 index 00000000..7e6cec8a --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/DestructiveMessagingAdmin.java @@ -0,0 +1,82 @@ +package dev.caskeleton.messaging.admin.runtime; + +import dev.caskeleton.messaging.admin.AdminApproval; +import dev.caskeleton.messaging.admin.DestructiveOperation; +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; + +/** + * The operations that destroy data an application cannot recreate. + * + *

A separate interface from {@link MessagingAdminService}, and no bean for it is ever registered + * in an application runtime. The separation is the control: an application that never receives this + * type cannot purge a topic even if every other guard is bypassed, because the method does not + * exist on anything it holds. + * + *

Each operation takes an {@link Approved} argument rather than an approval parameter, so the + * authorisation cannot be forgotten at a call site — there is no way to call these without one. + */ +public interface DestructiveMessagingAdmin { + + /** An authorised destructive request. */ + record Approved( + DestructiveOperation operation, + DestinationName destination, + AdminApproval approval, + long estimatedMessagesAffected) { + + public Approved { + Objects.requireNonNull(operation, "operation must not be null"); + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(approval, "approval must not be null"); + if (estimatedMessagesAffected < 0) { + throw new IllegalArgumentException("estimatedMessagesAffected must not be negative"); + } + } + } + + /** What a destructive operation did. */ + record DestructiveResult( + DestructiveOperation operation, + DestinationName destination, + long messagesAffected, + Duration elapsed, + Instant executedAt) { + + public DestructiveResult { + Objects.requireNonNull(operation, "operation must not be null"); + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(elapsed, "elapsed must not be null"); + Objects.requireNonNull(executedAt, "executedAt must not be null"); + if (messagesAffected < 0) { + throw new IllegalArgumentException("messagesAffected must not be negative"); + } + } + } + + /** + * Moves a consumer group's committed position. + * + * @param request the authorised request + * @return what the reset did + */ + DestructiveResult resetOffset(Approved request); + + /** + * Discards the messages a destination currently holds. + * + * @param request the authorised request + * @return what the purge did + */ + DestructiveResult purge(Approved request); + + /** + * Removes a destination entirely. + * + * @param request the authorised request + * @return what the deletion did + */ + DestructiveResult deleteDestination(Approved request); +} diff --git a/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/MessagingAdminService.java b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/MessagingAdminService.java new file mode 100644 index 00000000..37df640a --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/MessagingAdminService.java @@ -0,0 +1,65 @@ +package dev.caskeleton.messaging.admin.runtime; + +import dev.caskeleton.messaging.admin.ApprovedRedrivePlan; +import dev.caskeleton.messaging.admin.ApprovedReplayPlan; +import dev.caskeleton.messaging.admin.RedrivePlan; +import dev.caskeleton.messaging.admin.RedriveRequest; +import dev.caskeleton.messaging.admin.RedriveResult; +import dev.caskeleton.messaging.admin.ReplayPlan; +import dev.caskeleton.messaging.admin.ReplayRequest; +import dev.caskeleton.messaging.admin.ReplayResult; +import dev.caskeleton.messaging.admin.TopologyValidationReport; + +/** + * The non-destructive half of the admin plane. + * + *

Every mutating operation is split into plan and execute, and the execute methods take an + * {@code Approved*} type. A caller cannot execute something it has not planned, because it has no + * way to construct the argument — the plan/approve/execute sequence is enforced by the types rather + * than by a runtime check. + * + *

Destructive operations live in {@link DestructiveMessagingAdmin}, a separate interface that an + * application runtime never receives a bean for. Splitting them means a compromised handler that + * somehow reaches this service still has no method that deletes anything. + */ +public interface MessagingAdminService { + + /** + * Compares every declared topology against the broker. + * + * @return the discrepancies found + */ + TopologyValidationReport validateTopology(); + + /** + * Estimates what a replay would do, without reading anything. + * + * @param request the replay request + * @return the plan, including its impact estimate + */ + ReplayPlan planReplay(ReplayRequest request); + + /** + * Executes an approved replay. + * + * @param plan the approved plan + * @return what the replay actually did + */ + ReplayResult executeReplay(ApprovedReplayPlan plan); + + /** + * Estimates what a redrive would do, without moving anything. + * + * @param request the redrive request + * @return the plan, including how many candidates already failed a redrive + */ + RedrivePlan planRedrive(RedriveRequest request); + + /** + * Executes an approved redrive. + * + * @param plan the approved plan + * @return what the redrive actually did + */ + RedriveResult executeRedrive(ApprovedRedrivePlan plan); +} diff --git a/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/RedriveReport.java b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/RedriveReport.java new file mode 100644 index 00000000..3d76d3cf --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/RedriveReport.java @@ -0,0 +1,18 @@ +package dev.caskeleton.messaging.admin.runtime; + +/** + * What one redrive pass did. + * + * @param candidates how many messages were eligible + * @param moved how many were republished and settled + * @param failed how many did not confirm and stay parked + * @param dryRun whether this was a plan-only run + */ +public record RedriveReport(int candidates, int moved, int failed, boolean dryRun) { + + public RedriveReport { + if (candidates < 0 || moved < 0 || failed < 0) { + throw new IllegalArgumentException("redrive counters must not be negative"); + } + } +} diff --git a/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/RedriveService.java b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/RedriveService.java new file mode 100644 index 00000000..72de394f --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/RedriveService.java @@ -0,0 +1,152 @@ +package dev.caskeleton.messaging.admin.runtime; + +import dev.caskeleton.messaging.admin.AdminApproval; +import dev.caskeleton.messaging.admin.DestructiveOperation; +import dev.caskeleton.messaging.admin.DestructiveOperationGuard; +import dev.caskeleton.messaging.admin.RedriveRequest; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishResult; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Moves messages from a dead letter destination back to their source. + * + *

A redrive is a publish followed by a settlement, in that order, exactly like dead lettering in + * reverse. A message whose republish did not confirm stays in the dead letter destination: losing + * it on the way back would be the one outcome worse than leaving it parked. + * + *

The redrive id and a redrive counter travel with each message. Without them a message that + * fails again is indistinguishable from a new one, and a redrive loop is invisible until the dead + * letter destination is full. + */ +public final class RedriveService { + + private final DestructiveOperationGuard guard; + private final RedriveSource source; + private final RedrivePublisher publisher; + private final AuditSink audit; + + /** + * Creates a redrive service. + * + * @param guard the destructive operation guard + * @param source reads and settles dead letter messages + * @param publisher republishes to the target destination + * @param audit records the operation + */ + public RedriveService( + DestructiveOperationGuard guard, + RedriveSource source, + RedrivePublisher publisher, + AuditSink audit) { + this.guard = Objects.requireNonNull(guard, "guard must not be null"); + this.source = Objects.requireNonNull(source, "source must not be null"); + this.publisher = Objects.requireNonNull(publisher, "publisher must not be null"); + this.audit = Objects.requireNonNull(audit, "audit must not be null"); + } + + /** + * Runs one redrive pass. + * + * @param request what to move + * @param approval the approval, when one was supplied + * @param subject the operator identity + * @param now the current instant + * @return what the pass did + */ + public RedriveReport redrive( + RedriveRequest request, Optional approval, String subject, Instant now) { + Objects.requireNonNull(request, "request must not be null"); + guard.authorize( + DestructiveOperation.REDRIVE, request.source(), approval, request.dryRun(), now); + + List candidates = source.peek(request.source(), request.batchSize()); + if (request.dryRun()) { + return new RedriveReport(candidates.size(), 0, 0, true); + } + + List moved = new ArrayList<>(); + int failed = 0; + for (MessageId messageId : candidates) { + PublishResult result = publisher.republish(messageId, request.target(), request.redriveId()); + if (result.completion() == PublishCompletion.CONFIRMED) { + source.settle(request.source(), messageId); + moved.add(messageId); + } else { + failed++; + } + } + + audit.record( + new dev.caskeleton.messaging.observation.MessagingAuditEvent( + "REDRIVE", + subject, + request.source().value(), + approval.map(AdminApproval::ticket).orElse("dry-run"), + now, + java.util.Map.of( + "redriveId", request.redriveId().toString(), + "moved", Integer.toString(moved.size()), + "failed", Integer.toString(failed)))); + + return new RedriveReport(candidates.size(), moved.size(), failed, false); + } + + /** Reads and settles messages on a dead letter destination. */ + public interface RedriveSource { + + /** + * Returns the next candidates without settling them. + * + * @param destination the dead letter destination + * @param batchSize how many to return + * @return the candidate identities + */ + List peek( + dev.caskeleton.messaging.api.destination.DestinationName destination, int batchSize); + + /** + * Settles a message that has been successfully republished. + * + * @param destination the dead letter destination + * @param messageId the message identity + */ + void settle( + dev.caskeleton.messaging.api.destination.DestinationName destination, MessageId messageId); + } + + /** Republishes a dead lettered message to its target destination. */ + @FunctionalInterface + public interface RedrivePublisher { + + /** + * Republishes one message under its original identity. + * + * @param messageId the message identity + * @param target the destination to publish back to + * @param redriveId the operation identity stamped on the message + * @return the publish outcome + */ + PublishResult republish( + MessageId messageId, + dev.caskeleton.messaging.api.destination.DestinationName target, + java.util.UUID redriveId); + } + + /** Records privileged operations. */ + @FunctionalInterface + public interface AuditSink { + + /** + * Records one audit event. + * + * @param event the event + */ + void record(dev.caskeleton.messaging.observation.MessagingAuditEvent event); + } +} diff --git a/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/ReplayReport.java b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/ReplayReport.java new file mode 100644 index 00000000..e7d8c73e --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/ReplayReport.java @@ -0,0 +1,21 @@ +package dev.caskeleton.messaging.admin.runtime; + +import java.util.Objects; +import java.util.UUID; + +/** + * What one replay did. + * + * @param replayId the operation identity + * @param replayed how many messages were re-read + * @param dryRun whether this was a plan-only run + */ +public record ReplayReport(UUID replayId, long replayed, boolean dryRun) { + + public ReplayReport { + Objects.requireNonNull(replayId, "replayId must not be null"); + if (replayed < 0) { + throw new IllegalArgumentException("replayed must not be negative"); + } + } +} diff --git a/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/ReplayService.java b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/ReplayService.java new file mode 100644 index 00000000..81063e10 --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/ReplayService.java @@ -0,0 +1,99 @@ +package dev.caskeleton.messaging.admin.runtime; + +import dev.caskeleton.messaging.admin.AdminApproval; +import dev.caskeleton.messaging.admin.DestructiveOperation; +import dev.caskeleton.messaging.admin.DestructiveOperationGuard; +import dev.caskeleton.messaging.admin.ReplayRequest; +import dev.caskeleton.messaging.observation.MessagingAuditEvent; +import java.time.Instant; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Re-reads a destination from an earlier position. + * + *

An isolated replay reads alongside the live consumer and needs no approval, because it changes + * nothing: a throwaway group has its own offsets. Replaying into an existing production group is a + * different operation entirely — it rewinds a live consumer and reprocesses everything since — so + * it goes through the destructive guard. + * + *

Making the safe form free and the destructive form approved is what keeps operators from + * reaching for the destructive one out of convenience. + */ +public final class ReplayService { + + private final DestructiveOperationGuard guard; + private final ReplayExecutor executor; + private final RedriveService.AuditSink audit; + + /** + * Creates a replay service. + * + * @param guard the destructive operation guard + * @param executor performs the replay + * @param audit records the operation + */ + public ReplayService( + DestructiveOperationGuard guard, ReplayExecutor executor, RedriveService.AuditSink audit) { + this.guard = Objects.requireNonNull(guard, "guard must not be null"); + this.executor = Objects.requireNonNull(executor, "executor must not be null"); + this.audit = Objects.requireNonNull(audit, "audit must not be null"); + } + + /** + * Runs one replay. + * + * @param request what to replay + * @param approval the approval, when one was supplied + * @param subject the operator identity + * @param now the current instant + * @return what the replay did + */ + public ReplayReport replay( + ReplayRequest request, Optional approval, String subject, Instant now) { + Objects.requireNonNull(request, "request must not be null"); + Objects.requireNonNull(approval, "approval must not be null"); + + boolean needsApproval = !request.isolatedConsumerGroup(); + guard.authorize( + DestructiveOperation.REPLAY, + request.destination(), + approval, + request.dryRun() || !needsApproval, + now); + + if (request.dryRun()) { + return new ReplayReport(request.replayId(), 0, true); + } + + long replayed = executor.replay(request); + + audit.record( + new MessagingAuditEvent( + "REPLAY", + subject, + request.destination().value(), + approval.map(AdminApproval::ticket).orElse("isolated"), + now, + Map.of( + "replayId", request.replayId().toString(), + "isolated", Boolean.toString(request.isolatedConsumerGroup()), + "replayed", Long.toString(replayed)))); + + return new ReplayReport(request.replayId(), replayed, false); + } + + /** Performs the replay against a broker. */ + @FunctionalInterface + public interface ReplayExecutor { + + /** + * Replays a destination and returns how many messages were re-read. + * + * @param request the replay request + * @return the replayed message count + */ + long replay(ReplayRequest request); + } +} diff --git a/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/TopologyValidationRuntime.java b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/TopologyValidationRuntime.java new file mode 100644 index 00000000..4a61db48 --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/TopologyValidationRuntime.java @@ -0,0 +1,88 @@ +package dev.caskeleton.messaging.admin.runtime; + +import dev.caskeleton.messaging.admin.TopologyManifest; +import dev.caskeleton.messaging.api.error.MessageTopologyException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Validates declared topology against what the broker actually has. + * + *

Validate-only, and it fails startup rather than logging. A partition count that silently + * differs from the manifest changes the ordering guarantee the destination advertises, and a + * missing {@code min.insync.replicas} changes what {@code acks=all} actually means — both are the + * kind of drift that is invisible until the incident. + */ +public final class TopologyValidationRuntime { + + private final TopologyReader reader; + + /** + * Creates a validator over a broker reader. + * + * @param reader reads the observed topology + */ + public TopologyValidationRuntime(TopologyReader reader) { + this.reader = Objects.requireNonNull(reader, "reader must not be null"); + } + + /** + * Validates every manifest, reporting all differences at once. + * + * @param manifests the declared topology + * @throws MessageTopologyException when the broker does not match + */ + public void validate(List manifests) { + Objects.requireNonNull(manifests, "manifests must not be null"); + List problems = new ArrayList<>(); + + for (TopologyManifest manifest : manifests) { + ObservedTopology observed = reader.read(manifest.physicalName()); + if (observed == null) { + problems.add(manifest.physicalName() + " does not exist"); + continue; + } + manifest + .differencesFrom( + observed.partitions(), observed.replicationFactor(), observed.configuration()) + .forEach(difference -> problems.add(manifest.physicalName() + ": " + difference)); + } + + if (!problems.isEmpty()) { + throw new MessageTopologyException( + "TOPOLOGY_MISMATCH", + "broker topology does not match the manifest: " + String.join("; ", problems)); + } + } + + /** Reads the observed topology for a physical destination. */ + @FunctionalInterface + public interface TopologyReader { + + /** + * Reads one destination's topology. + * + * @param physicalName the broker-side name + * @return the observed topology, or null when it does not exist + */ + ObservedTopology read(String physicalName); + } + + /** + * What the broker reports for a destination. + * + * @param partitions the observed partition count + * @param replicationFactor the observed replication factor + * @param configuration the observed configuration + */ + public record ObservedTopology( + int partitions, int replicationFactor, Map configuration) { + + public ObservedTopology { + Objects.requireNonNull(configuration, "configuration must not be null"); + configuration = Map.copyOf(configuration); + } + } +} diff --git a/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/TopologyValidator.java b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/TopologyValidator.java new file mode 100644 index 00000000..003a3268 --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/main/java/dev/caskeleton/messaging/admin/runtime/TopologyValidator.java @@ -0,0 +1,91 @@ +package dev.caskeleton.messaging.admin.runtime; + +import dev.caskeleton.messaging.admin.DestinationTopology; +import dev.caskeleton.messaging.admin.TopologyIssue; +import dev.caskeleton.messaging.admin.TopologyManifest; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Compares one declared topology against what the broker reports. + * + *

Which discrepancies block is a judgement encoded here rather than left to configuration. + * Replication factor and absence are blocking because a destination that is missing or unreplicated + * cannot deliver the durability its profile promises. A partition count that is higher + * than declared is advisory rather than blocking: extra partitions do not break durability, and + * someone scaling a topic up deliberately should not be met with a refusal to start. + * + *

A partition count that is lower is blocking, because it silently reduces the + * concurrency the destination was sized for and, on a keyed topic, changes which key lands where. + */ +public final class TopologyValidator { + + /** + * Compares a manifest against observed topology. + * + * @param manifest what was declared + * @param observed what the broker reports + * @return the discrepancies found, empty when they agree + */ + public List compare(TopologyManifest manifest, DestinationTopology observed) { + Objects.requireNonNull(manifest, "manifest must not be null"); + Objects.requireNonNull(observed, "observed must not be null"); + + List issues = new ArrayList<>(); + String destination = manifest.destination(); + + if (!observed.exists()) { + issues.add( + TopologyIssue.blocking(destination, "existence", manifest.physicalName(), "absent")); + return List.copyOf(issues); + } + + if (!manifest.physicalName().equals(observed.physicalName())) { + issues.add( + TopologyIssue.blocking( + destination, "physicalName", manifest.physicalName(), observed.physicalName())); + } + + if (observed.partitions() < manifest.partitions()) { + issues.add( + TopologyIssue.blocking( + destination, + "partitions", + Integer.toString(manifest.partitions()), + Integer.toString(observed.partitions()))); + } else if (observed.partitions() > manifest.partitions()) { + // Scaling a topic up is a legitimate operation; refusing to start would punish it. + issues.add( + TopologyIssue.advisory( + destination, + "partitions", + Integer.toString(manifest.partitions()), + Integer.toString(observed.partitions()))); + } + + if (observed.replicationFactor() < manifest.replicationFactor()) { + issues.add( + TopologyIssue.blocking( + destination, + "replicationFactor", + Integer.toString(manifest.replicationFactor()), + Integer.toString(observed.replicationFactor()))); + } + + for (Map.Entry required : manifest.requiredConfiguration().entrySet()) { + String actual = observed.configuration().get(required.getKey()); + if (!required.getValue().equals(actual)) { + issues.add( + TopologyIssue.blocking( + destination, + required.getKey(), + required.getValue(), + actual == null ? "unset" : actual)); + } + } + + return List.copyOf(issues); + } +} diff --git a/src/messaging/messaging-admin-runtime/src/test/java/dev/caskeleton/messaging/admin/runtime/ApprovedPlanExecutionTest.java b/src/messaging/messaging-admin-runtime/src/test/java/dev/caskeleton/messaging/admin/runtime/ApprovedPlanExecutionTest.java new file mode 100644 index 00000000..924509b4 --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/test/java/dev/caskeleton/messaging/admin/runtime/ApprovedPlanExecutionTest.java @@ -0,0 +1,190 @@ +package dev.caskeleton.messaging.admin.runtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.admin.AdminApproval; +import dev.caskeleton.messaging.admin.ApprovedRedrivePlan; +import dev.caskeleton.messaging.admin.ApprovedReplayPlan; +import dev.caskeleton.messaging.admin.RedrivePlan; +import dev.caskeleton.messaging.admin.RedriveRequest; +import dev.caskeleton.messaging.admin.RedriveResult; +import dev.caskeleton.messaging.admin.ReplayPlan; +import dev.caskeleton.messaging.admin.ReplayRequest; +import dev.caskeleton.messaging.admin.ReplayResult; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessageAuthorizationException; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.Test; + +class ApprovedPlanExecutionTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z"); + private static final UUID OPERATION_ID = UUID.fromString("0199aaaa-bbbb-7ccc-8ddd-eeeeffff0000"); + + private static AdminApproval approval() { + return new AdminApproval("CHG-1042", "ops@example.com", NOW, NOW.plus(Duration.ofHours(2))); + } + + private static ReplayRequest replayRequest() { + return new ReplayRequest( + OPERATION_ID, + new DestinationName("orders.v1"), + NOW.minus(Duration.ofDays(1)), + Optional.empty(), + true, + false); + } + + private static ReplayPlan replayPlan(String topologyVersion) { + return new ReplayPlan(replayRequest(), 4_200_000, NOW, topologyVersion, false); + } + + private static RedriveRequest redriveRequest() { + return new RedriveRequest( + OPERATION_ID, + new DestinationName("orders.v1.dlq"), + new DestinationName("orders.v1"), + 50, + false); + } + + private static RedrivePlan redrivePlan(int candidates, int alreadyRedriven) { + return new RedrivePlan(redriveRequest(), candidates, alreadyRedriven, NOW, "v1"); + } + + @Test + void anExpiredApprovalCannotExecute() { + ApprovedReplayPlan approved = new ApprovedReplayPlan(replayPlan("v1"), approval()); + + assertThatThrownBy(() -> approved.requireExecutable(NOW.plus(Duration.ofDays(1)), "v1")) + .isInstanceOf(MessageAuthorizationException.class) + .hasMessageContaining("CHG-1042"); + } + + @Test + void aTopologyChangeSinceApprovalInvalidatesThePlan() { + ApprovedReplayPlan approved = new ApprovedReplayPlan(replayPlan("v1"), approval()); + + assertThatThrownBy(() -> approved.requireExecutable(NOW, "v2")) + .as("every number in the plan was computed against the old topology") + .isInstanceOf(MessageAuthorizationException.class) + .hasMessageContaining("rebuilt"); + } + + @Test + void aValidApprovalOnUnchangedTopologyExecutes() { + assertThatCode( + () -> new ApprovedReplayPlan(replayPlan("v1"), approval()).requireExecutable(NOW, "v1")) + .doesNotThrowAnyException(); + } + + @Test + void aRedriveThatWouldLoopNeedsThatAcknowledgedExplicitly() { + ApprovedRedrivePlan approved = + new ApprovedRedrivePlan(redrivePlan(900, 400), approval(), false); + + assertThatThrownBy(() -> approved.requireExecutable(NOW, "v1")) + .isInstanceOf(MessageAuthorizationException.class) + .hasMessageContaining("looks like progress"); + } + + @Test + void anAcknowledgedLoopMayProceed() { + assertThatCode( + () -> + new ApprovedRedrivePlan(redrivePlan(900, 400), approval(), true) + .requireExecutable(NOW, "v1")) + .doesNotThrowAnyException(); + } + + @Test + void aRedriveWithNoPreviouslyRedrivenCandidatesNeedsNoAcknowledgement() { + assertThatCode( + () -> + new ApprovedRedrivePlan(redrivePlan(900, 0), approval(), false) + .requireExecutable(NOW, "v1")) + .doesNotThrowAnyException(); + } + + @Test + void thePlanDescribesItsImpactBeforeAnythingRuns() { + assertThat(replayPlan("v1").describeImpact()).contains("4200000").contains("isolated group"); + assertThat(redrivePlan(900, 400).describeImpact()) + .contains("already failed a previous redrive"); + } + + @Test + void aReplayIntoTheLiveGroupSaysSoInCapitals() { + ReplayPlan live = + new ReplayPlan( + new ReplayRequest( + OPERATION_ID, + new DestinationName("orders.v1"), + NOW.minus(Duration.ofDays(1)), + Optional.empty(), + false, + false), + 4_200_000, + NOW, + "v1", + true); + + assertThat(live.describeImpact()) + .as("re-delivering millions of messages into the live group is the decision to flag") + .contains("LIVE consumer group"); + } + + @Test + void anApprovalAuthorisesOneExecutionNotAStandingPermission() { + AdminOperationIdempotencyStore store = new AdminOperationIdempotencyStore(); + + assertThat(store.claim("CHG-1042", "op-1", NOW)).isEmpty(); + assertThat(store.claim("CHG-1042", "op-2", NOW.plus(Duration.ofMinutes(5)))) + .as("an arrow-up in the shell must not re-run an approved redrive") + .hasValueSatisfying(existing -> assertThat(existing.operationId()).isEqualTo("op-1")); + } + + @Test + void aDifferentApprovalIsClaimedIndependently() { + AdminOperationIdempotencyStore store = new AdminOperationIdempotencyStore(); + store.claim("CHG-1042", "op-1", NOW); + + assertThat(store.claim("CHG-1043", "op-2", NOW)).isEmpty(); + assertThat(store.size()).isEqualTo(2); + } + + @Test + void aReplayResultFlagsAShortfallAgainstTheApprovedEstimate() { + ReplayResult result = + new ReplayResult(OPERATION_ID, 1_000_000, 100_000, Duration.ofMinutes(3), true, false); + + assertThat(result.fellShortOfTheEstimate()) + .as("retention may have expired part of the approved window") + .isTrue(); + } + + @Test + void aRedriveResultMustAccountForEveryCandidate() { + RedriveResult accounted = + new RedriveResult(OPERATION_ID, 100, 80, 20, Duration.ofSeconds(4), false); + RedriveResult unaccounted = + new RedriveResult(OPERATION_ID, 100, 80, 10, Duration.ofSeconds(4), false); + + assertThat(accounted.isFullyAccounted()).isTrue(); + assertThat(unaccounted.isFullyAccounted()) + .as("a message that was neither moved nor left parked has been lost track of") + .isFalse(); + } + + @Test + void aDryRunCannotClaimToHaveMovedAnything() { + assertThatThrownBy( + () -> new RedriveResult(OPERATION_ID, 100, 5, 0, Duration.ofSeconds(1), true)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/src/messaging/messaging-admin-runtime/src/test/java/dev/caskeleton/messaging/admin/runtime/TopologyValidationRuntimeTest.java b/src/messaging/messaging-admin-runtime/src/test/java/dev/caskeleton/messaging/admin/runtime/TopologyValidationRuntimeTest.java new file mode 100644 index 00000000..2927ac29 --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/test/java/dev/caskeleton/messaging/admin/runtime/TopologyValidationRuntimeTest.java @@ -0,0 +1,65 @@ +package dev.caskeleton.messaging.admin.runtime; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.admin.TopologyManifest; +import dev.caskeleton.messaging.api.error.MessageTopologyException; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class TopologyValidationRuntimeTest { + + private static final TopologyManifest ORDERS = + new TopologyManifest( + "order-events", "order.events.v1", 6, 3, Map.of("min.insync.replicas", "2")); + + @Test + void aMatchingTopologyValidates() { + TopologyValidationRuntime runtime = + new TopologyValidationRuntime( + name -> + new TopologyValidationRuntime.ObservedTopology( + 6, 3, Map.of("min.insync.replicas", "2"))); + + assertThatCode(() -> runtime.validate(List.of(ORDERS))).doesNotThrowAnyException(); + } + + @Test + void aMissingDestinationFailsStartup() { + TopologyValidationRuntime runtime = new TopologyValidationRuntime(name -> null); + + assertThatThrownBy(() -> runtime.validate(List.of(ORDERS))) + .isInstanceOf(MessageTopologyException.class) + .hasMessageContaining("does not exist"); + } + + @Test + void aWeakenedReplicationSettingFailsStartup() { + TopologyValidationRuntime runtime = + new TopologyValidationRuntime( + name -> + new TopologyValidationRuntime.ObservedTopology( + 6, 3, Map.of("min.insync.replicas", "1"))); + + assertThatThrownBy(() -> runtime.validate(List.of(ORDERS))) + .isInstanceOf(MessageTopologyException.class) + .hasMessageContaining("min.insync.replicas"); + } + + @Test + void everyDifferenceIsReportedAtOnce() { + TopologyValidationRuntime runtime = + new TopologyValidationRuntime( + name -> + new TopologyValidationRuntime.ObservedTopology( + 3, 1, Map.of("min.insync.replicas", "1"))); + + assertThatThrownBy(() -> runtime.validate(List.of(ORDERS))) + .isInstanceOf(MessageTopologyException.class) + .hasMessageContaining("partitions") + .hasMessageContaining("replicationFactor") + .hasMessageContaining("min.insync.replicas"); + } +} diff --git a/src/messaging/messaging-admin-runtime/src/test/java/dev/caskeleton/messaging/admin/runtime/TopologyValidatorTest.java b/src/messaging/messaging-admin-runtime/src/test/java/dev/caskeleton/messaging/admin/runtime/TopologyValidatorTest.java new file mode 100644 index 00000000..a792ddd6 --- /dev/null +++ b/src/messaging/messaging-admin-runtime/src/test/java/dev/caskeleton/messaging/admin/runtime/TopologyValidatorTest.java @@ -0,0 +1,172 @@ +package dev.caskeleton.messaging.admin.runtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.admin.DestinationTopology; +import dev.caskeleton.messaging.admin.TopologyIssue; +import dev.caskeleton.messaging.admin.TopologyManagementMode; +import dev.caskeleton.messaging.admin.TopologyManifest; +import dev.caskeleton.messaging.admin.TopologyValidationReport; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class TopologyValidatorTest { + + private final TopologyValidator validator = new TopologyValidator(); + + private static TopologyManifest manifest() { + return new TopologyManifest( + "orders.v1", "orders-v1", 12, 3, Map.of("min.insync.replicas", "2")); + } + + private static DestinationTopology observed( + int partitions, int replication, Map config) { + return new DestinationTopology("orders-v1", partitions, replication, config, true); + } + + @Test + void anAgreeingTopologyProducesNoIssues() { + assertThat(validator.compare(manifest(), observed(12, 3, Map.of("min.insync.replicas", "2")))) + .isEmpty(); + } + + @Test + void anAbsentDestinationBlocksStartup() { + List issues = + validator.compare(manifest(), DestinationTopology.absent("orders-v1")); + + assertThat(issues) + .singleElement() + .satisfies( + issue -> { + assertThat(issue.attribute()).isEqualTo("existence"); + assertThat(issue.severity()).isEqualTo(TopologyIssue.Severity.BLOCKING); + }); + } + + @Test + void tooFewPartitionsBlocksBecauseKeysWouldLandDifferently() { + List issues = + validator.compare(manifest(), observed(6, 3, Map.of("min.insync.replicas", "2"))); + + assertThat(issues) + .singleElement() + .satisfies( + issue -> assertThat(issue.severity()).isEqualTo(TopologyIssue.Severity.BLOCKING)); + } + + @Test + void extraPartitionsAreAdvisoryBecauseScalingUpIsLegitimate() { + List issues = + validator.compare(manifest(), observed(24, 3, Map.of("min.insync.replicas", "2"))); + + assertThat(issues) + .singleElement() + .satisfies( + issue -> assertThat(issue.severity()).isEqualTo(TopologyIssue.Severity.ADVISORY)); + } + + @Test + void tooLittleReplicationBlocksBecauseDurabilityIsPromised() { + List issues = + validator.compare(manifest(), observed(12, 1, Map.of("min.insync.replicas", "2"))); + + assertThat(issues) + .singleElement() + .satisfies( + issue -> { + assertThat(issue.attribute()).isEqualTo("replicationFactor"); + assertThat(issue.severity()).isEqualTo(TopologyIssue.Severity.BLOCKING); + }); + } + + @Test + void aMissingRequiredConfigurationEntryBlocks() { + List issues = validator.compare(manifest(), observed(12, 3, Map.of())); + + assertThat(issues) + .singleElement() + .satisfies(issue -> assertThat(issue.actual()).isEqualTo("unset")); + } + + @Test + void everyIssueIsCollectedRatherThanFailingOnTheFirst() { + List issues = validator.compare(manifest(), observed(6, 1, Map.of())); + + assertThat(issues) + .as("discovering the next problem only after a redeploy turns one fix into several") + .hasSize(3); + } + + @Test + void aReportRefusesStartupWhenAnythingBlocks() { + TopologyValidationReport report = + new TopologyValidationReport( + validator.compare(manifest(), observed(12, 1, Map.of("min.insync.replicas", "2"))), 1); + + assertThat(report.isAcceptable()).isFalse(); + assertThatThrownBy(report::requireAcceptable) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("replicationFactor"); + } + + @Test + void anAdvisoryOnlyReportStillStarts() { + TopologyValidationReport report = + new TopologyValidationReport( + validator.compare(manifest(), observed(24, 3, Map.of("min.insync.replicas", "2"))), 1); + + assertThat(report.isAcceptable()).isTrue(); + assertThat(report.advisory()).hasSize(1); + assertThatCode(report::requireAcceptable).doesNotThrowAnyException(); + } + + @Test + void theCompositeValidatorChecksEveryManifest() { + CompositeTopologyValidator composite = + new CompositeTopologyValidator( + new BrokerTopologyInspector() { + @Override + public Optional describe(String physicalName) { + return Optional.empty(); + } + + @Override + public String topologyVersion() { + return "v1"; + } + }); + + TopologyValidationReport report = + composite.validate( + List.of( + manifest(), new TopologyManifest("payments.v1", "payments-v1", 3, 3, Map.of()))); + + assertThat(report.destinationsChecked()).isEqualTo(2); + assertThat(report.blocking()).hasSize(2); + } + + @Test + void autoCreationIsRefusedInProduction() { + assertThatThrownBy(() -> TopologyManagementMode.CREATE_IF_MISSING.requireSafeFor(true)) + .as("a mistyped destination would be created and look exactly like a real one") + .isInstanceOf(MessagingConfigurationException.class); + } + + @Test + void autoCreationIsAllowedOutsideProduction() { + assertThatCode(() -> TopologyManagementMode.CREATE_IF_MISSING.requireSafeFor(false)) + .doesNotThrowAnyException(); + } + + @Test + void validateOnlyIsAlwaysSafe() { + assertThatCode(() -> TopologyManagementMode.VALIDATE_ONLY.requireSafeFor(true)) + .doesNotThrowAnyException(); + } +} diff --git a/src/messaging/messaging-claim-check/build.gradle b/src/messaging/messaging-claim-check/build.gradle new file mode 100644 index 00000000..015b28b4 --- /dev/null +++ b/src/messaging/messaging-claim-check/build.gradle @@ -0,0 +1,6 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-reliability-api') +} diff --git a/src/messaging/messaging-claim-check/gradle.lockfile b/src/messaging/messaging-claim-check/gradle.lockfile new file mode 100644 index 00000000..599ff921 --- /dev/null +++ b/src/messaging/messaging-claim-check/gradle.lockfile @@ -0,0 +1,83 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty=compileClasspath,runtimeClasspath diff --git a/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckIntegrityException.java b/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckIntegrityException.java new file mode 100644 index 00000000..dbc13832 --- /dev/null +++ b/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckIntegrityException.java @@ -0,0 +1,44 @@ +package dev.caskeleton.messaging.claimcheck; + +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.error.MessagingException; +import java.io.Serial; +import java.util.Optional; + +/** + * The stored payload does not match the reference the message carried. + * + *

Not retryable. A digest mismatch means the object at that key is not the object the producer + * wrote — the key was reused, the object was overwritten, or something truncated it — and fetching + * it again returns the same wrong bytes. Retrying would only delay the dead-letter. + * + *

Deliberately distinct from "the object is gone". An expired claim check is an operational + * problem with a known cause and a known fix; a digest mismatch means something wrote data nobody + * expected, and the two must not be diagnosed as one. + */ +public class ClaimCheckIntegrityException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.POISON_MESSAGE; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public ClaimCheckIntegrityException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public ClaimCheckIntegrityException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckIntegrityGuard.java b/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckIntegrityGuard.java new file mode 100644 index 00000000..461f966a --- /dev/null +++ b/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckIntegrityGuard.java @@ -0,0 +1,71 @@ +package dev.caskeleton.messaging.claimcheck; + +import dev.caskeleton.messaging.api.error.MessageValidationException; +import dev.caskeleton.messaging.reliability.ClaimCheckReference; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.util.HexFormat; +import java.util.Objects; + +/** + * Verifies a fetched claim check payload before it is handed to a codec. + * + *

A claim check turns one message into two systems that can drift. The payload store has its own + * retention, its own replication, and its own access control, and none of them are coordinated with + * the broker's. So a consumer that fetches bytes and decodes them without checking is trusting + * something the message never proved. + * + *

Both checks fail closed. An expired reference is reported before the fetch, because a + * not-found from the store is ambiguous between "reaped" and "never written". A digest mismatch is + * reported as validation rather than deserialization, because the bytes are not corrupt JSON — they + * are the wrong bytes. + */ +public final class ClaimCheckIntegrityGuard { + + /** + * Verifies a payload against its reference. + * + * @param reference the claim check reference + * @param payload the bytes fetched from the store + * @param now the current instant + * @return the verified payload + * @throws MessageValidationException when the reference expired or the digest does not match + */ + public byte[] verify(ClaimCheckReference reference, byte[] payload, Instant now) { + Objects.requireNonNull(reference, "reference must not be null"); + Objects.requireNonNull(payload, "payload must not be null"); + Objects.requireNonNull(now, "now must not be null"); + + if (reference.isExpired(now)) { + throw new MessageValidationException( + "CLAIM_CHECK_EXPIRED", + "the claim check payload retention expired at " + reference.expiresAt()); + } + if (payload.length != reference.sizeBytes()) { + throw new MessageValidationException( + "CLAIM_CHECK_SIZE_MISMATCH", + "expected " + reference.sizeBytes() + " bytes but read " + payload.length); + } + String actual = sha256(payload); + if (!actual.equals(reference.sha256())) { + throw new MessageValidationException( + "CLAIM_CHECK_DIGEST_MISMATCH", "the fetched payload does not match its reference digest"); + } + return payload.clone(); + } + + /** + * Returns the lowercase hex SHA-256 of a payload. + * + * @param payload the bytes to digest + * @return the digest + */ + public static String sha256(byte[] payload) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(payload)); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("Java runtime does not provide SHA-256", exception); + } + } +} diff --git a/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckPolicy.java b/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckPolicy.java new file mode 100644 index 00000000..7458a4fc --- /dev/null +++ b/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckPolicy.java @@ -0,0 +1,89 @@ +package dev.caskeleton.messaging.claimcheck; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.time.Duration; +import java.util.Objects; + +/** + * When a payload is offloaded, and how long the object must outlive the message. + * + *

The retention rule is the one that matters. A claim check object deleted while its message is + * still deliverable turns a large message into an undeliverable one — the consumer fetches, gets + * nothing, and the message dead-letters for a reason that has nothing to do with the message. So + * retention must exceed the broker's own retention plus the full retry and dead-letter window, and + * the constructor refuses a configuration where it does not. + * + *

The threshold is separate from the destination's payload limit. Offloading starts well below + * the limit, because the limit is where the broker refuses the message and the threshold is where + * carrying it inline stops being a good idea. + * + * @param thresholdBytes the encoded size above which a payload is offloaded + * @param retention how long the stored object must remain readable + * @param brokerRetention how long the broker keeps the message + * @param maxRedeliveryWindow the longest retry and dead-letter path a message can take + */ +public record ClaimCheckPolicy( + int thresholdBytes, + Duration retention, + Duration brokerRetention, + Duration maxRedeliveryWindow) { + + /** The default offload threshold: a quarter of the portable payload limit. */ + public static final int DEFAULT_THRESHOLD_BYTES = 262_144; + + public ClaimCheckPolicy { + Objects.requireNonNull(retention, "retention must not be null"); + Objects.requireNonNull(brokerRetention, "brokerRetention must not be null"); + Objects.requireNonNull(maxRedeliveryWindow, "maxRedeliveryWindow must not be null"); + if (thresholdBytes < 1) { + throw new IllegalArgumentException("thresholdBytes must be positive"); + } + requirePositive(retention, "retention"); + requirePositive(brokerRetention, "brokerRetention"); + requirePositive(maxRedeliveryWindow, "maxRedeliveryWindow"); + + Duration required = brokerRetention.plus(maxRedeliveryWindow); + if (retention.compareTo(required) < 0) { + throw new MessagingConfigurationException( + "CLAIM_CHECK_RETENTION_TOO_SHORT", + "claim check retention of %s is below the %s the message can remain deliverable; the " + .formatted(retention, required) + + "object would be reaped while a consumer can still be handed its message"); + } + } + + /** + * Returns a policy for a broker retaining one day with a one-day retry path. + * + * @return the default policy + */ + public static ClaimCheckPolicy defaults() { + return new ClaimCheckPolicy( + DEFAULT_THRESHOLD_BYTES, Duration.ofDays(3), Duration.ofDays(1), Duration.ofDays(1)); + } + + /** + * Reports whether a payload of this size is offloaded. + * + * @param payloadBytes the encoded payload size + * @return true when the payload travels by reference + */ + public boolean shouldOffload(int payloadBytes) { + return payloadBytes > thresholdBytes; + } + + /** + * Returns the shortest retention this deployment allows. + * + * @return the minimum safe retention + */ + public Duration requiredRetention() { + return brokerRetention.plus(maxRedeliveryWindow); + } + + private static void requirePositive(Duration value, String field) { + if (value.isNegative() || value.isZero()) { + throw new IllegalArgumentException(field + " must be positive"); + } + } +} diff --git a/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckPublisher.java b/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckPublisher.java new file mode 100644 index 00000000..630b5035 --- /dev/null +++ b/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckPublisher.java @@ -0,0 +1,91 @@ +package dev.caskeleton.messaging.claimcheck; + +import dev.caskeleton.messaging.reliability.ClaimCheckReference; +import java.util.Objects; +import java.util.Optional; + +/** + * Decides whether a payload travels inline or by reference, and stores it when it does not. + * + *

The object is written before the message is published, and that order is the whole + * design. Publishing first would let a consumer receive a reference to an object that does not + * exist yet — a race that is rare in a test and routine under load, because the broker hop is + * faster than the object store write. + * + *

Nothing here deletes on failure. If the publish is rejected the object is left behind, and the + * retention sweep reclaims it; deleting eagerly would delete the object out from under a publish + * that turned out to be ambiguous rather than rejected. + */ +public final class ClaimCheckPublisher { + + private final ClaimCheckStore store; + private final ClaimCheckPolicy policy; + + /** + * Creates a claim check publisher. + * + * @param store the payload store + * @param policy the offload threshold and retention + */ + public ClaimCheckPublisher(ClaimCheckStore store, ClaimCheckPolicy policy) { + this.store = Objects.requireNonNull(store, "store must not be null"); + this.policy = Objects.requireNonNull(policy, "policy must not be null"); + } + + /** + * Offloads a payload when the policy calls for it. + * + * @param payload the encoded payload bytes + * @return the outcome, carrying either the inline payload or the stored reference + */ + public Offloaded offload(byte[] payload) { + Objects.requireNonNull(payload, "payload must not be null"); + + if (!policy.shouldOffload(payload.length)) { + return new Offloaded(payload.clone(), Optional.empty()); + } + ClaimCheckReference reference = store.put(payload, policy.retention()); + // The published message carries no payload bytes at all, only the reference. Carrying both + // would double the transfer for no benefit and let the two disagree. + return new Offloaded(new byte[0], Optional.of(reference)); + } + + /** + * Returns the policy this publisher applies. + * + * @return the claim check policy + */ + public ClaimCheckPolicy policy() { + return policy; + } + + /** + * What a payload became after the offload decision. + * + * @param payload the payload to publish, empty when offloaded + * @param reference the stored object's reference, present when offloaded + */ + @SuppressWarnings("ArrayRecordComponent") + public record Offloaded(byte[] payload, Optional reference) { + + public Offloaded { + Objects.requireNonNull(payload, "payload must not be null"); + Objects.requireNonNull(reference, "reference must not be null"); + payload = payload.clone(); + } + + @Override + public byte[] payload() { + return payload.clone(); + } + + /** + * Reports whether the payload travels by reference. + * + * @return true when the payload was offloaded + */ + public boolean isOffloaded() { + return reference.isPresent(); + } + } +} diff --git a/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckResolver.java b/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckResolver.java new file mode 100644 index 00000000..02d919f6 --- /dev/null +++ b/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckResolver.java @@ -0,0 +1,91 @@ +package dev.caskeleton.messaging.claimcheck; + +import dev.caskeleton.messaging.api.error.MessageValidationException; +import dev.caskeleton.messaging.reliability.ClaimCheckReference; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * Fetches an offloaded payload and verifies it before a handler ever sees it. + * + *

Verification is not optional and cannot be skipped by a caller. An object store key is a + * string, and a message carrying the wrong one — through a bug, a replay against a rotated bucket, + * or a deliberate tamper — fetches bytes that decode perfectly into the wrong object. The digest is + * the only thing standing between that and a handler acting on someone else's data. + * + *

Failures are classified rather than merged. An expired reference is an operational problem + * whose fix is a retention change; a digest mismatch means something wrote data nobody expected. + * Both dead-letter the message, but an operator seeing one code should not have to guess which + * happened. + */ +public final class ClaimCheckResolver { + + private final ClaimCheckStore store; + private final ClaimCheckIntegrityGuard guard = new ClaimCheckIntegrityGuard(); + + /** + * Creates a resolver. + * + * @param store the payload store + */ + public ClaimCheckResolver(ClaimCheckStore store) { + this.store = Objects.requireNonNull(store, "store must not be null"); + } + + /** + * Returns the payload a message carries, fetching it when it travels by reference. + * + * @param inline the payload bytes the message carried, empty when it travels by reference + * @param reference the claim check reference, when the message carried one + * @param now the current instant + * @return the payload the handler should see + * @throws ClaimCheckIntegrityException when the stored object is not the one referenced + * @throws MessageValidationException when the reference has expired + */ + public byte[] resolve(byte[] inline, Optional reference, Instant now) { + Objects.requireNonNull(inline, "inline must not be null"); + Objects.requireNonNull(reference, "reference must not be null"); + Objects.requireNonNull(now, "now must not be null"); + + if (reference.isEmpty()) { + return inline.clone(); + } + ClaimCheckReference claimCheck = reference.get(); + + if (claimCheck.isExpired(now)) { + // Checked before fetching. A store that still returns the object past its retention would + // otherwise hide a misconfiguration until the day the sweep caught up. + throw new MessageValidationException( + "CLAIM_CHECK_EXPIRED", + "the claim check retention expired at %s; the object may already be reaped" + .formatted(claimCheck.expiresAt())); + } + + return verify(claimCheck, fetch(claimCheck), now); + } + + private byte[] fetch(ClaimCheckReference reference) { + byte[] fetched = store.get(reference); + if (fetched == null) { + throw new MessageValidationException( + "CLAIM_CHECK_NOT_FOUND", + "no object exists at the referenced key; it was either reaped early or never written"); + } + return fetched; + } + + private byte[] verify(ClaimCheckReference reference, byte[] fetched, Instant now) { + try { + return guard.verify(reference, fetched, now); + } catch (MessageValidationException validation) { + // A size or digest mismatch is a poison message, not a validation failure to be retried: + // fetching the same key again returns the same wrong bytes. + if (validation.failure().code().endsWith("_MISMATCH")) { + throw new ClaimCheckIntegrityException( + validation.failure().code(), validation.failure().sanitizedMessage()); + } + throw validation; + } + } +} diff --git a/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckStore.java b/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckStore.java new file mode 100644 index 00000000..4d10dfe6 --- /dev/null +++ b/src/messaging/messaging-claim-check/src/main/java/dev/caskeleton/messaging/claimcheck/ClaimCheckStore.java @@ -0,0 +1,32 @@ +package dev.caskeleton.messaging.claimcheck; + +import dev.caskeleton.messaging.reliability.ClaimCheckReference; +import java.time.Duration; + +/** Stores and retrieves payloads that are too large to travel through the broker. */ +public interface ClaimCheckStore { + + /** + * Stores a payload and returns its reference. + * + * @param payload the bytes to store + * @param retention how long the object must remain readable + * @return the reference to publish in place of the payload + */ + ClaimCheckReference put(byte[] payload, Duration retention); + + /** + * Fetches a payload, verifying it against its reference. + * + * @param reference the claim check reference + * @return the stored bytes + */ + byte[] get(ClaimCheckReference reference); + + /** + * Deletes a stored payload. + * + * @param reference the claim check reference + */ + void delete(ClaimCheckReference reference); +} diff --git a/src/messaging/messaging-claim-check/src/test/java/dev/caskeleton/messaging/claimcheck/ClaimCheckIntegrityGuardTest.java b/src/messaging/messaging-claim-check/src/test/java/dev/caskeleton/messaging/claimcheck/ClaimCheckIntegrityGuardTest.java new file mode 100644 index 00000000..71a66382 --- /dev/null +++ b/src/messaging/messaging-claim-check/src/test/java/dev/caskeleton/messaging/claimcheck/ClaimCheckIntegrityGuardTest.java @@ -0,0 +1,86 @@ +package dev.caskeleton.messaging.claimcheck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.error.MessageValidationException; +import dev.caskeleton.messaging.reliability.ClaimCheckReference; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import org.junit.jupiter.api.Test; + +class ClaimCheckIntegrityGuardTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + private static final byte[] PAYLOAD = "a large document".getBytes(StandardCharsets.UTF_8); + + private final ClaimCheckIntegrityGuard guard = new ClaimCheckIntegrityGuard(); + + @Test + void acceptsAPayloadMatchingItsReference() { + ClaimCheckReference reference = + reference(PAYLOAD.length, ClaimCheckIntegrityGuard.sha256(PAYLOAD)); + + assertThat(guard.verify(reference, PAYLOAD, NOW)).isEqualTo(PAYLOAD); + } + + @Test + void rejectsAPayloadWhoseDigestDoesNotMatch() { + ClaimCheckReference reference = + reference( + PAYLOAD.length, + ClaimCheckIntegrityGuard.sha256( + "a different document".getBytes(StandardCharsets.UTF_8))); + + assertThatThrownBy(() -> guard.verify(reference, PAYLOAD, NOW)) + .isInstanceOf(MessageValidationException.class) + .hasMessageContaining("digest"); + } + + @Test + void rejectsATruncatedPayloadBeforeHashingIt() { + ClaimCheckReference reference = + reference(PAYLOAD.length + 10, ClaimCheckIntegrityGuard.sha256(PAYLOAD)); + + assertThatThrownBy(() -> guard.verify(reference, PAYLOAD, NOW)) + .isInstanceOf(MessageValidationException.class) + .hasMessageContaining("bytes"); + } + + @Test + void rejectsAnExpiredReferenceBeforeTheFetchIsTrusted() { + ClaimCheckReference reference = + new ClaimCheckReference( + "payloads/o-1", + PAYLOAD.length, + ClaimCheckIntegrityGuard.sha256(PAYLOAD), + NOW.minusSeconds(1)); + + assertThatThrownBy(() -> guard.verify(reference, PAYLOAD, NOW)) + .isInstanceOf(MessageValidationException.class) + .hasMessageContaining("retention"); + } + + @Test + void aReferenceRequiresALowercaseHexDigest() { + assertThatThrownBy( + () -> new ClaimCheckReference("payloads/o-1", 5, "NOT-A-DIGEST", NOW.plusSeconds(60))) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void theVerifiedPayloadIsACopy() { + ClaimCheckReference reference = + reference(PAYLOAD.length, ClaimCheckIntegrityGuard.sha256(PAYLOAD)); + + byte[] verified = guard.verify(reference, PAYLOAD, NOW); + verified[0] = 'z'; + + assertThatCode(() -> guard.verify(reference, PAYLOAD, NOW)).doesNotThrowAnyException(); + } + + private static ClaimCheckReference reference(long sizeBytes, String sha256) { + return new ClaimCheckReference("payloads/o-1", sizeBytes, sha256, NOW.plusSeconds(3600)); + } +} diff --git a/src/messaging/messaging-claim-check/src/test/java/dev/caskeleton/messaging/claimcheck/ClaimCheckResolverTest.java b/src/messaging/messaging-claim-check/src/test/java/dev/caskeleton/messaging/claimcheck/ClaimCheckResolverTest.java new file mode 100644 index 00000000..73e82044 --- /dev/null +++ b/src/messaging/messaging-claim-check/src/test/java/dev/caskeleton/messaging/claimcheck/ClaimCheckResolverTest.java @@ -0,0 +1,168 @@ +package dev.caskeleton.messaging.claimcheck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.error.MessageValidationException; +import dev.caskeleton.messaging.reliability.ClaimCheckReference; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class ClaimCheckResolverTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + private static final byte[] PAYLOAD = "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8); + + /** An in-memory store that can be made to return the wrong bytes on purpose. */ + private static final class FakeStore implements ClaimCheckStore { + + private final Map objects = new HashMap<>(); + private int puts; + + @Override + public ClaimCheckReference put(byte[] payload, Duration retention) { + puts++; + String key = "claim/" + puts; + objects.put(key, payload.clone()); + return new ClaimCheckReference( + key, payload.length, ClaimCheckIntegrityGuard.sha256(payload), NOW.plus(retention)); + } + + @Override + public byte[] get(ClaimCheckReference reference) { + return objects.get(reference.storageKey()); + } + + @Override + public void delete(ClaimCheckReference reference) { + objects.remove(reference.storageKey()); + } + + void overwrite(String key, byte[] replacement) { + objects.put(key, replacement); + } + + int puts() { + return puts; + } + } + + private static byte[] filled(int size) { + byte[] payload = new byte[size]; + java.util.Arrays.fill(payload, (byte) 'x'); + return payload; + } + + @Test + void aSmallPayloadTravelsInlineAndIsNeverStored() { + FakeStore store = new FakeStore(); + ClaimCheckPublisher publisher = new ClaimCheckPublisher(store, ClaimCheckPolicy.defaults()); + + ClaimCheckPublisher.Offloaded offloaded = publisher.offload(PAYLOAD); + + assertThat(offloaded.isOffloaded()).isFalse(); + assertThat(store.puts()).isZero(); + } + + @Test + void aLargePayloadIsStoredAndTheMessageCarriesNoBytes() { + FakeStore store = new FakeStore(); + ClaimCheckPublisher publisher = new ClaimCheckPublisher(store, ClaimCheckPolicy.defaults()); + + ClaimCheckPublisher.Offloaded offloaded = + publisher.offload(filled(ClaimCheckPolicy.DEFAULT_THRESHOLD_BYTES + 1)); + + assertThat(offloaded.isOffloaded()).isTrue(); + assertThat(offloaded.payload().length) + .as("carrying both would double the transfer and let the two disagree") + .isZero(); + } + + @Test + void anOffloadedPayloadRoundTripsThroughTheStore() { + FakeStore store = new FakeStore(); + ClaimCheckPolicy policy = + new ClaimCheckPolicy(8, Duration.ofDays(3), Duration.ofDays(1), Duration.ofDays(1)); + ClaimCheckPublisher.Offloaded offloaded = + new ClaimCheckPublisher(store, policy).offload(PAYLOAD); + + byte[] resolved = + new ClaimCheckResolver(store).resolve(offloaded.payload(), offloaded.reference(), NOW); + + assertThat(resolved).isEqualTo(PAYLOAD); + } + + @Test + void anInlinePayloadIsReturnedWithoutTouchingTheStore() { + FakeStore store = new FakeStore(); + + byte[] resolved = new ClaimCheckResolver(store).resolve(PAYLOAD, Optional.empty(), NOW); + + assertThat(resolved).isEqualTo(PAYLOAD); + } + + @Test + void anObjectThatWasSwappedUnderTheReferenceIsAPoisonMessage() { + FakeStore store = new FakeStore(); + ClaimCheckPolicy policy = + new ClaimCheckPolicy(8, Duration.ofDays(3), Duration.ofDays(1), Duration.ofDays(1)); + ClaimCheckPublisher.Offloaded offloaded = + new ClaimCheckPublisher(store, policy).offload(PAYLOAD); + store.overwrite( + offloaded.reference().orElseThrow().storageKey(), + "{\"orderId\":\"SOMEONE-ELSE\"}".getBytes(StandardCharsets.UTF_8)); + + assertThatThrownBy( + () -> + new ClaimCheckResolver(store) + .resolve(offloaded.payload(), offloaded.reference(), NOW)) + .as("the digest is the only thing between a swapped object and the wrong data") + .isInstanceOf(ClaimCheckIntegrityException.class); + } + + @Test + void anIntegrityFailureIsNotRetryableBecauseTheKeyReturnsTheSameBytes() { + ClaimCheckIntegrityException failure = + new ClaimCheckIntegrityException("CLAIM_CHECK_DIGEST_MISMATCH", "swapped"); + + assertThat(failure.failure().retryable()).isFalse(); + } + + @Test + void amissingObjectIsReportedSeparatelyFromASwappedOne() { + FakeStore store = new FakeStore(); + ClaimCheckPolicy policy = + new ClaimCheckPolicy(8, Duration.ofDays(3), Duration.ofDays(1), Duration.ofDays(1)); + ClaimCheckPublisher.Offloaded offloaded = + new ClaimCheckPublisher(store, policy).offload(PAYLOAD); + store.delete(offloaded.reference().orElseThrow()); + + assertThatThrownBy( + () -> + new ClaimCheckResolver(store) + .resolve(offloaded.payload(), offloaded.reference(), NOW)) + .isInstanceOf(MessageValidationException.class) + .hasMessageContaining("never written"); + } + + @Test + void anExpiredReferenceIsRefusedBeforeTheStoreIsEvenAsked() { + FakeStore store = new FakeStore(); + ClaimCheckReference expired = + new ClaimCheckReference( + "claim/1", + PAYLOAD.length, + ClaimCheckIntegrityGuard.sha256(PAYLOAD), + NOW.minus(Duration.ofHours(1))); + + assertThatThrownBy( + () -> new ClaimCheckResolver(store).resolve(new byte[0], Optional.of(expired), NOW)) + .isInstanceOf(MessageValidationException.class) + .hasMessageContaining("expired"); + } +} diff --git a/src/messaging/messaging-claim-check/src/test/java/dev/caskeleton/messaging/claimcheck/ClaimCheckRetentionValidatorTest.java b/src/messaging/messaging-claim-check/src/test/java/dev/caskeleton/messaging/claimcheck/ClaimCheckRetentionValidatorTest.java new file mode 100644 index 00000000..50fc16ac --- /dev/null +++ b/src/messaging/messaging-claim-check/src/test/java/dev/caskeleton/messaging/claimcheck/ClaimCheckRetentionValidatorTest.java @@ -0,0 +1,79 @@ +package dev.caskeleton.messaging.claimcheck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.time.Duration; +import org.junit.jupiter.api.Test; + +class ClaimCheckRetentionValidatorTest { + + @Test + void retentionShorterThanTheMessageLifetimeIsRefused() { + assertThatThrownBy( + () -> + new ClaimCheckPolicy( + 1024, Duration.ofHours(6), Duration.ofDays(1), Duration.ofDays(1))) + .as("the object would be reaped while a consumer can still be handed its message") + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("reaped"); + } + + @Test + void retentionCoveringBrokerRetentionPlusTheRetryPathIsAccepted() { + assertThatCode( + () -> + new ClaimCheckPolicy( + 1024, Duration.ofDays(2), Duration.ofDays(1), Duration.ofDays(1))) + .doesNotThrowAnyException(); + } + + @Test + void theRequiredRetentionIsBrokerRetentionPlusTheRedeliveryWindow() { + ClaimCheckPolicy policy = + new ClaimCheckPolicy(1024, Duration.ofDays(5), Duration.ofDays(2), Duration.ofDays(1)); + + assertThat(policy.requiredRetention()).isEqualTo(Duration.ofDays(3)); + } + + @Test + void theDefaultsSatisfyTheirOwnRule() { + assertThatCode(ClaimCheckPolicy::defaults).doesNotThrowAnyException(); + } + + /** The portable payload limit the platform documents, restated here so the two cannot drift. */ + private static final int PORTABLE_PAYLOAD_LIMIT_BYTES = 1_048_576; + + @Test + void theOffloadThresholdSitsWellBelowThePortablePayloadLimit() { + assertThat(ClaimCheckPolicy.DEFAULT_THRESHOLD_BYTES) + .as( + "offloading starts where carrying inline stops being wise, not where the broker refuses") + .isLessThan(PORTABLE_PAYLOAD_LIMIT_BYTES); + } + + @Test + void aPayloadAtTheThresholdStillTravelsInline() { + ClaimCheckPolicy policy = ClaimCheckPolicy.defaults(); + + assertThat(policy.shouldOffload(ClaimCheckPolicy.DEFAULT_THRESHOLD_BYTES)).isFalse(); + assertThat(policy.shouldOffload(ClaimCheckPolicy.DEFAULT_THRESHOLD_BYTES + 1)).isTrue(); + } + + @Test + void aNonPositiveDurationIsRefused() { + assertThatThrownBy( + () -> new ClaimCheckPolicy(1024, Duration.ZERO, Duration.ofDays(1), Duration.ofDays(1))) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void aNonPositiveThresholdIsRefused() { + assertThatThrownBy( + () -> + new ClaimCheckPolicy(0, Duration.ofDays(3), Duration.ofDays(1), Duration.ofDays(1))) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/src/messaging/messaging-cloudevents/build.gradle b/src/messaging/messaging-cloudevents/build.gradle new file mode 100644 index 00000000..59fb5a7a --- /dev/null +++ b/src/messaging/messaging-cloudevents/build.gradle @@ -0,0 +1,9 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-schema-api') + + implementation 'io.cloudevents:cloudevents-api:4.0.1' + implementation 'io.cloudevents:cloudevents-core:4.0.1' +} diff --git a/src/messaging/messaging-cloudevents/gradle.lockfile b/src/messaging/messaging-cloudevents/gradle.lockfile new file mode 100644 index 00000000..fbfe6abf --- /dev/null +++ b/src/messaging/messaging-cloudevents/gradle.lockfile @@ -0,0 +1,85 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.cloudevents:cloudevents-api:4.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.cloudevents:cloudevents-core:4.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty= diff --git a/src/messaging/messaging-cloudevents/src/main/java/dev/caskeleton/messaging/cloudevents/CloudEventExtensions.java b/src/messaging/messaging-cloudevents/src/main/java/dev/caskeleton/messaging/cloudevents/CloudEventExtensions.java new file mode 100644 index 00000000..a0b5494e --- /dev/null +++ b/src/messaging/messaging-cloudevents/src/main/java/dev/caskeleton/messaging/cloudevents/CloudEventExtensions.java @@ -0,0 +1,24 @@ +package dev.caskeleton.messaging.cloudevents; + +/** + * The CloudEvents extension attribute names this profile writes. + * + *

CloudEvents requires extension names to be lowercase alphanumeric, which is why these are not + * simply the envelope field names. + */ +public final class CloudEventExtensions { + + /** Carries the envelope's correlation id. */ + public static final String CORRELATION_ID = "correlationid"; + + /** Carries the envelope's causation id. */ + public static final String CAUSATION_ID = "causationid"; + + /** Carries the envelope's schema version. */ + public static final String SCHEMA_VERSION = "schemaversion"; + + /** Carries the envelope's tenant identity. */ + public static final String TENANT_CONTEXT = "tenantcontext"; + + private CloudEventExtensions() {} +} diff --git a/src/messaging/messaging-cloudevents/src/main/java/dev/caskeleton/messaging/cloudevents/CloudEventMapper.java b/src/messaging/messaging-cloudevents/src/main/java/dev/caskeleton/messaging/cloudevents/CloudEventMapper.java new file mode 100644 index 00000000..b6d0bf42 --- /dev/null +++ b/src/messaging/messaging-cloudevents/src/main/java/dev/caskeleton/messaging/cloudevents/CloudEventMapper.java @@ -0,0 +1,33 @@ +package dev.caskeleton.messaging.cloudevents; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.schema.EncodedMessage; +import io.cloudevents.CloudEvent; +import java.net.URI; + +/** + * Maps between the platform envelope and CloudEvents 1.0.2. + * + *

Offered for domain and integration events only. Commands and work items are not forced through + * CloudEvents: they are internal contracts where the interoperability the specification buys does + * not pay for the attributes it requires. + */ +public interface CloudEventMapper { + + /** + * Converts an envelope to a CloudEvent. + * + * @param envelope the envelope carrying an already-encoded payload + * @param source the event source URI + * @return the CloudEvent + */ + CloudEvent toCloudEvent(MessageEnvelope envelope, URI source); + + /** + * Converts a CloudEvent back to an envelope. + * + * @param event the CloudEvent + * @return an envelope whose payload is the still-encoded data + */ + MessageEnvelope fromCloudEvent(CloudEvent event); +} diff --git a/src/messaging/messaging-cloudevents/src/main/java/dev/caskeleton/messaging/cloudevents/DefaultCloudEventMapper.java b/src/messaging/messaging-cloudevents/src/main/java/dev/caskeleton/messaging/cloudevents/DefaultCloudEventMapper.java new file mode 100644 index 00000000..5d64692a --- /dev/null +++ b/src/messaging/messaging-cloudevents/src/main/java/dev/caskeleton/messaging/cloudevents/DefaultCloudEventMapper.java @@ -0,0 +1,171 @@ +package dev.caskeleton.messaging.cloudevents; + +import dev.caskeleton.messaging.api.CausationId; +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.CorrelationId; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TenantContext; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.error.MessageValidationException; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import io.cloudevents.CloudEvent; +import io.cloudevents.CloudEventData; +import io.cloudevents.core.builder.CloudEventBuilder; +import io.cloudevents.core.data.BytesCloudEventData; +import java.net.URI; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; + +/** + * The CloudEvents 1.0.2 compatible profile. + * + *

Two mapping decisions are deliberate. An event without {@code occurredAt} is rejected rather + * than defaulted to the production instant, because {@code time} is read downstream as when the + * fact happened, not when the platform got around to serialising it. And an event with no data maps + * to an envelope with empty bytes, never to a Kafka null value: a tombstone deletes a key, and + * inventing one from an absent CloudEvent payload would turn an empty notification into a deletion. + */ +public final class DefaultCloudEventMapper implements CloudEventMapper { + + private static final String SPEC_CONTENT_TYPE_FALLBACK = "application/json"; + + @Override + public CloudEvent toCloudEvent(MessageEnvelope envelope, URI source) { + Objects.requireNonNull(envelope, "envelope must not be null"); + Objects.requireNonNull(source, "source must not be null"); + + Instant occurredAt = + envelope + .occurredAt() + .orElseThrow( + () -> + new MessageValidationException( + "CLOUDEVENT_TIME_REQUIRED", + "an event mapped to CloudEvents requires occurredAt")); + + CloudEventBuilder builder = + CloudEventBuilder.v1() + .withId(envelope.messageId().value().toString()) + .withSource(source) + .withType(envelope.messageType().value()) + .withTime(OffsetDateTime.ofInstant(occurredAt, ZoneOffset.UTC)) + .withDataContentType(envelope.contentType().value()) + .withExtension( + CloudEventExtensions.SCHEMA_VERSION, + Integer.toString(envelope.schemaVersion().value())); + + envelope + .correlationId() + .ifPresent( + value -> builder.withExtension(CloudEventExtensions.CORRELATION_ID, value.value())); + envelope + .causationId() + .ifPresent( + value -> + builder.withExtension( + CloudEventExtensions.CAUSATION_ID, value.value().value().toString())); + envelope + .tenantContext() + .ifPresent( + value -> builder.withExtension(CloudEventExtensions.TENANT_CONTEXT, value.tenantId())); + + if (envelope.payload() instanceof EncodedMessage encoded) { + encoded + .schemaReference() + .flatMap(reference -> reference.schemaUri()) + .ifPresent(builder::withDataSchema); + builder.withData(BytesCloudEventData.wrap(encoded.bytes())); + } else if (envelope.payload() instanceof byte[] bytes) { + builder.withData(BytesCloudEventData.wrap(bytes.clone())); + } else { + throw new MessageValidationException( + "CLOUDEVENT_PAYLOAD_NOT_ENCODED", + "CloudEvents mapping requires an already-encoded payload"); + } + + return builder.build(); + } + + @Override + public MessageEnvelope fromCloudEvent(CloudEvent event) { + Objects.requireNonNull(event, "event must not be null"); + + OffsetDateTime time = event.getTime(); + if (time == null) { + throw new MessageValidationException( + "CLOUDEVENT_TIME_REQUIRED", "a CloudEvent mapped to an envelope requires time"); + } + + ContentType contentType = + new ContentType( + Optional.ofNullable(event.getDataContentType()).orElse(SPEC_CONTENT_TYPE_FALLBACK)); + CloudEventData data = event.getData(); + byte[] bytes = data == null ? new byte[0] : data.toBytes(); + + Instant occurredAt = time.toInstant(); + return new MessageEnvelope<>( + new MessageId(UUID.fromString(event.getId())), + new MessageType(event.getType()), + new SchemaVersion(intExtension(event, CloudEventExtensions.SCHEMA_VERSION)), + occurredAt, + Optional.of(occurredAt), + new ProducerId(producerFrom(event.getSource())), + stringExtension(event, CloudEventExtensions.CORRELATION_ID).map(CorrelationId::new), + stringExtension(event, CloudEventExtensions.CAUSATION_ID) + .map(value -> new CausationId(new MessageId(UUID.fromString(value)))), + contentType, + Optional.empty(), + Optional.empty(), + stringExtension(event, CloudEventExtensions.TENANT_CONTEXT).map(TenantContext::new), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage(bytes, contentType, Optional.empty())); + } + + private static Optional stringExtension(CloudEvent event, String name) { + return Optional.ofNullable(event.getExtension(name)).map(Object::toString); + } + + private static int intExtension(CloudEvent event, String name) { + return stringExtension(event, name) + .map( + value -> { + try { + return Integer.valueOf(value); + } catch (NumberFormatException exception) { + throw new MessageValidationException( + "CLOUDEVENT_SCHEMA_VERSION_INVALID", + "schemaversion extension is not an integer", + exception); + } + }) + .orElseThrow( + () -> + new MessageValidationException( + "CLOUDEVENT_SCHEMA_VERSION_REQUIRED", + "schemaversion extension is required by this profile")); + } + + /** + * Derives a bounded producer id from the source URI. + * + *

The last path or scheme-specific segment is used so that a long URI does not become an + * unbounded producer name, which would leak straight into metric tags. + */ + private static String producerFrom(URI source) { + String text = source.toString(); + int separator = Math.max(text.lastIndexOf('/'), text.lastIndexOf(':')); + String candidate = + separator >= 0 && separator + 1 < text.length() ? text.substring(separator + 1) : text; + return candidate.isBlank() ? "unknown" : candidate; + } +} diff --git a/src/messaging/messaging-cloudevents/src/test/java/dev/caskeleton/messaging/cloudevents/CloudEventMappingTest.java b/src/messaging/messaging-cloudevents/src/test/java/dev/caskeleton/messaging/cloudevents/CloudEventMappingTest.java new file mode 100644 index 00000000..447666e2 --- /dev/null +++ b/src/messaging/messaging-cloudevents/src/test/java/dev/caskeleton/messaging/cloudevents/CloudEventMappingTest.java @@ -0,0 +1,160 @@ +package dev.caskeleton.messaging.cloudevents; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.CorrelationId; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TenantContext; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.error.MessageValidationException; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import io.cloudevents.CloudEvent; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.Test; + +class CloudEventMappingTest { + + private static final URI SOURCE = URI.create("urn:service:order-api"); + + private final DefaultCloudEventMapper mapper = new DefaultCloudEventMapper(); + + @Test + void mapsLogicalIdentityAndExtensions() { + MessageEnvelope envelope = CloudEventFixture.orderCreatedEnvelope(); + + CloudEvent event = mapper.toCloudEvent(envelope, SOURCE); + + assertThat(event.getId()).isEqualTo(envelope.messageId().value().toString()); + assertThat(event.getType()).isEqualTo("order.created"); + assertThat(event.getExtension(CloudEventExtensions.SCHEMA_VERSION)).isEqualTo("1"); + assertThat(event.getSource()).isEqualTo(SOURCE); + assertThat(event.getDataContentType()).isEqualTo("application/json"); + } + + @Test + void mapsCorrelationAndTenantAsExtensions() { + CloudEvent event = mapper.toCloudEvent(CloudEventFixture.orderCreatedEnvelope(), SOURCE); + + assertThat(event.getExtension(CloudEventExtensions.CORRELATION_ID)).isEqualTo("wf-1"); + assertThat(event.getExtension(CloudEventExtensions.TENANT_CONTEXT)).isEqualTo("acme"); + } + + @Test + void mapsOccurredAtToEventTime() { + CloudEvent event = mapper.toCloudEvent(CloudEventFixture.orderCreatedEnvelope(), SOURCE); + + assertThat(event.getTime()).isNotNull(); + assertThat(event.getTime().toInstant()).isEqualTo(Instant.parse("2026-08-10T09:15:00Z")); + } + + @Test + void rejectsAnEventEnvelopeWithoutOccurredAt() { + MessageEnvelope withoutOccurredAt = + CloudEventFixture.envelope(Optional.empty()); + + assertThatThrownBy(() -> mapper.toCloudEvent(withoutOccurredAt, SOURCE)) + .isInstanceOf(MessageValidationException.class) + .hasMessageContaining("occurredAt"); + } + + @Test + void roundTripsBackToAnEnvelopeWithoutInventingATombstone() { + MessageEnvelope original = CloudEventFixture.orderCreatedEnvelope(); + + MessageEnvelope restored = + mapper.fromCloudEvent(mapper.toCloudEvent(original, SOURCE)); + + assertThat(restored.messageId()).isEqualTo(original.messageId()); + assertThat(restored.messageType()).isEqualTo(original.messageType()); + assertThat(restored.schemaVersion()).isEqualTo(original.schemaVersion()); + assertThat(restored.correlationId()).contains(new CorrelationId("wf-1")); + assertThat(restored.tenantContext()).contains(new TenantContext("acme")); + assertThat(restored.payload().bytes()).isEqualTo(original.payload().bytes()); + } + + @Test + void aCloudEventWithNoDataBecomesAnEmptyPayloadNotANullValue() { + CloudEvent noData = + io.cloudevents.core.builder.CloudEventBuilder.v1() + .withId(UUID.randomUUID().toString()) + .withSource(SOURCE) + .withType("order.created") + .withTime(java.time.OffsetDateTime.parse("2026-08-10T09:15:00Z")) + .withDataContentType("application/json") + .withExtension(CloudEventExtensions.SCHEMA_VERSION, "1") + .build(); + + MessageEnvelope restored = mapper.fromCloudEvent(noData); + + assertThat(restored.payload()).isNotNull(); + assertThat(restored.payload().size()).isZero(); + } + + @Test + void rejectsAnUnencodedPayload() { + MessageEnvelope unencoded = + new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + Instant.parse("2026-08-10T09:15:00Z"), + Optional.of(Instant.parse("2026-08-10T09:15:00Z")), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + "not encoded"); + + assertThatThrownBy(() -> mapper.toCloudEvent(unencoded, SOURCE)) + .isInstanceOf(MessageValidationException.class); + } +} + +/** Builds CloudEvents mapping fixtures. */ +final class CloudEventFixture { + + private static final MessageId FIXED_ID = + new MessageId(UUID.fromString("0190f4aa-0000-7000-8000-000000000001")); + + private CloudEventFixture() {} + + static MessageEnvelope orderCreatedEnvelope() { + return envelope(Optional.of(Instant.parse("2026-08-10T09:15:00Z"))); + } + + static MessageEnvelope envelope(Optional occurredAt) { + byte[] payload = "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8); + return new MessageEnvelope<>( + FIXED_ID, + new MessageType("order.created"), + new SchemaVersion(1), + Instant.parse("2026-08-10T09:15:01Z"), + occurredAt, + new ProducerId("order-api"), + Optional.of(new CorrelationId("wf-1")), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.of(new TenantContext("acme")), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage(payload, ContentType.JSON, Optional.empty())); + } +} diff --git a/src/messaging/messaging-core-api/build.gradle b/src/messaging/messaging-core-api/build.gradle new file mode 100644 index 00000000..d67a2411 --- /dev/null +++ b/src/messaging/messaging-core-api/build.gradle @@ -0,0 +1,4 @@ +apply plugin: 'java-library' + +dependencies { +} diff --git a/src/messaging/messaging-core-api/gradle.lockfile b/src/messaging/messaging-core-api/gradle.lockfile new file mode 100644 index 00000000..599ff921 --- /dev/null +++ b/src/messaging/messaging-core-api/gradle.lockfile @@ -0,0 +1,83 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty=compileClasspath,runtimeClasspath diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/CausationId.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/CausationId.java new file mode 100644 index 00000000..40e64438 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/CausationId.java @@ -0,0 +1,18 @@ +package dev.caskeleton.messaging.api; + +import java.util.Objects; + +/** + * Identity of the message that directly caused this one. + * + *

Unlike {@link CorrelationId}, which spans a whole workflow, this points at exactly one + * predecessor and forms the causal edge used when reconstructing a flow. + * + * @param value the predecessor message identity + */ +public record CausationId(MessageId value) { + + public CausationId { + Objects.requireNonNull(value, "causationId value must not be null"); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/ContentType.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/ContentType.java new file mode 100644 index 00000000..9e4c7c78 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/ContentType.java @@ -0,0 +1,35 @@ +package dev.caskeleton.messaging.api; + +import java.util.Locale; + +/** + * Media type of an encoded payload, resolved through the codec registry. + * + * @param value the media type, for example {@code application/json} + */ +public record ContentType(String value) { + + private static final int MAX_LENGTH = 160; + + /** The Stable default codec's content type. */ + public static final ContentType JSON = new ContentType("application/json"); + + /** Avro binary content type. */ + public static final ContentType AVRO = new ContentType("application/avro"); + + /** Protobuf binary content type. */ + public static final ContentType PROTOBUF = new ContentType("application/x-protobuf"); + + /** Opaque bytes, only reachable through the M2 raw codec. */ + public static final ContentType OCTET_STREAM = new ContentType("application/octet-stream"); + + public ContentType { + if (value == null || value.isBlank() || value.length() > MAX_LENGTH) { + throw new IllegalArgumentException("contentType must contain 1 to 160 characters"); + } + if (value.indexOf('/') < 0) { + throw new IllegalArgumentException("contentType must be a media type: " + value); + } + value = value.toLowerCase(Locale.ROOT); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/CorrelationId.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/CorrelationId.java new file mode 100644 index 00000000..3d63b91a --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/CorrelationId.java @@ -0,0 +1,17 @@ +package dev.caskeleton.messaging.api; + +/** + * Workflow-scoped correlation value shared by every message of one business flow. + * + * @param value the correlation value, 1 to 160 characters + */ +public record CorrelationId(String value) { + + private static final int MAX_LENGTH = 160; + + public CorrelationId { + if (value == null || value.isBlank() || value.length() > MAX_LENGTH) { + throw new IllegalArgumentException("correlationId must contain 1 to 160 characters"); + } + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageEnvelope.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageEnvelope.java new file mode 100644 index 00000000..bc554b14 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageEnvelope.java @@ -0,0 +1,146 @@ +package dev.caskeleton.messaging.api; + +import dev.caskeleton.messaging.api.header.MessageHeaders; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * The platform's unit of transfer: identity, provenance, routing intent, and payload. + * + *

The payload is never null. A null Kafka value is a tombstone, which is a distinct + * broker-native operation with different retention semantics, so generalising it into "an envelope + * with no payload" would silently turn a delete into an event on brokers that have no such concept. + * + * @param the payload type + * @param messageId logical identity, preserved across retry, DLQ, and redrive + * @param messageType stable catalog type + * @param schemaVersion payload schema revision + * @param producedAt instant the platform created this envelope + * @param occurredAt instant the business fact occurred; required for events + * @param producer logical producing service + * @param correlationId workflow correlation + * @param causationId directly causing message + * @param contentType codec media type + * @param partitionKey distribution key + * @param orderingKey ordering key + * @param tenantContext bounded tenant identity + * @param traceContext trace propagation values + * @param headers bounded application headers + * @param payload the non-null payload + */ +public record MessageEnvelope( + MessageId messageId, + MessageType messageType, + SchemaVersion schemaVersion, + Instant producedAt, + Optional occurredAt, + ProducerId producer, + Optional correlationId, + Optional causationId, + ContentType contentType, + Optional partitionKey, + Optional orderingKey, + Optional tenantContext, + TraceContext traceContext, + MessageHeaders headers, + T payload) { + + public MessageEnvelope { + Objects.requireNonNull(messageId, "messageId must not be null"); + Objects.requireNonNull(messageType, "messageType must not be null"); + Objects.requireNonNull(schemaVersion, "schemaVersion must not be null"); + Objects.requireNonNull(producedAt, "producedAt must not be null"); + Objects.requireNonNull(occurredAt, "occurredAt must not be null"); + Objects.requireNonNull(producer, "producer must not be null"); + Objects.requireNonNull(correlationId, "correlationId must not be null"); + Objects.requireNonNull(causationId, "causationId must not be null"); + Objects.requireNonNull(contentType, "contentType must not be null"); + Objects.requireNonNull(partitionKey, "partitionKey must not be null"); + Objects.requireNonNull(orderingKey, "orderingKey must not be null"); + Objects.requireNonNull(tenantContext, "tenantContext must not be null"); + Objects.requireNonNull(traceContext, "traceContext must not be null"); + Objects.requireNonNull(headers, "headers must not be null"); + Objects.requireNonNull(payload, "payload must not be null"); + } + + /** + * Returns a copy of this envelope carrying a different payload representation. + * + *

Encoding, decoding, Claim Check offloading, and DLQ forwarding all need this, and every one + * of them must keep {@link #messageId()} intact — which is exactly what this method guarantees by + * construction. + * + * @param the replacement payload type + * @param replacement the new payload + * @return an envelope with identical identity and metadata + */ + public MessageEnvelope withPayload(R replacement) { + return new MessageEnvelope<>( + messageId, + messageType, + schemaVersion, + producedAt, + occurredAt, + producer, + correlationId, + causationId, + contentType, + partitionKey, + orderingKey, + tenantContext, + traceContext, + headers, + replacement); + } + + /** + * Returns a copy of this envelope carrying a different content type. + * + * @param replacement the new content type + * @return an envelope with identical identity and payload + */ + public MessageEnvelope withContentType(ContentType replacement) { + return new MessageEnvelope<>( + messageId, + messageType, + schemaVersion, + producedAt, + occurredAt, + producer, + correlationId, + causationId, + Objects.requireNonNull(replacement, "contentType must not be null"), + partitionKey, + orderingKey, + tenantContext, + traceContext, + headers, + payload); + } + + /** + * Returns a copy of this envelope carrying replacement headers. + * + * @param replacement the new headers + * @return an envelope with identical identity and payload + */ + public MessageEnvelope withHeaders(MessageHeaders replacement) { + return new MessageEnvelope<>( + messageId, + messageType, + schemaVersion, + producedAt, + occurredAt, + producer, + correlationId, + causationId, + contentType, + partitionKey, + orderingKey, + tenantContext, + traceContext, + Objects.requireNonNull(replacement, "headers must not be null"), + payload); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageId.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageId.java new file mode 100644 index 00000000..94189cc3 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageId.java @@ -0,0 +1,29 @@ +package dev.caskeleton.messaging.api; + +import java.util.Objects; +import java.util.UUID; + +/** + * Logical identity of a message. + * + *

This value survives publish retry, broker redelivery, retry destinations, dead lettering, and + * redrive. A new {@code MessageId} is minted only for a genuinely new business fact or command, so + * an Inbox can use it to suppress duplicate side effects. + * + * @param value the UUIDv7 identity + */ +public record MessageId(UUID value) { + + public MessageId { + Objects.requireNonNull(value, "messageId value must not be null"); + } + + /** + * Mints a new logical message identity. + * + * @return a fresh time-ordered identity + */ + public static MessageId newId() { + return new MessageId(UuidV7.next()); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageType.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageType.java new file mode 100644 index 00000000..b137cebb --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageType.java @@ -0,0 +1,20 @@ +package dev.caskeleton.messaging.api; + +/** + * Stable catalog name of a message, such as {@code order.created}. + * + *

Java class names are deliberately not usable as a message type: renaming or repackaging a + * class must never change the wire contract. + * + * @param value the catalog name, 1 to 240 characters + */ +public record MessageType(String value) { + + private static final int MAX_LENGTH = 240; + + public MessageType { + if (value == null || value.isBlank() || value.length() > MAX_LENGTH) { + throw new IllegalArgumentException("messageType must contain 1 to 240 characters"); + } + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/ProducerId.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/ProducerId.java new file mode 100644 index 00000000..29190158 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/ProducerId.java @@ -0,0 +1,20 @@ +package dev.caskeleton.messaging.api; + +/** + * Logical service identity of the component that produced a message. + * + *

This is a deployment-independent service name, not a host, pod, or connection identity, so + * that it stays a bounded value safe for metric tags. + * + * @param value the service name, 1 to 120 characters + */ +public record ProducerId(String value) { + + private static final int MAX_LENGTH = 120; + + public ProducerId { + if (value == null || value.isBlank() || value.length() > MAX_LENGTH) { + throw new IllegalArgumentException("producerId must contain 1 to 120 characters"); + } + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/SchemaVersion.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/SchemaVersion.java new file mode 100644 index 00000000..5c6b46a2 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/SchemaVersion.java @@ -0,0 +1,15 @@ +package dev.caskeleton.messaging.api; + +/** + * Monotonic schema revision of a {@link MessageType}. + * + * @param value the revision, starting at 1 + */ +public record SchemaVersion(int value) { + + public SchemaVersion { + if (value < 1) { + throw new IllegalArgumentException("schemaVersion must be positive"); + } + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/TenantContext.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/TenantContext.java new file mode 100644 index 00000000..a71c33cc --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/TenantContext.java @@ -0,0 +1,24 @@ +package dev.caskeleton.messaging.api; + +import java.util.regex.Pattern; + +/** + * Bounded tenant identity carried with a message. + * + *

The value is deliberately constrained to a short slug. Tenant identity is one of the few + * envelope fields that observability code is tempted to use as a metric label, and an unbounded + * tenant id turns that into a cardinality explosion. + * + * @param tenantId the tenant slug + */ +public record TenantContext(String tenantId) { + + private static final Pattern VALID = Pattern.compile("[a-z0-9][a-z0-9._-]{0,63}"); + + public TenantContext { + if (tenantId == null || !VALID.matcher(tenantId).matches()) { + throw new IllegalArgumentException( + "tenantId must match [a-z0-9][a-z0-9._-]{0,63}: " + tenantId); + } + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/TraceContext.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/TraceContext.java new file mode 100644 index 00000000..ee553f3c --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/TraceContext.java @@ -0,0 +1,44 @@ +package dev.caskeleton.messaging.api; + +import java.util.Objects; +import java.util.Optional; + +/** + * W3C trace propagation values carried with a message. + * + *

The platform creates and propagates these; handlers do not set them. Keeping them on the + * envelope rather than only in headers means a trace survives an Outbox round trip through the + * database, where broker headers do not exist yet. + * + * @param traceparent the {@code traceparent} value when a trace is active + * @param tracestate the {@code tracestate} value when present + * @param baggage the {@code baggage} value when present + */ +public record TraceContext( + Optional traceparent, Optional tracestate, Optional baggage) { + + public TraceContext { + Objects.requireNonNull(traceparent, "traceparent must not be null"); + Objects.requireNonNull(tracestate, "tracestate must not be null"); + Objects.requireNonNull(baggage, "baggage must not be null"); + } + + /** + * Returns a context with no active trace. + * + * @return an empty trace context + */ + public static TraceContext none() { + return new TraceContext(Optional.empty(), Optional.empty(), Optional.empty()); + } + + /** + * Returns a context carrying only a trace parent. + * + * @param traceparent the {@code traceparent} value + * @return a trace context + */ + public static TraceContext of(String traceparent) { + return new TraceContext(Optional.of(traceparent), Optional.empty(), Optional.empty()); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/UuidV7.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/UuidV7.java new file mode 100644 index 00000000..b5d920d1 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/UuidV7.java @@ -0,0 +1,62 @@ +package dev.caskeleton.messaging.api; + +import java.security.SecureRandom; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Generates RFC 9562 UUIDv7 values. + * + *

The platform needs message identifiers that sort by creation time so that outbox scans, DLQ + * listings, and broker partitions stay locality-friendly, while still being globally unique. The 12 + * bit {@code rand_a} field is used as a monotonic intra-millisecond counter instead of random bits: + * two calls in the same millisecond are then still strictly ordered, which is what makes "same + * logical message keeps the same id" auditable across a retry. + */ +public final class UuidV7 { + + private static final SecureRandom RANDOM = new SecureRandom(); + + /** Packs the 48-bit millisecond timestamp in the high bits and the 12-bit counter in the low. */ + private static final AtomicLong STATE = new AtomicLong(); + + private static final long COUNTER_BITS = 12L; + private static final long COUNTER_MASK = 0xFFFL; + private static final long VERSION_7 = 0x7L; + private static final long VARIANT_RFC9562 = 0x8000_0000_0000_0000L; + private static final long RANDOM_B_MASK = 0x3FFF_FFFF_FFFF_FFFFL; + + private UuidV7() {} + + /** + * Returns the next monotonically increasing UUIDv7. + * + * @return a version 7, variant 2 UUID + */ + public static UUID next() { + long state = STATE.updateAndGet(UuidV7::advance); + long timestamp = state >>> COUNTER_BITS; + long counter = state & COUNTER_MASK; + + long mostSignificantBits = (timestamp << 16) | (VERSION_7 << COUNTER_BITS) | counter; + long leastSignificantBits = (RANDOM.nextLong() & RANDOM_B_MASK) | VARIANT_RFC9562; + return new UUID(mostSignificantBits, leastSignificantBits); + } + + /** + * Advances the packed state. + * + *

When the clock moved forward the counter restarts at zero. Otherwise the packed value is + * simply incremented: that bumps the counter and, once the 12-bit counter overflows, carries into + * the timestamp field. A backwards clock step therefore never produces a duplicate or a + * descending id, it only borrows from the future. + */ + private static long advance(long previous) { + long now = System.currentTimeMillis(); + long previousTimestamp = previous >>> COUNTER_BITS; + if (now > previousTimestamp) { + return now << COUNTER_BITS; + } + return previous + 1; + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/BatchDeliveryMetadata.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/BatchDeliveryMetadata.java new file mode 100644 index 00000000..61b9ae7e --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/BatchDeliveryMetadata.java @@ -0,0 +1,46 @@ +package dev.caskeleton.messaging.api.delivery; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * Batch-wide facts about one delivered batch. + * + *

{@code orderingUnit} is what makes a batch safe to hand to an ordered destination. A batch + * drawn from two partitions cannot be settled or retried as a unit without reordering one of them, + * so an ordered destination requires the batch to name exactly one ordering unit and the runtime + * rejects a batch that spans more. + * + * @param destination the logical destination + * @param size the number of deliveries in the batch + * @param orderingUnit the single partition or queue the batch was drawn from, when it has one + * @param settlableAsBatch whether the broker can settle the whole batch in one operation + * @param receivedAt when the consumer assembled the batch + */ +public record BatchDeliveryMetadata( + DestinationName destination, + int size, + Optional orderingUnit, + boolean settlableAsBatch, + Instant receivedAt) { + + public BatchDeliveryMetadata { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(orderingUnit, "orderingUnit must not be null"); + Objects.requireNonNull(receivedAt, "receivedAt must not be null"); + if (size < 1) { + throw new IllegalArgumentException("a delivered batch has at least one delivery"); + } + } + + /** + * Reports whether this batch may be handed to a destination with strict ordering. + * + * @return true when the batch came from exactly one ordering unit + */ + public boolean isSafeForOrderedDestination() { + return orderingUnit.isPresent(); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/BatchMessageDelivery.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/BatchMessageDelivery.java new file mode 100644 index 00000000..54c035c8 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/BatchMessageDelivery.java @@ -0,0 +1,29 @@ +package dev.caskeleton.messaging.api.delivery; + +import java.util.List; +import java.util.Objects; + +/** + * A batch of decoded messages handed to a batch handler. + * + * @param the payload type + * @param deliveries the individual deliveries, in broker order + * @param metadata batch-wide facts + */ +public record BatchMessageDelivery( + List> deliveries, BatchDeliveryMetadata metadata) { + + public BatchMessageDelivery { + Objects.requireNonNull(deliveries, "deliveries must not be null"); + Objects.requireNonNull(metadata, "metadata must not be null"); + deliveries = List.copyOf(deliveries); + if (deliveries.isEmpty()) { + throw new IllegalArgumentException("a delivered batch is never empty"); + } + if (deliveries.size() != metadata.size()) { + throw new IllegalArgumentException( + "metadata size %d does not match %d deliveries" + .formatted(metadata.size(), deliveries.size())); + } + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/BatchMessageHandler.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/BatchMessageHandler.java new file mode 100644 index 00000000..655e920f --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/BatchMessageHandler.java @@ -0,0 +1,24 @@ +package dev.caskeleton.messaging.api.delivery; + +import java.util.concurrent.CompletionStage; + +/** + * The M2 batch consume entry point. + * + *

The handler returns one {@link HandleResult} for the whole batch. On a broker that settles + * batches atomically the runtime applies that result once; on a broker that settles per message it + * applies the same result to each delivery. Either way the handler is not asked to reason about + * which settlement mode it is running under. + * + * @param the payload type + */ +public interface BatchMessageHandler { + + /** + * Handles one delivered batch. + * + * @param batch the batch to handle + * @return a stage completing with the outcome for the whole batch + */ + CompletionStage handle(BatchMessageDelivery batch); +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/DeliveryContext.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/DeliveryContext.java new file mode 100644 index 00000000..2a44d71f --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/DeliveryContext.java @@ -0,0 +1,36 @@ +package dev.caskeleton.messaging.api.delivery; + +import java.time.Instant; +import java.util.Objects; + +/** + * Runtime context handed to a handler alongside a delivery. + * + *

{@code shutdownRequested} is visible to handlers on purpose: during a graceful drain the + * platform stops creating new retry attempts, and a long-running handler that can wind down early + * shortens the drain instead of being cancelled at the deadline. + * + * @param handlerDeadline the instant after which the handler is considered timed out + * @param shutdownRequested whether the consumer has begun draining + * @param consumerId the stable, low-cardinality consumer identity + */ +public record DeliveryContext( + Instant handlerDeadline, boolean shutdownRequested, String consumerId) { + + public DeliveryContext { + Objects.requireNonNull(handlerDeadline, "handlerDeadline must not be null"); + if (consumerId == null || consumerId.isBlank()) { + throw new IllegalArgumentException("consumerId must not be blank"); + } + } + + /** + * Reports whether the handler deadline has passed. + * + * @param now the current instant + * @return true when the deadline has elapsed + */ + public boolean isExpired(Instant now) { + return !now.isBefore(handlerDeadline); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/DeliveryGuarantee.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/DeliveryGuarantee.java new file mode 100644 index 00000000..492247c0 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/DeliveryGuarantee.java @@ -0,0 +1,18 @@ +package dev.caskeleton.messaging.api.delivery; + +/** + * Broker-level delivery guarantee offered by the common contract. + * + *

There is deliberately no {@code EXACTLY_ONCE} constant. No broker delivers exactly-once across + * an external side effect; what real systems provide is at-least-once delivery combined with an + * idempotent or transactional consumer. Naming a guarantee the platform cannot honour would push + * that responsibility out of sight, so the enum stops where the evidence stops. + */ +public enum DeliveryGuarantee { + + /** Delivery may be lost; duplicate suppression is preferred over durability. */ + AT_MOST_ONCE, + + /** Redelivery is possible; durability is preferred over duplicate suppression. */ + AT_LEAST_ONCE +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/DeliveryMetadata.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/DeliveryMetadata.java new file mode 100644 index 00000000..3e107c2f --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/DeliveryMetadata.java @@ -0,0 +1,46 @@ +package dev.caskeleton.messaging.api.delivery; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.publish.BrokerPosition; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * Transport-side facts about one delivery attempt. + * + *

The first delivery is attempt one, not zero. Off-by-one confusion here directly changes how + * many times a poison message is replayed before it is parked, so the counting rule is fixed at the + * contract rather than left to each adapter. + * + * @param destination the logical destination + * @param deliveryAttempt the attempt number, starting at one + * @param redelivered whether the broker flagged this as a redelivery + * @param brokerPosition the broker coordinate when available + * @param partitionOrQueue the ordering unit when available + * @param consumerGroup the consumer group when applicable + * @param receivedAt when the consumer received the delivery + */ +public record DeliveryMetadata( + DestinationName destination, + int deliveryAttempt, + boolean redelivered, + Optional brokerPosition, + Optional partitionOrQueue, + Optional consumerGroup, + Instant receivedAt) { + + public DeliveryMetadata { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(brokerPosition, "brokerPosition must not be null"); + Objects.requireNonNull(partitionOrQueue, "partitionOrQueue must not be null"); + Objects.requireNonNull(consumerGroup, "consumerGroup must not be null"); + Objects.requireNonNull(receivedAt, "receivedAt must not be null"); + if (deliveryAttempt < 1) { + throw new IllegalArgumentException("deliveryAttempt counts the first delivery as 1"); + } + if (deliveryAttempt == 1 && redelivered) { + throw new IllegalArgumentException("the first delivery attempt cannot be a redelivery"); + } + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/ExternalSideEffectGuarantee.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/ExternalSideEffectGuarantee.java new file mode 100644 index 00000000..8ae2a278 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/ExternalSideEffectGuarantee.java @@ -0,0 +1,14 @@ +package dev.caskeleton.messaging.api.delivery; + +/** How a handler's external side effect is protected against redelivery. */ +public enum ExternalSideEffectGuarantee { + + /** Nothing protects the side effect; only valid where replay is harmless. */ + NONE, + + /** The handler must make the side effect idempotent itself. */ + IDEMPOTENCY_REQUIRED, + + /** An Inbox row and the side effect commit inside the same database transaction. */ + INBOX_TRANSACTIONAL +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/HandleResult.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/HandleResult.java new file mode 100644 index 00000000..f4cf76f5 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/HandleResult.java @@ -0,0 +1,60 @@ +package dev.caskeleton.messaging.api.delivery; + +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import java.util.Objects; + +/** + * What an M1 handler decided about a delivery. + * + *

The handler states an intent; the platform performs the settlement. That split is what keeps + * "acknowledge only after the handler succeeded" a platform invariant rather than something each + * handler has to remember, and it is why no variant here carries a broker acknowledgement handle. + */ +public sealed interface HandleResult + permits HandleResult.Success, HandleResult.Retry, HandleResult.DeadLetter, HandleResult.Reject { + + /** Processing succeeded; the platform may settle the source. */ + record Success() implements HandleResult {} + + /** + * Processing failed in a way that may succeed later. + * + * @param failure the sanitized failure description + */ + record Retry(FailureDescriptor failure) implements HandleResult { + public Retry { + Objects.requireNonNull(failure, "failure must not be null"); + } + } + + /** + * Processing failed permanently; route to the dead letter destination. + * + * @param failure the sanitized failure description + */ + record DeadLetter(FailureDescriptor failure) implements HandleResult { + public DeadLetter { + Objects.requireNonNull(failure, "failure must not be null"); + } + } + + /** + * Discard the message without dead lettering. + * + * @param failure the sanitized failure description + */ + record Reject(FailureDescriptor failure) implements HandleResult { + public Reject { + Objects.requireNonNull(failure, "failure must not be null"); + } + } + + /** + * Returns a successful result. + * + * @return the success variant + */ + static HandleResult success() { + return new Success(); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/MessageDelivery.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/MessageDelivery.java new file mode 100644 index 00000000..ac1bc230 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/MessageDelivery.java @@ -0,0 +1,22 @@ +package dev.caskeleton.messaging.api.delivery; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import java.util.Objects; + +/** + * One decoded message handed to a handler. + * + * @param the payload type + * @param message the decoded envelope + * @param metadata transport-side delivery facts + * @param context runtime context for this attempt + */ +public record MessageDelivery( + MessageEnvelope message, DeliveryMetadata metadata, DeliveryContext context) { + + public MessageDelivery { + Objects.requireNonNull(message, "message must not be null"); + Objects.requireNonNull(metadata, "metadata must not be null"); + Objects.requireNonNull(context, "context must not be null"); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/MessageHandler.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/MessageHandler.java new file mode 100644 index 00000000..4fc54b16 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/MessageHandler.java @@ -0,0 +1,23 @@ +package dev.caskeleton.messaging.api.delivery; + +import java.util.concurrent.CompletionStage; + +/** + * The M1 typed handler implemented by ordinary business code. + * + *

Handlers state an outcome and never touch broker acknowledgement APIs. Duplicate delivery is a + * normal condition, not an error: implementations are expected to be idempotent, or to sit behind + * the Inbox. + * + * @param the payload type + */ +public interface MessageHandler { + + /** + * Handles one delivery. + * + * @param delivery the decoded message and its metadata + * @return a stage completing with the handling outcome + */ + CompletionStage handle(MessageDelivery delivery); +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/OrderingScope.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/OrderingScope.java new file mode 100644 index 00000000..8ae6dd1b --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/OrderingScope.java @@ -0,0 +1,23 @@ +package dev.caskeleton.messaging.api.delivery; + +/** + * The real scope inside which message order is preserved. + * + *

There is deliberately no {@code GLOBAL} constant. Ordering is a property of a partition, a key + * mapping, or a single consumer — never of a whole destination — and advertising a global scope + * would promise something no partitioned broker can keep. + */ +public enum OrderingScope { + + /** No order is promised. */ + NONE, + + /** Order holds across the destination, which requires a single ordering unit. */ + DESTINATION, + + /** Order holds inside one broker partition. */ + PARTITION, + + /** Order holds for one key while its mapping to an ordering unit is stable. */ + KEY +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/PauseResumeController.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/PauseResumeController.java new file mode 100644 index 00000000..5c862382 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/PauseResumeController.java @@ -0,0 +1,33 @@ +package dev.caskeleton.messaging.api.delivery; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.util.concurrent.CompletionStage; + +/** + * The M2 consumer flow-control entry point. + * + *

Pausing stops new deliveries; it does not abandon the ones already in flight. A paused + * consumer stays a member of its group and keeps its assignment, which is the point: leaving the + * group to stop consuming would trigger a rebalance and hand the work to another instance that is + * just as overloaded. + */ +public interface PauseResumeController { + + /** + * Stops new deliveries for a destination scope. + * + * @param destination the logical destination + * @param scope the partition, queue, or {@code "*"} for every assigned unit + * @return a stage completing once no further deliveries will be dispatched + */ + CompletionStage pause(DestinationName destination, String scope); + + /** + * Resumes deliveries for a destination scope. + * + * @param destination the logical destination + * @param scope the partition, queue, or {@code "*"} for every assigned unit + * @return a stage completing once deliveries may flow again + */ + CompletionStage resume(DestinationName destination, String scope); +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/ProcessingGuarantee.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/ProcessingGuarantee.java new file mode 100644 index 00000000..f6f537de --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/delivery/ProcessingGuarantee.java @@ -0,0 +1,11 @@ +package dev.caskeleton.messaging.api.delivery; + +/** How duplicate processing is neutralised once a message has been delivered. */ +public enum ProcessingGuarantee { + + /** The handler suppresses duplicate effects using the message id or a business key. */ + APPLICATION_IDEMPOTENT, + + /** Atomicity holds only inside the transaction scope the broker itself defines. */ + BROKER_TRANSACTIONAL +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/CapabilityRegistry.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/CapabilityRegistry.java new file mode 100644 index 00000000..b4628f5b --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/CapabilityRegistry.java @@ -0,0 +1,15 @@ +package dev.caskeleton.messaging.api.destination; + +/** Resolves the capability snapshot for a logical destination. */ +public interface CapabilityRegistry { + + /** + * Returns the capabilities of a destination. + * + * @param destination the logical destination + * @return the capability snapshot + * @throws dev.caskeleton.messaging.api.error.MessagingConfigurationException when the destination + * is not registered + */ + DestinationCapabilities capabilities(DestinationName destination); +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/ConfirmationRequirement.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/ConfirmationRequirement.java new file mode 100644 index 00000000..e260c916 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/ConfirmationRequirement.java @@ -0,0 +1,20 @@ +package dev.caskeleton.messaging.api.destination; + +/** + * The publish confirmation a destination profile demands. + * + *

This is the requested level. What the broker actually supplied is reported separately as a + * confirmation level on the publish evidence, so a profile asking for replication evidence against + * an adapter that can only prove a broker ack fails at startup instead of silently downgrading. + */ +public enum ConfirmationRequirement { + + /** No confirmation is required; only valid for at-most-once profiles. */ + NONE, + + /** The broker must acknowledge receipt. */ + BROKER_ACK, + + /** The broker must acknowledge replication or persistence. */ + REPLICATION_OR_PERSISTENCE_ACK +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/DestinationCapabilities.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/DestinationCapabilities.java new file mode 100644 index 00000000..fc212e0e --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/DestinationCapabilities.java @@ -0,0 +1,26 @@ +package dev.caskeleton.messaging.api.destination; + +import java.util.Objects; + +/** + * A capability snapshot bound to one logical destination. + * + *

Capabilities are per destination, not per broker: the same Kafka cluster offers keyed ordering + * on a partitioned topic and none on a share group, and the same RabbitMQ node offers native dead + * lettering only where the queue was declared with one. + * + * @param destination the logical destination + * @param broker the adapter's broker name + * @param capabilities what the adapter can prove for this destination + */ +public record DestinationCapabilities( + DestinationName destination, String broker, MessagingCapabilities capabilities) { + + public DestinationCapabilities { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(capabilities, "capabilities must not be null"); + if (broker == null || broker.isBlank()) { + throw new IllegalArgumentException("broker must not be blank"); + } + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/DestinationKind.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/DestinationKind.java new file mode 100644 index 00000000..7051e788 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/DestinationKind.java @@ -0,0 +1,32 @@ +package dev.caskeleton.messaging.api.destination; + +/** + * The interaction pattern a destination implements. + * + *

This drives validation rather than transport: an event stream may keep ordering and replay, a + * work queue may not, and a request-reply destination needs a correlation strategy no other kind + * requires. + */ +public enum DestinationKind { + + /** Point-to-point command delivered to exactly one logical handler. */ + ASYNC_COMMAND, + + /** Fact published inside one bounded context. */ + DOMAIN_EVENT, + + /** Fact published across bounded contexts under a stable schema contract. */ + INTEGRATION_EVENT, + + /** Competing consumers draining a shared backlog. */ + WORK_QUEUE, + + /** Fan-out to independent subscribers. */ + PUBLISH_SUBSCRIBE, + + /** Retained, replayable, partitioned log. */ + EVENT_STREAM, + + /** Correlated request and reply, available only as an M2 capability. */ + REQUEST_REPLY +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/DestinationName.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/DestinationName.java new file mode 100644 index 00000000..c5a12dfb --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/DestinationName.java @@ -0,0 +1,24 @@ +package dev.caskeleton.messaging.api.destination; + +import java.util.regex.Pattern; + +/** + * Logical name of a destination. + * + *

The pattern excludes {@code :}, {@code /}, and whitespace so that a broker address can never + * be smuggled in as a logical name. {@code topic://orders} has to fail here, otherwise the physical + * mapping owned by the destination profile could be bypassed from application code. + * + * @param value the logical name + */ +public record DestinationName(String value) { + + private static final Pattern VALID = Pattern.compile("[a-z0-9][a-z0-9.-]{0,159}"); + + public DestinationName { + if (value == null || !VALID.matcher(value).matches()) { + throw new IllegalArgumentException( + "destination name must match [a-z0-9][a-z0-9.-]{0,159}: " + value); + } + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/MessageDestination.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/MessageDestination.java new file mode 100644 index 00000000..253a588f --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/MessageDestination.java @@ -0,0 +1,26 @@ +package dev.caskeleton.messaging.api.destination; + +import dev.caskeleton.messaging.api.MessageType; +import java.util.Objects; + +/** + * The application-facing handle for a destination. + * + *

It carries a logical name, the catalog message type, and the payload class — never a topic, + * exchange, queue, or subject. The physical mapping lives in the destination profile, which is what + * lets the same code run against Kafka in production and an in-memory harness in tests. + * + * @param the payload type + * @param name the logical destination name + * @param messageType the catalog message type + * @param payloadType the payload class + */ +public record MessageDestination( + DestinationName name, MessageType messageType, Class payloadType) { + + public MessageDestination { + Objects.requireNonNull(name, "destination name must not be null"); + Objects.requireNonNull(messageType, "destination messageType must not be null"); + Objects.requireNonNull(payloadType, "destination payloadType must not be null"); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/MessagingCapabilities.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/MessagingCapabilities.java new file mode 100644 index 00000000..fce026ca --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/destination/MessagingCapabilities.java @@ -0,0 +1,47 @@ +package dev.caskeleton.messaging.api.destination; + +/** + * What a broker adapter can actually prove or do. + * + *

The common API does not assume every broker supports every feature. When a profile asks for + * something absent here the platform fails loudly — at startup where possible, otherwise with a + * capability exception — rather than quietly degrading, because a silently weakened guarantee is + * indistinguishable from a working one until the incident. + * + * @param brokerAcknowledgement the adapter can prove the broker accepted a publish + * @param replicationOrPersistenceEvidence the adapter can prove replication or persistence + * @param perMessageSettlement individual messages can be settled + * @param batchSettlement batches can be settled as a unit + * @param orderedStream the destination preserves order inside an ordering unit + * @param keyedOrdering order is preserved per key + * @param replay historical messages can be re-read + * @param delayedDelivery delivery can be scheduled for a future instant + * @param brokerTransaction the broker offers a transaction scope + * @param deduplicatedPublish the broker suppresses duplicate publishes of a stable id + * @param nativeDeadLetter the broker provides dead lettering itself + * @param topologyManagement topology can be inspected or created through the adapter + */ +public record MessagingCapabilities( + boolean brokerAcknowledgement, + boolean replicationOrPersistenceEvidence, + boolean perMessageSettlement, + boolean batchSettlement, + boolean orderedStream, + boolean keyedOrdering, + boolean replay, + boolean delayedDelivery, + boolean brokerTransaction, + boolean deduplicatedPublish, + boolean nativeDeadLetter, + boolean topologyManagement) { + + /** + * Returns a capability set with nothing enabled. + * + * @return the empty capability set + */ + public static MessagingCapabilities none() { + return new MessagingCapabilities( + false, false, false, false, false, false, false, false, false, false, false, false); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/FailureCategory.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/FailureCategory.java new file mode 100644 index 00000000..c7966aa9 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/FailureCategory.java @@ -0,0 +1,40 @@ +package dev.caskeleton.messaging.api.error; + +/** + * The stable classification a retry engine, DLQ router, and dashboard all agree on. + * + *

Categories exist so that retry decisions are made from a declared class of failure rather than + * from exception type matching, which drifts every time a client library is upgraded. + */ +public enum FailureCategory { + + /** Broker or network fault expected to clear on its own. */ + TRANSIENT_INFRASTRUCTURE, + + /** The broker or a downstream applied backpressure or a quota. */ + THROTTLED, + + /** The handler failed in a way that may succeed on redelivery. */ + PROCESSING_TRANSIENT, + + /** The business rejected the message; redelivery cannot help. */ + PERMANENT_BUSINESS, + + /** The message repeatedly destroys its consumer and must be parked. */ + POISON_MESSAGE, + + /** The payload could not be decoded against its schema. */ + DESERIALIZATION, + + /** Broker authentication failed. */ + AUTHENTICATION, + + /** Broker authorization denied the operation. */ + AUTHORIZATION, + + /** The outcome could not be determined. */ + AMBIGUOUS, + + /** The platform or destination is misconfigured. */ + CONFIGURATION +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/FailureDescriptor.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/FailureDescriptor.java new file mode 100644 index 00000000..5c00b102 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/FailureDescriptor.java @@ -0,0 +1,80 @@ +package dev.caskeleton.messaging.api.error; + +import java.util.Objects; +import java.util.Optional; + +/** + * A sanitized, transportable description of a failure. + * + *

This travels in reserved headers to retry destinations and DLQs, so it deliberately holds no + * payload, no stack trace, no credential, and no actual message key. Stack traces belong in secure + * log storage; a DLQ is read by more people than the log is. + * + * @param category the stable classification + * @param code a stable, machine-readable code + * @param retryable whether automatic retry is permitted + * @param sanitizedMessage a short operator-facing description + * @param exceptionType the originating exception's simple type name, when known + */ +public record FailureDescriptor( + FailureCategory category, + String code, + boolean retryable, + String sanitizedMessage, + Optional exceptionType) { + + private static final int MAX_MESSAGE_LENGTH = 512; + private static final int MAX_CODE_LENGTH = 120; + + public FailureDescriptor { + Objects.requireNonNull(category, "failure category must not be null"); + Objects.requireNonNull(exceptionType, "exceptionType must not be null"); + if (code == null || code.isBlank() || code.length() > MAX_CODE_LENGTH) { + throw new IllegalArgumentException("failure code must contain 1 to 120 characters"); + } + if (sanitizedMessage == null) { + throw new IllegalArgumentException("sanitizedMessage must not be null"); + } + if (sanitizedMessage.length() > MAX_MESSAGE_LENGTH) { + sanitizedMessage = sanitizedMessage.substring(0, MAX_MESSAGE_LENGTH); + } + } + + /** + * Builds a descriptor whose retryability follows the category's default. + * + * @param category the classification + * @param code the stable code + * @param sanitizedMessage the operator-facing description + * @return a descriptor + */ + public static FailureDescriptor of( + FailureCategory category, String code, String sanitizedMessage) { + return new FailureDescriptor( + category, code, defaultRetryable(category), sanitizedMessage, Optional.empty()); + } + + /** + * Reports whether a category is a retry candidate by default. + * + *

Deserialization, authentication, authorization, and configuration failures are never retried + * automatically: each of them fails identically on every redelivery, so retrying only multiplies + * the load while the message is still poison. + * + * @param category the classification + * @return true when automatic retry is allowed by default + */ + public static boolean defaultRetryable(FailureCategory category) { + return switch (category) { + case TRANSIENT_INFRASTRUCTURE, THROTTLED, PROCESSING_TRANSIENT -> true; + case PERMANENT_BUSINESS, + POISON_MESSAGE, + DESERIALIZATION, + AUTHENTICATION, + AUTHORIZATION, + AMBIGUOUS, + CONFIGURATION -> + false; + }; + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageAuthenticationException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageAuthenticationException.java new file mode 100644 index 00000000..c7bc1e86 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageAuthenticationException.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** Broker authentication failed. */ +public class MessageAuthenticationException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.AUTHENTICATION; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageAuthenticationException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageAuthenticationException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageAuthenticationException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageAuthorizationException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageAuthorizationException.java new file mode 100644 index 00000000..5f373006 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageAuthorizationException.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** The broker denied the operation for these credentials. */ +public class MessageAuthorizationException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.AUTHORIZATION; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageAuthorizationException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageAuthorizationException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageAuthorizationException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageBackpressureException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageBackpressureException.java new file mode 100644 index 00000000..aee51346 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageBackpressureException.java @@ -0,0 +1,40 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** + * Admission was refused because the platform is already at its in-flight or buffer ceiling. + * + *

Retryable, and deliberately raised rather than absorbed by an unbounded wait. Blocking the + * caller until a slot frees turns producer-side saturation into thread exhaustion in the calling + * application, which is a far worse failure than a fast rejection the caller can shed or retry. + * + *

Nothing was transmitted when this is thrown, so the message has no ambiguity: the caller may + * resubmit it under the same {@code messageId} without risking a duplicate. + */ +public class MessageBackpressureException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.TRANSIENT_INFRASTRUCTURE; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageBackpressureException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, true, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageBackpressureException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageBrokerUnavailableException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageBrokerUnavailableException.java new file mode 100644 index 00000000..65a96407 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageBrokerUnavailableException.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** The broker could not be reached. */ +public class MessageBrokerUnavailableException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.TRANSIENT_INFRASTRUCTURE; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageBrokerUnavailableException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, true, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageBrokerUnavailableException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + true, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageBrokerUnavailableException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageConsumerException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageConsumerException.java new file mode 100644 index 00000000..3b3b0953 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageConsumerException.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** The consumer runtime failed while handling a delivery. */ +public class MessageConsumerException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.PROCESSING_TRANSIENT; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageConsumerException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, true, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageConsumerException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + true, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageConsumerException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageDeadLetterException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageDeadLetterException.java new file mode 100644 index 00000000..210acd70 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageDeadLetterException.java @@ -0,0 +1,54 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** + * Publishing to the dead letter destination failed. + * + *

The source message stays unsettled. Acknowledging a source whose DLQ publish failed would + * destroy the only remaining copy. + */ +public class MessageDeadLetterException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.TRANSIENT_INFRASTRUCTURE; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageDeadLetterException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, true, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageDeadLetterException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + true, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageDeadLetterException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageHandlerTimeoutException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageHandlerTimeoutException.java new file mode 100644 index 00000000..9632b7f8 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageHandlerTimeoutException.java @@ -0,0 +1,54 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** + * The handler exceeded its timeout. + * + *

Whether the handler committed a side effect before the deadline is unknown, so the delivery is + * recorded as possibly duplicated when it is retried. + */ +public class MessageHandlerTimeoutException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.PROCESSING_TRANSIENT; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageHandlerTimeoutException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, true, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageHandlerTimeoutException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + true, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageHandlerTimeoutException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageHeaderRejectedException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageHeaderRejectedException.java new file mode 100644 index 00000000..7730c351 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageHeaderRejectedException.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** A header was reserved, secret, or exceeded a limit. */ +public class MessageHeaderRejectedException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.PERMANENT_BUSINESS; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageHeaderRejectedException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageHeaderRejectedException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageHeaderRejectedException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagePublishAmbiguousException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagePublishAmbiguousException.java new file mode 100644 index 00000000..26af714e --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagePublishAmbiguousException.java @@ -0,0 +1,54 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** + * The publish outcome could not be determined. + * + *

The broker may hold the message. Republishing is permitted only under the same logical message + * id, so that broker deduplication or a downstream Inbox can collapse the duplicate. + */ +public class MessagePublishAmbiguousException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.AMBIGUOUS; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessagePublishAmbiguousException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessagePublishAmbiguousException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessagePublishAmbiguousException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagePublishRejectedException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagePublishRejectedException.java new file mode 100644 index 00000000..6f913b10 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagePublishRejectedException.java @@ -0,0 +1,53 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** + * The broker definitively refused the publish. + * + *

The message is known not to be stored, so abandoning this attempt is safe. + */ +public class MessagePublishRejectedException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.PERMANENT_BUSINESS; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessagePublishRejectedException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessagePublishRejectedException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessagePublishRejectedException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagePublishTimeoutException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagePublishTimeoutException.java new file mode 100644 index 00000000..959cd7a8 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagePublishTimeoutException.java @@ -0,0 +1,54 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** + * The publish deadline expired before a confirmation arrived. + * + *

A timeout is not a rejection. The bytes may already be replicated, so this is classified as + * ambiguous rather than failed. + */ +public class MessagePublishTimeoutException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.AMBIGUOUS; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessagePublishTimeoutException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessagePublishTimeoutException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessagePublishTimeoutException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageRedriveException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageRedriveException.java new file mode 100644 index 00000000..9d4b6578 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageRedriveException.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** A redrive operation failed. */ +public class MessageRedriveException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.TRANSIENT_INFRASTRUCTURE; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageRedriveException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, true, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageRedriveException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + true, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageRedriveException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageRetryExhaustedException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageRetryExhaustedException.java new file mode 100644 index 00000000..2ffea59a --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageRetryExhaustedException.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** The retry policy's attempt budget was exhausted. */ +public class MessageRetryExhaustedException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.PERMANENT_BUSINESS; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageRetryExhaustedException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageRetryExhaustedException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageRetryExhaustedException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageRoutingException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageRoutingException.java new file mode 100644 index 00000000..c5ea68fd --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageRoutingException.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** The broker accepted the publish but could not route it. */ +public class MessageRoutingException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.PERMANENT_BUSINESS; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageRoutingException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageRoutingException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageRoutingException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSchemaIncompatibleException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSchemaIncompatibleException.java new file mode 100644 index 00000000..5709d961 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSchemaIncompatibleException.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** The payload's schema is incompatible with the registered contract. */ +public class MessageSchemaIncompatibleException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.DESERIALIZATION; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageSchemaIncompatibleException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageSchemaIncompatibleException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageSchemaIncompatibleException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSerializationException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSerializationException.java new file mode 100644 index 00000000..fce4c892 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSerializationException.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** The payload could not be encoded or decoded. */ +public class MessageSerializationException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.DESERIALIZATION; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageSerializationException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageSerializationException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageSerializationException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSettlementException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSettlementException.java new file mode 100644 index 00000000..63cd72b8 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSettlementException.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** Settlement was refused by the broker. */ +public class MessageSettlementException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.TRANSIENT_INFRASTRUCTURE; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageSettlementException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, true, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageSettlementException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + true, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageSettlementException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSettlementUnknownException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSettlementUnknownException.java new file mode 100644 index 00000000..27072f1a --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageSettlementUnknownException.java @@ -0,0 +1,53 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** + * Settlement was sent but not confirmed. + * + *

Redelivery is possible. Callers must not treat this as a successful settlement. + */ +public class MessageSettlementUnknownException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.AMBIGUOUS; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageSettlementUnknownException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageSettlementUnknownException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageSettlementUnknownException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageTooLargeException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageTooLargeException.java new file mode 100644 index 00000000..c62fb934 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageTooLargeException.java @@ -0,0 +1,54 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** + * The encoded payload exceeds the destination's byte limit. + * + *

Payloads above the portability limit are expected to use Claim Check rather than a larger + * broker frame, so this failure points at a design choice and never at a transient fault. + */ +public class MessageTooLargeException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.PERMANENT_BUSINESS; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageTooLargeException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageTooLargeException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageTooLargeException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageTopologyException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageTopologyException.java new file mode 100644 index 00000000..7a156a2e --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageTopologyException.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** The broker topology does not match the declared manifest. */ +public class MessageTopologyException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.CONFIGURATION; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageTopologyException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageTopologyException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageTopologyException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageValidationException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageValidationException.java new file mode 100644 index 00000000..56cdcf69 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessageValidationException.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** The envelope failed platform validation before transport. */ +public class MessageValidationException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.PERMANENT_BUSINESS; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessageValidationException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessageValidationException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessageValidationException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagingCapabilityUnavailableException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagingCapabilityUnavailableException.java new file mode 100644 index 00000000..84dc2678 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagingCapabilityUnavailableException.java @@ -0,0 +1,56 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** + * A requested feature is not supported by the destination's adapter. + * + *

Thrown instead of quietly degrading. Downgrading replication evidence to a bare ack, or + * ordered delivery to unordered, produces a system that looks healthy right up to the moment the + * guarantee actually mattered. + */ +public class MessagingCapabilityUnavailableException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.CONFIGURATION; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessagingCapabilityUnavailableException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessagingCapabilityUnavailableException( + String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessagingCapabilityUnavailableException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagingConfigurationException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagingConfigurationException.java new file mode 100644 index 00000000..c32678d7 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagingConfigurationException.java @@ -0,0 +1,55 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Optional; + +/** + * The platform or a destination profile is misconfigured. + * + *

Raised at startup wherever possible. A configuration fault discovered at publish time has + * already cost a request, so the destination profile validator promotes as many of these as it can + * to application startup. + */ +public class MessagingConfigurationException extends MessagingException { + + @Serial private static final long serialVersionUID = 1L; + + private static final FailureCategory CATEGORY = FailureCategory.CONFIGURATION; + + /** + * Creates the exception with a stable code and sanitized message. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + */ + public MessagingConfigurationException(String code, String sanitizedMessage) { + super(new FailureDescriptor(CATEGORY, code, false, sanitizedMessage, Optional.empty())); + } + + /** + * Creates the exception with an originating cause. + * + * @param code the stable failure code + * @param sanitizedMessage the operator-facing description + * @param cause the originating throwable + */ + public MessagingConfigurationException(String code, String sanitizedMessage, Throwable cause) { + super( + new FailureDescriptor( + CATEGORY, + code, + false, + sanitizedMessage, + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())), + cause); + } + + /** + * Creates the exception from an explicit descriptor. + * + * @param failure the sanitized failure description + */ + public MessagingConfigurationException(FailureDescriptor failure) { + super(failure); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagingException.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagingException.java new file mode 100644 index 00000000..81b648c4 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/MessagingException.java @@ -0,0 +1,66 @@ +package dev.caskeleton.messaging.api.error; + +import java.io.Serial; +import java.util.Objects; + +/** + * Root of the platform's stable exception hierarchy. + * + *

Every subclass carries a {@link FailureDescriptor} so that a caller catching the base type can + * still classify and route the failure without matching on exception classes. The message exposed + * here is the sanitized one; nothing on this type carries payload or credentials. + */ +public abstract class MessagingException extends RuntimeException { + + @Serial private static final long serialVersionUID = 1L; + + private final FailureDescriptor failure; + + /** + * Creates an exception from a descriptor. + * + * @param failure the sanitized failure description + */ + protected MessagingException(FailureDescriptor failure) { + super(Objects.requireNonNull(failure, "failure must not be null").sanitizedMessage()); + this.failure = failure; + } + + /** + * Creates an exception from a descriptor and an originating cause. + * + * @param failure the sanitized failure description + * @param cause the originating throwable + */ + protected MessagingException(FailureDescriptor failure, Throwable cause) { + super(Objects.requireNonNull(failure, "failure must not be null").sanitizedMessage(), cause); + this.failure = failure; + } + + /** + * Returns the sanitized failure description. + * + * @return the descriptor + */ + public FailureDescriptor failure() { + return failure; + } + + /** + * Returns the stable classification. + * + * @return the failure category + */ + public FailureCategory category() { + return failure.category(); + } + + /** + * Reports whether automatic retry is permitted. + * + * @return true when retryable + */ + public boolean retryable() { + return failure.retryable(); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/HeaderName.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/HeaderName.java new file mode 100644 index 00000000..26f6c2ef --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/HeaderName.java @@ -0,0 +1,36 @@ +package dev.caskeleton.messaging.api.header; + +import java.nio.charset.StandardCharsets; + +/** + * Name of a single message header. + * + *

Header names are compared case-insensitively because brokers disagree on case handling: AMQP + * preserves it, HTTP-style bindings fold it. Storing the name verbatim but normalising for lookup + * keeps a forged {@code MSG.ID} from slipping past the reserved-name check. + * + * @param value the header name as written by the caller + */ +public record HeaderName(String value) { + + private static final int MAX_BYTES = 128; + + public HeaderName { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException("header name must not be blank"); + } + int bytes = value.getBytes(StandardCharsets.UTF_8).length; + if (bytes > MAX_BYTES) { + throw new IllegalArgumentException("header name exceeds 128 bytes: " + bytes); + } + } + + /** + * Returns the lowercase form used for reserved and secret name matching. + * + * @return the normalised name + */ + public String normalized() { + return value.toLowerCase(java.util.Locale.ROOT); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/HeaderValue.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/HeaderValue.java new file mode 100644 index 00000000..12c3199a --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/HeaderValue.java @@ -0,0 +1,23 @@ +package dev.caskeleton.messaging.api.header; + +import java.nio.charset.StandardCharsets; + +/** + * Value of a single message header. + * + * @param value the header value, at most 4096 UTF-8 bytes + */ +public record HeaderValue(String value) { + + private static final int MAX_BYTES = 4096; + + public HeaderValue { + if (value == null) { + throw new IllegalArgumentException("header value must not be null"); + } + int bytes = value.getBytes(StandardCharsets.UTF_8).length; + if (bytes > MAX_BYTES) { + throw new IllegalArgumentException("header value exceeds 4096 bytes: " + bytes); + } + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/MessageHeaders.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/MessageHeaders.java new file mode 100644 index 00000000..328412a3 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/MessageHeaders.java @@ -0,0 +1,164 @@ +package dev.caskeleton.messaging.api.header; + +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * An immutable, bounded header map. + * + *

Two separate factories exist on purpose. {@link #application(Map)} is what business code calls + * and it refuses both reserved and secret names; {@link #platform(Map)} is what adapters call when + * rehydrating an envelope off the wire and it may write reserved names. Secret names are refused in + * both: a credential that reaches a header ends up in broker storage, DLQ dumps, and operator + * tooling, and no downstream redaction can undo that. + */ +public final class MessageHeaders { + + private static final int MAX_COUNT = 64; + private static final int MAX_TOTAL_BYTES = 32_768; + + private static final Set SECRET_NAMES = + Set.of( + "authorization", + "proxy-authorization", + "cookie", + "set-cookie", + "access_token", + "refresh_token", + "api_key", + "password", + "client_secret"); + + private static final MessageHeaders EMPTY = new MessageHeaders(Map.of()); + + private final Map values; + + private MessageHeaders(Map values) { + this.values = values; + } + + /** + * Returns the empty header map. + * + * @return headers with no entries + */ + public static MessageHeaders empty() { + return EMPTY; + } + + /** + * Validates and copies application-supplied headers. + * + * @param input the caller's headers + * @return an immutable copy + * @throws IllegalArgumentException when a reserved or secret name is present, or a limit is + * exceeded + */ + public static MessageHeaders application(Map input) { + return validateAndCopy(input, false); + } + + /** + * Validates and copies platform-supplied headers, allowing reserved names. + * + * @param input the platform's headers + * @return an immutable copy + * @throws IllegalArgumentException when a secret name is present or a limit is exceeded + */ + public static MessageHeaders platform(Map input) { + return validateAndCopy(input, true); + } + + private static MessageHeaders validateAndCopy( + Map input, boolean reservedAllowed) { + if (input == null) { + throw new IllegalArgumentException("message headers must not be null"); + } + if (input.size() > MAX_COUNT) { + throw new IllegalArgumentException("message header count exceeds 64: " + input.size()); + } + + int bytes = 0; + Map copy = new LinkedHashMap<>(); + Set seen = new java.util.HashSet<>(); + for (Map.Entry entry : input.entrySet()) { + HeaderName name = entry.getKey(); + HeaderValue value = entry.getValue(); + if (name == null || value == null) { + throw new IllegalArgumentException("message header name and value must not be null"); + } + String normalized = name.normalized(); + if (!reservedAllowed && ReservedHeaders.isReserved(normalized)) { + throw new IllegalArgumentException("message header is not allowed: " + normalized); + } + if (SECRET_NAMES.contains(normalized)) { + throw new IllegalArgumentException("message header is not allowed: " + normalized); + } + if (!seen.add(normalized)) { + throw new IllegalArgumentException("duplicate message header name: " + normalized); + } + bytes += name.value().getBytes(StandardCharsets.UTF_8).length; + bytes += value.value().getBytes(StandardCharsets.UTF_8).length; + copy.put(name, value); + } + if (bytes > MAX_TOTAL_BYTES) { + throw new IllegalArgumentException("message header bytes exceed 32768: " + bytes); + } + return new MessageHeaders(Collections.unmodifiableMap(copy)); + } + + /** + * Returns the headers as an immutable map. + * + * @return the header entries in insertion order + */ + public Map asMap() { + return values; + } + + /** + * Looks a header up case-insensitively. + * + * @param name the header name in any case + * @return the value when present + */ + public Optional find(String name) { + if (name == null) { + return Optional.empty(); + } + String normalized = name.toLowerCase(Locale.ROOT); + return values.entrySet().stream() + .filter(entry -> entry.getKey().normalized().equals(normalized)) + .map(Map.Entry::getValue) + .findFirst(); + } + + /** + * Returns the number of headers. + * + * @return the entry count + */ + public int size() { + return values.size(); + } + + @Override + public boolean equals(Object other) { + return other instanceof MessageHeaders headers && values.equals(headers.values); + } + + @Override + public int hashCode() { + return values.hashCode(); + } + + @Override + public String toString() { + return "MessageHeaders[size=" + values.size() + "]"; + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/ReservedHeaders.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/ReservedHeaders.java new file mode 100644 index 00000000..d06ee8c0 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/ReservedHeaders.java @@ -0,0 +1,126 @@ +package dev.caskeleton.messaging.api.header; + +import java.util.Locale; +import java.util.Set; + +/** + * The header names the platform owns. + * + *

These carry envelope identity, trace propagation, and redrive bookkeeping. Applications cannot + * write them: if a caller could set {@code msg.id}, the logical identity that Inbox deduplication + * and DLQ correlation depend on would become caller-controlled. + */ +public final class ReservedHeaders { + + /** Logical message identity. */ + public static final String MESSAGE_ID = "msg.id"; + + /** Stable catalog message type. */ + public static final String MESSAGE_TYPE = "msg.type"; + + /** Schema revision of the payload. */ + public static final String SCHEMA_VERSION = "msg.schema-version"; + + /** Logical producing service. */ + public static final String PRODUCER = "msg.producer"; + + /** Instant the platform created the envelope. */ + public static final String PRODUCED_AT = "msg.produced-at"; + + /** Instant the business fact occurred. */ + public static final String OCCURRED_AT = "msg.occurred-at"; + + /** Workflow correlation value. */ + public static final String CORRELATION_ID = "msg.correlation-id"; + + /** Identity of the directly causing message. */ + public static final String CAUSATION_ID = "msg.causation-id"; + + /** Codec content type. */ + public static final String CONTENT_TYPE = "msg.content-type"; + + /** Distribution key. */ + public static final String PARTITION_KEY = "msg.partition-key"; + + /** Ordering key. */ + public static final String ORDERING_KEY = "msg.ordering-key"; + + /** Identity of the redrive operation that replayed this message. */ + public static final String REDRIVE_ID = "msg.redrive-id"; + + /** How many times this message has been redriven. */ + public static final String REDRIVE_COUNT = "msg.redrive-count"; + + /** Current retry attempt, counting the first delivery as one. */ + public static final String RETRY_ATTEMPT = "msg.retry-attempt"; + + /** Instant of the first recorded failure. */ + public static final String FIRST_FAILURE_AT = "msg.first-failure-at"; + + /** Instant of the most recent recorded failure. */ + public static final String LAST_FAILURE_AT = "msg.last-failure-at"; + + /** Stable failure category recorded when parking or dead lettering. */ + public static final String FAILURE_CATEGORY = "msg.failure-category"; + + /** Stable failure code recorded when parking or dead lettering. */ + public static final String FAILURE_CODE = "msg.failure-code"; + + /** Logical destination the message was originally published to. */ + public static final String ORIGIN_DESTINATION = "msg.origin-destination"; + + /** W3C trace parent. */ + public static final String TRACEPARENT = "traceparent"; + + /** W3C trace state. */ + public static final String TRACESTATE = "tracestate"; + + /** W3C baggage. */ + public static final String BAGGAGE = "baggage"; + + private static final Set NAMES = + Set.of( + MESSAGE_ID, + MESSAGE_TYPE, + SCHEMA_VERSION, + PRODUCER, + PRODUCED_AT, + OCCURRED_AT, + CORRELATION_ID, + CAUSATION_ID, + CONTENT_TYPE, + PARTITION_KEY, + ORDERING_KEY, + REDRIVE_ID, + REDRIVE_COUNT, + RETRY_ATTEMPT, + FIRST_FAILURE_AT, + LAST_FAILURE_AT, + FAILURE_CATEGORY, + FAILURE_CODE, + ORIGIN_DESTINATION, + TRACEPARENT, + TRACESTATE, + BAGGAGE); + + private ReservedHeaders() {} + + /** + * Reports whether a header name belongs to the platform. + * + * @param name the header name in any case + * @return true when the platform owns the name + */ + public static boolean isReserved(String name) { + return name != null && NAMES.contains(name.toLowerCase(Locale.ROOT)); + } + + /** + * Returns every reserved header name. + * + * @return an immutable set of lowercase names + */ + public static Set names() { + return NAMES; + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchMessagePublisher.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchMessagePublisher.java new file mode 100644 index 00000000..6a568245 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchMessagePublisher.java @@ -0,0 +1,25 @@ +package dev.caskeleton.messaging.api.publish; + +import java.util.List; +import java.util.concurrent.CompletionStage; + +/** + * The M2 batch publish entry point. + * + *

A batch is a throughput optimisation, not a transaction. Entries succeed, fail, and go + * ambiguous independently, and {@link BatchPublishResult} keeps every entry's outcome against its + * submission index so the caller can act per entry. Nothing here rolls back: an adapter that + * reported a confirmed entry cannot un-publish it because a later entry was rejected. + */ +public interface BatchMessagePublisher { + + /** + * Publishes a batch of independent requests. + * + * @param requests the entries to publish, in submission order + * @param options the batch-wide options + * @return a stage completing with one outcome per submitted entry + */ + CompletionStage publish( + List> requests, BatchPublishOptions options); +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchPublishItemResult.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchPublishItemResult.java new file mode 100644 index 00000000..a275061b --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchPublishItemResult.java @@ -0,0 +1,27 @@ +package dev.caskeleton.messaging.api.publish; + +import dev.caskeleton.messaging.api.MessageId; +import java.util.Objects; + +/** + * One item's outcome inside a batch publish. + * + *

Kept per index rather than collapsed into a batch-level success flag. A batch is not a + * transaction: some items confirm, some are rejected, and some are ambiguous, and the caller has to + * be able to act differently on each. A single boolean would force it to re-submit everything, + * which duplicates the items that already landed. + * + * @param index the item's position in the submitted list + * @param messageId the item's logical identity + * @param result the item's outcome and evidence + */ +public record BatchPublishItemResult(int index, MessageId messageId, PublishResult result) { + + public BatchPublishItemResult { + Objects.requireNonNull(messageId, "messageId must not be null"); + Objects.requireNonNull(result, "result must not be null"); + if (index < 0) { + throw new IllegalArgumentException("index must not be negative"); + } + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchPublishOptions.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchPublishOptions.java new file mode 100644 index 00000000..83614380 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchPublishOptions.java @@ -0,0 +1,42 @@ +package dev.caskeleton.messaging.api.publish; + +import java.time.Duration; +import java.util.Objects; + +/** + * Options covering a whole batch publish. + * + *

There is no retry setting. A batch is not a transaction, and retrying the batch would resubmit + * entries that already confirmed, so the platform never retries one on the caller's behalf; the + * caller resubmits the individual entries it knows are duplicate-safe under their original {@code + * messageId}. + * + * @param timeout the deadline for the whole batch + * @param maxBatchSize the largest number of entries accepted in one call + * @param stopOnFirstRejection whether to skip remaining entries once one is rejected + */ +public record BatchPublishOptions( + Duration timeout, int maxBatchSize, boolean stopOnFirstRejection) { + + private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30); + private static final int DEFAULT_MAX_BATCH_SIZE = 500; + + public BatchPublishOptions { + Objects.requireNonNull(timeout, "timeout must not be null"); + if (timeout.isNegative() || timeout.isZero()) { + throw new IllegalArgumentException("batch timeout must be positive"); + } + if (maxBatchSize < 1) { + throw new IllegalArgumentException("maxBatchSize must be at least 1"); + } + } + + /** + * Returns the defaults: thirty seconds, 500 entries, best-effort completion of every entry. + * + * @return the default options + */ + public static BatchPublishOptions defaults() { + return new BatchPublishOptions(DEFAULT_TIMEOUT, DEFAULT_MAX_BATCH_SIZE, false); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchPublishResult.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchPublishResult.java new file mode 100644 index 00000000..707bee0b --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BatchPublishResult.java @@ -0,0 +1,44 @@ +package dev.caskeleton.messaging.api.publish; + +import java.time.Duration; +import java.util.List; +import java.util.Objects; + +/** + * The outcome of one batch publish. + * + *

There is no automatic batch retry. Re-submitting a whole batch because part of it failed + * duplicates the part that succeeded; the caller re-submits only the items it decides are + * duplicate-safe, under their original message ids. + * + * @param items each item's outcome, in submission order + * @param elapsed how long the batch took + */ +public record BatchPublishResult(List items, Duration elapsed) { + + public BatchPublishResult { + Objects.requireNonNull(items, "items must not be null"); + Objects.requireNonNull(elapsed, "elapsed must not be null"); + items = List.copyOf(items); + } + + /** + * Returns the items with a given completion. + * + * @param completion the completion to select + * @return the matching items + */ + public List withCompletion(PublishCompletion completion) { + return items.stream().filter(item -> item.result().completion() == completion).toList(); + } + + /** + * Reports whether every item confirmed. + * + * @return true when nothing was rejected or ambiguous + */ + public boolean allConfirmed() { + return items.stream() + .allMatch(item -> item.result().completion() == PublishCompletion.CONFIRMED); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BlockingMessagePublisher.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BlockingMessagePublisher.java new file mode 100644 index 00000000..5b81bf88 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BlockingMessagePublisher.java @@ -0,0 +1,30 @@ +package dev.caskeleton.messaging.api.publish; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.destination.MessageDestination; + +/** + * The blocking facade over the asynchronous publisher. + * + *

Offered because most business code is written against a blocking data access layer, and + * forcing a {@code CompletionStage} through it produces worse code than a blocking call does. It is + * a facade rather than a second implementation: the core stays asynchronous, and this only converts + * the lifecycle. + * + *

It respects the publish deadline rather than waiting indefinitely, so a stalled broker + * surfaces as a timeout with ambiguous evidence instead of an exhausted thread pool. + */ +public interface BlockingMessagePublisher { + + /** + * Publishes one message and waits for its outcome. + * + * @param the payload type + * @param destination the logical destination + * @param message the envelope to publish + * @param options per-call options, including the deadline + * @return the outcome and its evidence + */ + PublishResult publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options); +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BrokerPosition.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BrokerPosition.java new file mode 100644 index 00000000..7119c399 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/BrokerPosition.java @@ -0,0 +1,27 @@ +package dev.caskeleton.messaging.api.publish; + +import java.util.Map; + +/** + * A broker-specific coordinate for a published or delivered message. + * + *

Positions exist for diagnostics and replay. Business logic must not branch on them: the whole + * point of the logical destination model is that a handler behaves identically whether it is + * reading a Kafka offset or a RabbitMQ delivery tag. + */ +public interface BrokerPosition { + + /** + * Returns the broker family that produced this position. + * + * @return a stable, low-cardinality broker name + */ + String broker(); + + /** + * Returns the position as diagnostic attributes. + * + * @return an immutable map safe for structured logs + */ + Map diagnosticAttributes(); +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/ConfirmationLevel.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/ConfirmationLevel.java new file mode 100644 index 00000000..6b576584 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/ConfirmationLevel.java @@ -0,0 +1,20 @@ +package dev.caskeleton.messaging.api.publish; + +/** + * The strength of confirmation the broker actually supplied. + * + *

Kept separate from the requested confirmation requirement so that "we asked for replication + * and got a leader ack" is representable, and therefore rejectable, instead of being rounded up to + * success. + */ +public enum ConfirmationLevel { + + /** No confirmation was received. */ + NONE, + + /** The broker acknowledged receipt without proving durability. */ + BROKER_ACK, + + /** The broker acknowledged replication or persistence. */ + REPLICATION_OR_PERSISTENCE_ACK +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/DelayedMessagePublisher.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/DelayedMessagePublisher.java new file mode 100644 index 00000000..9b9749a1 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/DelayedMessagePublisher.java @@ -0,0 +1,28 @@ +package dev.caskeleton.messaging.api.publish; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import java.time.Instant; +import java.util.concurrent.CompletionStage; + +/** + * The M2 scheduled-delivery entry point. + * + *

Only destinations whose runtime declares {@code delayedDelivery} accept these calls. A plain + * Kafka topic does not: emulating a delay by holding the message in the producer would make the + * publisher's own liveness the schedule, so the platform rejects the call instead of pretending. + */ +public interface DelayedMessagePublisher { + + /** + * Publishes a message the broker must not deliver before the given instant. + * + * @param the payload type + * @param destination the logical destination + * @param message the envelope to publish + * @param deliverAt the earliest delivery time + * @return a stage completing with the publish outcome and its evidence + */ + CompletionStage publish( + MessageDestination destination, MessageEnvelope message, Instant deliverAt); +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/MessagePublisher.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/MessagePublisher.java new file mode 100644 index 00000000..5ffca2a1 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/MessagePublisher.java @@ -0,0 +1,26 @@ +package dev.caskeleton.messaging.api.publish; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import java.util.concurrent.CompletionStage; + +/** + * The M1 typed publish entry point used by ordinary business code. + * + *

The core contract is {@link CompletionStage}. Blocking and Reactor facades live in separate + * modules so that {@code messaging-core-api} keeps no dependency on either programming model. + */ +public interface MessagePublisher { + + /** + * Publishes one message. + * + * @param the payload type + * @param destination the logical destination + * @param message the envelope to publish + * @param options per-call options + * @return a stage completing with the outcome and its evidence + */ + CompletionStage publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options); +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishCompletion.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishCompletion.java new file mode 100644 index 00000000..f9f7847a --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishCompletion.java @@ -0,0 +1,20 @@ +package dev.caskeleton.messaging.api.publish; + +/** + * How a publish attempt ended. + * + *

Publish is not a boolean. Collapsing "the broker refused this" and "we never learned what the + * broker did" into one failure is what produces duplicate orders: the first is safe to abandon, the + * second is not safe to retry with a new identity. + */ +public enum PublishCompletion { + + /** The broker accepted the message at the required confirmation level. */ + CONFIRMED, + + /** The message was definitively not accepted. It is safe to abandon this attempt. */ + REJECTED, + + /** The outcome is unknown. The broker may or may not hold the message. */ + AMBIGUOUS +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishDeduplication.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishDeduplication.java new file mode 100644 index 00000000..2421550d --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishDeduplication.java @@ -0,0 +1,27 @@ +package dev.caskeleton.messaging.api.publish; + +import java.time.Duration; +import java.util.Objects; + +/** + * Requests broker-side duplicate suppression for a publish. + * + *

The deduplication key defaults to the logical message id, which is what makes an Outbox relay + * able to retry an ambiguous publish without creating a second message. Requesting this on a broker + * without the {@code deduplicatedPublish} capability is a startup failure, not a silent no-op. + * + * @param key the stable deduplication key + * @param window how long the broker should remember the key + */ +public record PublishDeduplication(String key, Duration window) { + + public PublishDeduplication { + Objects.requireNonNull(window, "deduplication window must not be null"); + if (key == null || key.isBlank()) { + throw new IllegalArgumentException("deduplication key must not be blank"); + } + if (window.isNegative() || window.isZero()) { + throw new IllegalArgumentException("deduplication window must be positive"); + } + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishEvidence.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishEvidence.java new file mode 100644 index 00000000..9ad9611a --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishEvidence.java @@ -0,0 +1,66 @@ +package dev.caskeleton.messaging.api.publish; + +import java.util.Objects; + +/** + * The observable facts behind a publish outcome. + * + *

Evidence is recorded before a completion is chosen, not derived from it. That ordering is what + * lets an operator answer "could the broker be holding this message?" from a stored result. + * + * @param queuedLocally the adapter buffered the message in process + * @param transmission what is known about bytes reaching the broker + * @param brokerAccepted the broker positively acknowledged the message + * @param confirmationLevel the strength of that acknowledgement + */ +public record PublishEvidence( + boolean queuedLocally, + TransmissionEvidence transmission, + boolean brokerAccepted, + ConfirmationLevel confirmationLevel) { + + public PublishEvidence { + Objects.requireNonNull(transmission, "transmission must not be null"); + Objects.requireNonNull(confirmationLevel, "confirmationLevel must not be null"); + if (!brokerAccepted && confirmationLevel != ConfirmationLevel.NONE) { + throw new IllegalArgumentException( + "confirmation level requires broker acceptance: " + confirmationLevel); + } + if (transmission == TransmissionEvidence.NOT_TRANSMITTED && brokerAccepted) { + throw new IllegalArgumentException("untransmitted message cannot be broker accepted"); + } + } + + /** + * Returns evidence for a message rejected before anything was sent. + * + * @return local rejection evidence + */ + public static PublishEvidence notTransmitted() { + return new PublishEvidence( + false, TransmissionEvidence.NOT_TRANSMITTED, false, ConfirmationLevel.NONE); + } + + /** + * Returns evidence for a message whose fate is unknown. + * + * @return ambiguous evidence + */ + public static PublishEvidence ambiguous() { + return new PublishEvidence( + true, TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED, false, ConfirmationLevel.NONE); + } + + /** + * Returns evidence for a confirmed publish. + * + * @param level the confirmation the broker supplied + * @return confirmed evidence + */ + public static PublishEvidence confirmed(ConfirmationLevel level) { + if (level == ConfirmationLevel.NONE) { + throw new IllegalArgumentException("confirmed evidence requires a confirmation level"); + } + return new PublishEvidence(true, TransmissionEvidence.TRANSMITTED, true, level); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishOptions.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishOptions.java new file mode 100644 index 00000000..f89c75a9 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishOptions.java @@ -0,0 +1,61 @@ +package dev.caskeleton.messaging.api.publish; + +import dev.caskeleton.messaging.api.destination.ConfirmationRequirement; +import java.time.Duration; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Per-call publish options. + * + *

{@code brokerHints} must be empty for M1 callers. Only an M3 native capability may populate + * it, because a hint map on the general API becomes an escape hatch that quietly bypasses + * destination policy. + * + * @param timeout the publish operation deadline + * @param confirmation the confirmation this call requires + * @param deduplication optional broker-side duplicate suppression + * @param brokerHints M3-only adapter hints + */ +public record PublishOptions( + Duration timeout, + ConfirmationRequirement confirmation, + Optional deduplication, + Map brokerHints) { + + private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(5); + + public PublishOptions { + Objects.requireNonNull(timeout, "timeout must not be null"); + Objects.requireNonNull(confirmation, "confirmation must not be null"); + Objects.requireNonNull(deduplication, "deduplication must not be null"); + Objects.requireNonNull(brokerHints, "brokerHints must not be null"); + if (timeout.isNegative() || timeout.isZero()) { + throw new IllegalArgumentException("publish timeout must be positive"); + } + brokerHints = Map.copyOf(brokerHints); + } + + /** + * Returns the M1 default: five second timeout, replication evidence, no hints. + * + * @return the default options + */ + public static PublishOptions defaults() { + return new PublishOptions( + DEFAULT_TIMEOUT, + ConfirmationRequirement.REPLICATION_OR_PERSISTENCE_ACK, + Optional.empty(), + Map.of()); + } + + /** + * Reports whether these options stay inside the M1 surface. + * + * @return true when no broker hints are present + */ + public boolean isM1Compatible() { + return brokerHints.isEmpty(); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishRequest.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishRequest.java new file mode 100644 index 00000000..50730b67 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishRequest.java @@ -0,0 +1,40 @@ +package dev.caskeleton.messaging.api.publish; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import java.util.Objects; + +/** + * One entry of a batch publish, pairing a destination with the envelope bound for it. + * + *

A batch is a list of independent publishes, not one publish of many payloads, so each entry + * carries its own destination. Nothing forces the entries of a batch to share a destination; an + * adapter groups them itself when its wire protocol rewards grouping. + * + * @param the payload type + * @param destination the logical destination for this entry + * @param message the envelope to publish + * @param options the per-entry options + */ +public record PublishRequest( + MessageDestination destination, MessageEnvelope message, PublishOptions options) { + + public PublishRequest { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(message, "message must not be null"); + Objects.requireNonNull(options, "options must not be null"); + } + + /** + * Creates a request using the default publish options. + * + * @param the payload type + * @param destination the logical destination + * @param message the envelope to publish + * @return the request + */ + public static PublishRequest of( + MessageDestination destination, MessageEnvelope message) { + return new PublishRequest<>(destination, message, PublishOptions.defaults()); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishResult.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishResult.java new file mode 100644 index 00000000..38c714f4 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishResult.java @@ -0,0 +1,88 @@ +package dev.caskeleton.messaging.api.publish; + +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; + +/** + * The outcome of one publish call, with the evidence that justifies it. + * + *

The constructor refuses combinations that would let a caller read a stronger guarantee than + * was actually obtained. An adapter that cannot decide must report {@link + * PublishCompletion#AMBIGUOUS} with no confirmation level, which is the conservative answer, rather + * than picking a completion that happens to make the call site simpler. + * + * @param completion how the attempt ended + * @param evidence the observable facts behind that outcome + * @param routingOutcome whether the broker routed the message + * @param position the broker coordinate when known + * @param attempts transport attempts made, counting the first as one + * @param elapsed wall-clock time spent + * @param failure the sanitized failure when the attempt did not confirm + */ +public record PublishResult( + PublishCompletion completion, + PublishEvidence evidence, + RoutingOutcome routingOutcome, + Optional position, + int attempts, + Duration elapsed, + Optional failure) { + + public PublishResult { + Objects.requireNonNull(completion, "completion must not be null"); + Objects.requireNonNull(evidence, "evidence must not be null"); + Objects.requireNonNull(routingOutcome, "routingOutcome must not be null"); + Objects.requireNonNull(position, "position must not be null"); + Objects.requireNonNull(elapsed, "elapsed must not be null"); + Objects.requireNonNull(failure, "failure must not be null"); + + if (attempts < 1) { + throw new IllegalArgumentException("publish attempts must be at least 1"); + } + if (elapsed.isNegative()) { + throw new IllegalArgumentException("publish elapsed must not be negative"); + } + + if (completion == PublishCompletion.CONFIRMED && !evidence.brokerAccepted()) { + throw new IllegalArgumentException("confirmed publish requires broker acceptance"); + } + if (completion == PublishCompletion.CONFIRMED + && evidence.confirmationLevel() == ConfirmationLevel.NONE) { + throw new IllegalArgumentException("confirmed publish requires a confirmation level"); + } + if (completion == PublishCompletion.CONFIRMED && routingOutcome == RoutingOutcome.UNROUTABLE) { + throw new IllegalArgumentException("unroutable publish cannot be confirmed"); + } + if (completion == PublishCompletion.AMBIGUOUS + && evidence.confirmationLevel() != ConfirmationLevel.NONE) { + throw new IllegalArgumentException("ambiguous publish cannot claim confirmation"); + } + if (completion == PublishCompletion.AMBIGUOUS && evidence.brokerAccepted()) { + throw new IllegalArgumentException("ambiguous publish cannot claim broker acceptance"); + } + if (completion == PublishCompletion.AMBIGUOUS + && evidence.transmission() == TransmissionEvidence.NOT_TRANSMITTED) { + throw new IllegalArgumentException("untransmitted publish is rejected, not ambiguous"); + } + if (completion != PublishCompletion.CONFIRMED && failure.isEmpty()) { + throw new IllegalArgumentException("non-confirmed publish requires a failure descriptor"); + } + if (completion == PublishCompletion.CONFIRMED && failure.isPresent()) { + throw new IllegalArgumentException("confirmed publish must not carry a failure descriptor"); + } + } + + /** + * Reports whether the broker may be holding this message. + * + *

Callers use this to decide between abandoning an attempt and re-publishing under the same + * logical id. + * + * @return true when the outcome is ambiguous + */ + public boolean mayHaveBeenStored() { + return completion == PublishCompletion.AMBIGUOUS; + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/RoutingOutcome.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/RoutingOutcome.java new file mode 100644 index 00000000..601aef65 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/RoutingOutcome.java @@ -0,0 +1,23 @@ +package dev.caskeleton.messaging.api.publish; + +/** + * Whether the broker could route the message to a destination. + * + *

Routing is separate from confirmation because RabbitMQ can confirm a publish the exchange + * accepted and simultaneously return it as unroutable. Reporting only the confirm would turn a + * dropped message into a success. + */ +public enum RoutingOutcome { + + /** The broker has no routing stage, as with a Kafka topic publish. */ + NOT_APPLICABLE, + + /** The message reached at least one destination. */ + ROUTED, + + /** The broker accepted the publish but found no destination. */ + UNROUTABLE, + + /** Routing could not be determined. */ + UNKNOWN +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/TransmissionEvidence.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/TransmissionEvidence.java new file mode 100644 index 00000000..1bb8434f --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/TransmissionEvidence.java @@ -0,0 +1,14 @@ +package dev.caskeleton.messaging.api.publish; + +/** What the adapter can prove about bytes leaving the process. */ +public enum TransmissionEvidence { + + /** Nothing was written to the broker connection. */ + NOT_TRANSMITTED, + + /** Bytes may have been written; the connection failed before that could be established. */ + MAY_HAVE_BEEN_TRANSMITTED, + + /** Bytes were written to the broker connection. */ + TRANSMITTED +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/ManualMessageHandler.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/ManualMessageHandler.java new file mode 100644 index 00000000..69f1620c --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/ManualMessageHandler.java @@ -0,0 +1,25 @@ +package dev.caskeleton.messaging.api.settlement; + +import dev.caskeleton.messaging.api.delivery.MessageDelivery; +import java.util.concurrent.CompletionStage; + +/** + * The M2 handler that settles its own deliveries. + * + *

Available only to modules granted the advanced capability. A handler that returns without + * calling a terminal settlement method is a defect the runtime detects and reports rather than + * silently leaving the message in flight. + * + * @param the payload type + */ +public interface ManualMessageHandler { + + /** + * Handles one delivery, settling it through the supplied controller. + * + * @param delivery the decoded message and its metadata + * @param settlement the settlement handle for this delivery + * @return a stage completing when handling has finished + */ + CompletionStage handle(MessageDelivery delivery, SettlementController settlement); +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementCompletion.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementCompletion.java new file mode 100644 index 00000000..2773188e --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementCompletion.java @@ -0,0 +1,14 @@ +package dev.caskeleton.messaging.api.settlement; + +/** How a settlement attempt ended. */ +public enum SettlementCompletion { + + /** The broker confirmed the settlement. */ + SETTLED, + + /** The broker refused the settlement. */ + REJECTED, + + /** The settlement outcome is unknown and redelivery remains possible. */ + UNKNOWN +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementController.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementController.java new file mode 100644 index 00000000..e17e6b60 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementController.java @@ -0,0 +1,50 @@ +package dev.caskeleton.messaging.api.settlement; + +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import java.time.Duration; +import java.util.concurrent.CompletionStage; + +/** + * The M2 manual settlement handle. + * + *

No method here accepts a broker channel, delivery tag, or offset. Manual settlement changes + * when the platform settles, never who settles, so the same DLQ-before-ack + * ordering and payload limits still apply. + * + *

Exactly one terminal call is permitted per delivery; a second call fails rather than racing + * the first. + */ +public interface SettlementController { + + /** + * Acknowledges the message. + * + * @return a stage completing with the settlement outcome + */ + CompletionStage ack(); + + /** + * Schedules the message for another attempt. + * + * @param delay how long to wait before redelivery + * @return a stage completing with the settlement outcome + */ + CompletionStage retry(Duration delay); + + /** + * Routes the message to the dead letter destination, settling the source only after that publish + * is confirmed. + * + * @param failure the sanitized failure description + * @return a stage completing with the settlement outcome + */ + CompletionStage deadLetter(FailureDescriptor failure); + + /** + * Discards the message without dead lettering. + * + * @param failure the sanitized failure description + * @return a stage completing with the settlement outcome + */ + CompletionStage reject(FailureDescriptor failure); +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementEvidence.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementEvidence.java new file mode 100644 index 00000000..f6400cc2 --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementEvidence.java @@ -0,0 +1,66 @@ +package dev.caskeleton.messaging.api.settlement; + +import java.util.Objects; + +/** + * What is known about a settlement reaching the broker. + * + * @param transmitted the settlement was written to the broker connection + * @param brokerConfirmed the broker confirmed the settlement + * @param redeliveryPossible the message may still be redelivered + */ +public record SettlementEvidence( + boolean transmitted, boolean brokerConfirmed, boolean redeliveryPossible) { + + public SettlementEvidence { + if (brokerConfirmed && !transmitted) { + throw new IllegalArgumentException("an unsent settlement cannot be broker confirmed"); + } + } + + /** + * Returns evidence for a confirmed settlement. + * + * @return confirmed evidence + */ + public static SettlementEvidence confirmed() { + return new SettlementEvidence(true, true, false); + } + + /** + * Returns evidence for a settlement whose fate is unknown. + * + * @return unknown evidence + */ + public static SettlementEvidence unknown() { + return new SettlementEvidence(true, false, true); + } + + /** + * Returns evidence for a settlement that never left the process. + * + * @return unsent evidence + */ + public static SettlementEvidence notTransmitted() { + return new SettlementEvidence(false, false, true); + } + + /** + * Returns a settlement evidence value, validating the combination. + * + * @param object the value to compare + * @return whether the values are equal + */ + @Override + public boolean equals(Object object) { + return object instanceof SettlementEvidence other + && transmitted == other.transmitted + && brokerConfirmed == other.brokerConfirmed + && redeliveryPossible == other.redeliveryPossible; + } + + @Override + public int hashCode() { + return Objects.hash(transmitted, brokerConfirmed, redeliveryPossible); + } +} diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementResult.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementResult.java new file mode 100644 index 00000000..9712901d --- /dev/null +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/settlement/SettlementResult.java @@ -0,0 +1,47 @@ +package dev.caskeleton.messaging.api.settlement; + +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import java.util.Objects; +import java.util.Optional; + +/** + * The outcome of one settlement call. + * + *

{@link SettlementCompletion#UNKNOWN} means redelivery remains possible and must not be read as + * success. Treating an unconfirmed acknowledgement as settled is the classic route to a message + * that looks processed in logs and is processed again minutes later. + * + * @param completion how the settlement ended + * @param evidence what is known about it reaching the broker + * @param failure the sanitized failure when settlement did not confirm + */ +public record SettlementResult( + SettlementCompletion completion, + SettlementEvidence evidence, + Optional failure) { + + public SettlementResult { + Objects.requireNonNull(completion, "completion must not be null"); + Objects.requireNonNull(evidence, "evidence must not be null"); + Objects.requireNonNull(failure, "failure must not be null"); + if (completion == SettlementCompletion.SETTLED && !evidence.brokerConfirmed()) { + throw new IllegalArgumentException("settled result requires broker confirmation"); + } + if (completion == SettlementCompletion.SETTLED && evidence.redeliveryPossible()) { + throw new IllegalArgumentException("settled result cannot leave redelivery possible"); + } + if (completion != SettlementCompletion.SETTLED && failure.isEmpty()) { + throw new IllegalArgumentException("non-settled result requires a failure descriptor"); + } + } + + /** + * Returns a confirmed settlement result. + * + * @return a settled result + */ + public static SettlementResult settled() { + return new SettlementResult( + SettlementCompletion.SETTLED, SettlementEvidence.confirmed(), Optional.empty()); + } +} diff --git a/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/CoreValueTypesTest.java b/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/CoreValueTypesTest.java new file mode 100644 index 00000000..38d374cb --- /dev/null +++ b/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/CoreValueTypesTest.java @@ -0,0 +1,60 @@ +package dev.caskeleton.messaging.api; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import java.util.Arrays; +import java.util.UUID; +import org.junit.jupiter.api.Test; + +class CoreValueTypesTest { + + @Test + void messageTypeRejectsBlankValue() { + assertThatThrownBy(() -> new MessageType(" ")).isInstanceOf(IllegalArgumentException.class); + } + + @Test + void schemaVersionMustBePositive() { + assertThatThrownBy(() -> new SchemaVersion(0)).isInstanceOf(IllegalArgumentException.class); + } + + @Test + void guaranteeEnumsDoNotAdvertiseUnsupportedSemantics() { + assertThat(Arrays.stream(DeliveryGuarantee.values()).map(Enum::name)) + .doesNotContain("EXACTLY_ONCE"); + assertThat(Arrays.stream(OrderingScope.values()).map(Enum::name)).doesNotContain("GLOBAL"); + } + + @Test + void newMessageIdIsVersionSevenAndTimeOrdered() { + MessageId first = MessageId.newId(); + MessageId second = MessageId.newId(); + + assertThat(first.value().version()).isEqualTo(7); + assertThat(first.value().variant()).isEqualTo(2); + assertThat(first.value().compareTo(second.value())).isNegative(); + } + + @Test + void messageIdRejectsNullValue() { + assertThatThrownBy(() -> new MessageId(null)).isInstanceOf(NullPointerException.class); + } + + @Test + void causationIdWrapsAPredecessorMessageId() { + MessageId predecessor = new MessageId(UUID.fromString("0190f4aa-0000-7000-8000-000000000001")); + + assertThat(new CausationId(predecessor).value()).isEqualTo(predecessor); + } + + @Test + void producerAndCorrelationIdentifiersStayBounded() { + assertThatThrownBy(() -> new ProducerId("p".repeat(121))) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new CorrelationId("c".repeat(161))) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/MessageEnvelopeTest.java b/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/MessageEnvelopeTest.java new file mode 100644 index 00000000..c33ce515 --- /dev/null +++ b/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/MessageEnvelopeTest.java @@ -0,0 +1,141 @@ +package dev.caskeleton.messaging.api; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.header.HeaderName; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class MessageEnvelopeTest { + + @Test + void applicationCannotSetReservedHeader() { + Map forged = + Map.of(new HeaderName("msg.id"), new HeaderValue("forged")); + + assertThatThrownBy(() -> MessageHeaders.application(forged)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void reservedHeaderRejectionIsCaseInsensitive() { + Map forged = + Map.of(new HeaderName("MSG.Id"), new HeaderValue("forged")); + + assertThatThrownBy(() -> MessageHeaders.application(forged)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void platformMayWriteReservedHeaders() { + MessageHeaders headers = + MessageHeaders.platform(Map.of(new HeaderName("msg.id"), new HeaderValue("0190f4aa"))); + + assertThat(headers.find("MSG.ID")).map(HeaderValue::value).hasValue("0190f4aa"); + } + + @Test + void secretHeadersAreRejected() { + Map secret = + Map.of(new HeaderName("Authorization"), new HeaderValue("Bearer secret")); + + assertThatThrownBy(() -> MessageHeaders.application(secret)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void platformCannotWriteSecretHeadersEither() { + Map secret = + Map.of(new HeaderName("client_secret"), new HeaderValue("s3cr3t")); + + assertThatThrownBy(() -> MessageHeaders.platform(secret)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void headerCountAboveSixtyFourIsRejected() { + Map tooMany = new LinkedHashMap<>(); + for (int i = 0; i < 65; i++) { + tooMany.put(new HeaderName("h" + i), new HeaderValue("v")); + } + + assertThatThrownBy(() -> MessageHeaders.application(tooMany)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("64"); + } + + @Test + void headerBytesAboveThirtyTwoKibibytesAreRejected() { + Map heavy = new LinkedHashMap<>(); + for (int i = 0; i < 16; i++) { + heavy.put(new HeaderName("h" + i), new HeaderValue("v".repeat(4096))); + } + + assertThatThrownBy(() -> MessageHeaders.application(heavy)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("32768"); + } + + @Test + void headerNameAboveOneHundredTwentyEightBytesIsRejected() { + assertThatThrownBy(() -> new HeaderName("n".repeat(129))) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void headerValueAboveFourKibibytesIsRejected() { + assertThatThrownBy(() -> new HeaderValue("v".repeat(4097))) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void payloadCannotBeNull() { + assertThatThrownBy(() -> TestEnvelopeFactory.envelope(null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void withPayloadPreservesLogicalIdentity() { + MessageEnvelope original = TestEnvelopeFactory.envelope("original"); + + MessageEnvelope encoded = original.withPayload(new byte[] {1, 2, 3}); + + assertThat(encoded.messageId()).isEqualTo(original.messageId()); + assertThat(encoded.messageType()).isEqualTo(original.messageType()); + assertThat(encoded.schemaVersion()).isEqualTo(original.schemaVersion()); + } +} + +/** + * Builds envelopes for tests. Package-private on purpose: production code must go through the + * publisher, which is what applies destination policy. + */ +final class TestEnvelopeFactory { + + private TestEnvelopeFactory() {} + + static MessageEnvelope envelope(T payload) { + return new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + Instant.parse("2026-08-10T00:00:00Z"), + Optional.of(Instant.parse("2026-08-10T00:00:00Z")), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + payload); + } +} diff --git a/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/ModuleSmokeTest.java b/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/ModuleSmokeTest.java new file mode 100644 index 00000000..782b6253 --- /dev/null +++ b/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/ModuleSmokeTest.java @@ -0,0 +1,13 @@ +package dev.caskeleton.messaging.api; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class ModuleSmokeTest { + + @Test + void coreApiModuleLoads() { + assertThat(ModuleSmokeTest.class.getPackageName()).isEqualTo("dev.caskeleton.messaging.api"); + } +} diff --git a/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/delivery/ConsumerContractTest.java b/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/delivery/ConsumerContractTest.java new file mode 100644 index 00000000..3de62ce8 --- /dev/null +++ b/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/delivery/ConsumerContractTest.java @@ -0,0 +1,129 @@ +package dev.caskeleton.messaging.api.delivery; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.settlement.SettlementCompletion; +import dev.caskeleton.messaging.api.settlement.SettlementEvidence; +import dev.caskeleton.messaging.api.settlement.SettlementResult; +import java.time.Instant; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class ConsumerContractTest { + + @Test + void retryCarriesStableFailureCategory() { + FailureDescriptor failure = + new FailureDescriptor( + FailureCategory.PROCESSING_TRANSIENT, + "DOWNSTREAM_TIMEOUT", + true, + "downstream timed out", + Optional.of("TimeoutException")); + + HandleResult result = new HandleResult.Retry(failure); + + assertThat(((HandleResult.Retry) result).failure().retryable()).isTrue(); + } + + @Test + void deliveryMetadataCountsInitialDeliveryAsAttemptOne() { + DeliveryMetadata metadata = DeliveryMetadataFixture.initial(); + + assertThat(metadata.deliveryAttempt()).isEqualTo(1); + assertThat(metadata.redelivered()).isFalse(); + } + + @Test + void firstDeliveryCannotBeFlaggedAsRedelivered() { + assertThatThrownBy( + () -> + new DeliveryMetadata( + new DestinationName("order-events"), + 1, + true, + Optional.empty(), + Optional.empty(), + Optional.empty(), + Instant.parse("2026-08-10T00:00:00Z"))) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void deliveryAttemptIsNeverZero() { + assertThatThrownBy( + () -> + new DeliveryMetadata( + new DestinationName("order-events"), + 0, + false, + Optional.empty(), + Optional.empty(), + Optional.empty(), + Instant.parse("2026-08-10T00:00:00Z"))) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void handleResultPermitsExactlyTheFourDeclaredOutcomes() { + assertThat(HandleResult.class.getPermittedSubclasses()) + .containsExactlyInAnyOrder( + HandleResult.Success.class, + HandleResult.Retry.class, + HandleResult.DeadLetter.class, + HandleResult.Reject.class); + } + + @Test + void unknownSettlementIsNotSuccess() { + SettlementResult result = + new SettlementResult( + SettlementCompletion.UNKNOWN, + SettlementEvidence.unknown(), + Optional.of( + FailureDescriptor.of( + FailureCategory.AMBIGUOUS, "SETTLEMENT_UNKNOWN", "ack not confirmed"))); + + assertThat(result.completion()).isNotEqualTo(SettlementCompletion.SETTLED); + assertThat(result.evidence().redeliveryPossible()).isTrue(); + } + + @Test + void settledResultCannotLeaveRedeliveryPossible() { + assertThatThrownBy( + () -> + new SettlementResult( + SettlementCompletion.SETTLED, SettlementEvidence.unknown(), Optional.empty())) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void deliveryContextReportsHandlerDeadlineExpiry() { + DeliveryContext context = + new DeliveryContext(Instant.parse("2026-08-10T00:00:30Z"), false, "order-projection"); + + assertThat(context.isExpired(Instant.parse("2026-08-10T00:00:29Z"))).isFalse(); + assertThat(context.isExpired(Instant.parse("2026-08-10T00:00:30Z"))).isTrue(); + } +} + +/** Builds delivery metadata for tests without exposing a production factory. */ +final class DeliveryMetadataFixture { + + private DeliveryMetadataFixture() {} + + static DeliveryMetadata initial() { + return new DeliveryMetadata( + new DestinationName("order-events"), + 1, + false, + Optional.empty(), + Optional.of("order.events.v1-0"), + Optional.of("order-projection"), + Instant.parse("2026-08-10T00:00:00Z")); + } +} diff --git a/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/destination/DestinationCapabilityTest.java b/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/destination/DestinationCapabilityTest.java new file mode 100644 index 00000000..eaf9e328 --- /dev/null +++ b/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/destination/DestinationCapabilityTest.java @@ -0,0 +1,60 @@ +package dev.caskeleton.messaging.api.destination; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.MessageType; +import org.junit.jupiter.api.Test; + +class DestinationCapabilityTest { + + @Test + void logicalDestinationDoesNotContainBrokerSpecificAddress() { + MessageDestination destination = + new MessageDestination<>( + new DestinationName("order-events"), new MessageType("order.created"), String.class); + + assertThat(destination.name().value()).isEqualTo("order-events"); + } + + @Test + void destinationNameRejectsBrokerSeparatorsAndWhitespace() { + assertThatThrownBy(() -> new DestinationName("topic://orders")) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new DestinationName("order events")) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new DestinationName("Order-Events")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void emptyCapabilitySetAdvertisesNothing() { + MessagingCapabilities capabilities = MessagingCapabilities.none(); + + assertThat(capabilities.brokerAcknowledgement()).isFalse(); + assertThat(capabilities.replay()).isFalse(); + assertThat(capabilities.brokerTransaction()).isFalse(); + } + + @Test + void destinationCapabilitiesRequireABrokerName() { + MessagingCapabilities capabilities = MessagingCapabilities.none(); + DestinationName name = new DestinationName("order-events"); + + assertThatThrownBy(() -> new DestinationCapabilities(name, " ", capabilities)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void destinationKindCoversEveryDeclaredInteractionPattern() { + assertThat(DestinationKind.values()) + .containsExactly( + DestinationKind.ASYNC_COMMAND, + DestinationKind.DOMAIN_EVENT, + DestinationKind.INTEGRATION_EVENT, + DestinationKind.WORK_QUEUE, + DestinationKind.PUBLISH_SUBSCRIBE, + DestinationKind.EVENT_STREAM, + DestinationKind.REQUEST_REPLY); + } +} diff --git a/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/publish/PublishResultTest.java b/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/publish/PublishResultTest.java new file mode 100644 index 00000000..dc342197 --- /dev/null +++ b/src/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/publish/PublishResultTest.java @@ -0,0 +1,166 @@ +package dev.caskeleton.messaging.api.publish; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import java.time.Duration; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class PublishResultTest { + + private static final FailureDescriptor AMBIGUOUS_FAILURE = + FailureDescriptor.of(FailureCategory.AMBIGUOUS, "CONFIRM_TIMEOUT", "confirm timed out"); + + @Test + void confirmedResultRequiresBrokerAcceptance() { + PublishEvidence evidence = + new PublishEvidence(false, TransmissionEvidence.TRANSMITTED, false, ConfirmationLevel.NONE); + + assertThatThrownBy( + () -> + new PublishResult( + PublishCompletion.CONFIRMED, + evidence, + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + Duration.ofMillis(10), + Optional.empty())) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void evidenceCannotClaimConfirmationWithoutBrokerAcceptance() { + assertThatThrownBy( + () -> + new PublishEvidence( + true, + TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED, + false, + ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void ambiguousResultCannotClaimReplicationConfirmation() { + PublishEvidence evidence = + new PublishEvidence( + true, + TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED, + true, + ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK); + + assertThatThrownBy( + () -> + new PublishResult( + PublishCompletion.AMBIGUOUS, + evidence, + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + Duration.ofSeconds(5), + Optional.of(AMBIGUOUS_FAILURE))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("ambiguous"); + } + + @Test + void confirmedResultCannotBeUnroutable() { + assertThatThrownBy( + () -> + new PublishResult( + PublishCompletion.CONFIRMED, + PublishEvidence.confirmed(ConfirmationLevel.BROKER_ACK), + RoutingOutcome.UNROUTABLE, + Optional.empty(), + 1, + Duration.ofMillis(3), + Optional.empty())) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void untransmittedPublishIsRejectedRatherThanAmbiguous() { + assertThatThrownBy( + () -> + new PublishResult( + PublishCompletion.AMBIGUOUS, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ofMillis(1), + Optional.of(AMBIGUOUS_FAILURE))) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void nonConfirmedResultRequiresAFailureDescriptor() { + assertThatThrownBy( + () -> + new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ofMillis(1), + Optional.empty())) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void confirmedPublishIsWellFormedAndNotAmbiguous() { + PublishResult result = + new PublishResult( + PublishCompletion.CONFIRMED, + PublishEvidence.confirmed(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK), + RoutingOutcome.ROUTED, + Optional.empty(), + 1, + Duration.ofMillis(12), + Optional.empty()); + + assertThat(result.mayHaveBeenStored()).isFalse(); + assertThat(result.evidence().confirmationLevel()) + .isEqualTo(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK); + } + + @Test + void ambiguousPublishReportsThatTheBrokerMayHoldTheMessage() { + PublishResult result = + new PublishResult( + PublishCompletion.AMBIGUOUS, + PublishEvidence.ambiguous(), + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + Duration.ofSeconds(5), + Optional.of(AMBIGUOUS_FAILURE)); + + assertThat(result.mayHaveBeenStored()).isTrue(); + } + + @Test + void attemptsCountTheFirstTransportAttemptAsOne() { + assertThatThrownBy( + () -> + new PublishResult( + PublishCompletion.CONFIRMED, + PublishEvidence.confirmed(ConfirmationLevel.BROKER_ACK), + RoutingOutcome.ROUTED, + Optional.empty(), + 0, + Duration.ofMillis(1), + Optional.empty())) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void m1OptionsCarryNoBrokerHints() { + assertThat(PublishOptions.defaults().isM1Compatible()).isTrue(); + } +} diff --git a/src/messaging/messaging-inbox-jpa/build.gradle b/src/messaging/messaging-inbox-jpa/build.gradle new file mode 100644 index 00000000..c866982e --- /dev/null +++ b/src/messaging/messaging-inbox-jpa/build.gradle @@ -0,0 +1,16 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-reliability-api') + + implementation 'org.springframework:spring-jdbc' + implementation 'org.springframework:spring-tx' + + // Live-database certification. The reliability patterns are claims about transaction + // boundaries and uniqueness constraints, and only a real database can settle them. + testImplementation project(':messaging:messaging-testkit') + testImplementation 'org.testcontainers:testcontainers-postgresql' + testImplementation 'org.testcontainers:testcontainers-junit-jupiter' + testImplementation 'org.postgresql:postgresql' +} diff --git a/src/messaging/messaging-inbox-jpa/gradle.lockfile b/src/messaging/messaging-inbox-jpa/gradle.lockfile new file mode 100644 index 00000000..a52b0019 --- /dev/null +++ b/src/messaging/messaging-inbox-jpa/gradle.lockfile @@ -0,0 +1,105 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.fasterxml.jackson.core:jackson-annotations:2.20=testCompileClasspath,testRuntimeClasspath +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.docker-java:docker-java-api:3.7.0=testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport-zerodep:3.7.0=testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport:3.7.0=testCompileClasspath,testRuntimeClasspath +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-codec:commons-codec:1.19.0=testCompileClasspath,testRuntimeClasspath +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.20.0=testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.21.0=spotbugs +commons-logging:commons-logging:1.3.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna:5.18.1=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-compress:1.28.0=testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.checkerframework:checker-qual:3.49.5=testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jetbrains:annotations:17.0.0=testCompileClasspath,testRuntimeClasspath +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,compileClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.postgresql:postgresql:42.7.8=testCompileClasspath,testRuntimeClasspath +org.reflections:reflections:0.10.2=checkstyle +org.rnorth.duct-tape:duct-tape:1.0.8=testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.springframework:spring-beans:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-core:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-jdbc:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-tx:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-database-commons:2.0.2=testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-jdbc:2.0.2=testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-junit-jupiter:2.0.2=testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-postgresql:2.0.2=testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers:2.0.2=testCompileClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty= diff --git a/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/IdempotentConsumer.java b/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/IdempotentConsumer.java new file mode 100644 index 00000000..35f2bef6 --- /dev/null +++ b/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/IdempotentConsumer.java @@ -0,0 +1,77 @@ +package dev.caskeleton.messaging.inbox; + +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.reliability.InboxRepository; +import java.time.Instant; +import java.util.Objects; +import java.util.function.Supplier; + +/** + * Runs a handler's side effect at most once per message and consumer. + * + *

The reservation and the side effect must share one transaction. This class does not open that + * transaction itself — the caller supplies a runner that does — because the boundary belongs to the + * application's data access layer, and a nested or separate transaction here would silently break + * the guarantee while still looking correct. + * + *

A duplicate is not an error. It is the expected consequence of at-least-once delivery, so the + * skip path is a normal outcome rather than an exception. + */ +public final class IdempotentConsumer { + + private final InboxRepository inbox; + private final TransactionRunner transactions; + + /** + * Creates an idempotent consumer. + * + * @param inbox the inbox storage + * @param transactions runs work inside one database transaction + */ + public IdempotentConsumer(InboxRepository inbox, TransactionRunner transactions) { + this.inbox = Objects.requireNonNull(inbox, "inbox must not be null"); + this.transactions = Objects.requireNonNull(transactions, "transactions must not be null"); + } + + /** + * Runs the side effect unless this message was already processed. + * + * @param the side effect's result type + * @param messageId the logical message identity + * @param consumerId the consumer's stable identity + * @param now the current instant + * @param sideEffect the business work + * @return the outcome, including whether the work ran + */ + public InboxOutcome runOnce( + MessageId messageId, String consumerId, Instant now, Supplier sideEffect) { + Objects.requireNonNull(messageId, "messageId must not be null"); + Objects.requireNonNull(sideEffect, "sideEffect must not be null"); + Objects.requireNonNull(now, "now must not be null"); + if (consumerId == null || consumerId.isBlank()) { + throw new IllegalArgumentException("consumerId must not be blank"); + } + + return transactions.inTransaction( + () -> { + if (!inbox.reserve(messageId, consumerId, now)) { + return InboxOutcome.duplicate(); + } + return InboxOutcome.processed(sideEffect.get()); + }); + } + + /** Runs work inside exactly one database transaction. */ + @FunctionalInterface + public interface TransactionRunner { + + /** + * Runs the supplied work in a transaction and returns its result. + * + * @param the result type + * @param work the work to run + * @return the work's result + */ + T inTransaction(Supplier work); + } +} diff --git a/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/InboxCleanupJob.java b/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/InboxCleanupJob.java new file mode 100644 index 00000000..fbeda2e7 --- /dev/null +++ b/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/InboxCleanupJob.java @@ -0,0 +1,65 @@ +package dev.caskeleton.messaging.inbox; + +import dev.caskeleton.messaging.reliability.InboxRepository; +import java.time.Instant; +import java.util.Objects; + +/** + * Deletes inbox rows that are older than the retention policy allows. + * + *

Deletes in bounded batches. A single unbounded {@code DELETE} over a table that has been + * accumulating for weeks holds locks long enough to block the very reservations the inbox exists to + * serve, so the cleanup would cause the outage it is meant to prevent. + * + *

The policy is validated before the first deletion. Running a cleanup under a retention that is + * shorter than the redelivery window would actively create the duplicate-processing bug, so the job + * refuses to start rather than dutifully deleting the rows. + */ +public final class InboxCleanupJob { + + /** How many rows one pass deletes before yielding. */ + public static final int DEFAULT_BATCH_SIZE = 1_000; + + private final InboxRepository inbox; + private final InboxRetentionPolicy policy; + private final int maxBatches; + + /** + * Creates a cleanup job. + * + * @param inbox the inbox storage + * @param policy the retention policy + * @param maxBatches how many batches one run may delete + */ + public InboxCleanupJob(InboxRepository inbox, InboxRetentionPolicy policy, int maxBatches) { + this.inbox = Objects.requireNonNull(inbox, "inbox must not be null"); + this.policy = Objects.requireNonNull(policy, "policy must not be null"); + if (maxBatches < 1) { + throw new IllegalArgumentException("maxBatches must be at least 1"); + } + this.maxBatches = maxBatches; + policy.validate(); + } + + /** + * Deletes expired rows, up to the configured batch ceiling. + * + * @param now the current instant + * @return how many rows were removed + */ + public int runOnce(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + Instant cutoff = policy.cutoff(now); + + int removed = 0; + for (int batch = 0; batch < maxBatches; batch++) { + int deleted = inbox.purgeProcessedBefore(cutoff); + removed += deleted; + // A short batch means the backlog is drained; continuing would just re-scan an empty range. + if (deleted == 0) { + break; + } + } + return removed; + } +} diff --git a/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/InboxOutcome.java b/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/InboxOutcome.java new file mode 100644 index 00000000..cd7ddbed --- /dev/null +++ b/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/InboxOutcome.java @@ -0,0 +1,34 @@ +package dev.caskeleton.messaging.inbox; + +import java.util.Optional; + +/** + * Whether a message's side effect ran or was suppressed as a duplicate. + * + * @param the side effect's result type + * @param processed true when the side effect ran for the first time + * @param result the side effect's result, absent for a duplicate + */ +public record InboxOutcome(boolean processed, Optional result) { + + /** + * Returns an outcome for a first-time delivery. + * + * @param the result type + * @param value the side effect's result + * @return the processed outcome + */ + public static InboxOutcome processed(T value) { + return new InboxOutcome<>(true, Optional.ofNullable(value)); + } + + /** + * Returns an outcome for a suppressed duplicate. + * + * @param the result type + * @return the duplicate outcome + */ + public static InboxOutcome duplicate() { + return new InboxOutcome<>(false, Optional.empty()); + } +} diff --git a/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/InboxRetentionPolicy.java b/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/InboxRetentionPolicy.java new file mode 100644 index 00000000..ccafcaf1 --- /dev/null +++ b/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/InboxRetentionPolicy.java @@ -0,0 +1,78 @@ +package dev.caskeleton.messaging.inbox; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; + +/** + * How long inbox rows are kept, and the rule that makes the number safe. + * + *

Retention must exceed the broker's maximum redelivery window. That is not a tuning preference: + * a row pruned while the broker can still redeliver its message turns the inbox into a no-op for + * exactly that message, and the side effect runs a second time. The failure is silent, rare, and + * only happens under the conditions that already made the day bad. + * + *

The safety margin is multiplicative rather than additive so that it scales with the window + * itself. A stream whose redelivery window is measured in days needs more slack than one measured + * in minutes, for the same reason: the estimate of that window is proportionally less certain. + * + * @param retention how long a processed row is kept + * @param maximumRedeliveryWindow the longest the broker may redelivery a message + */ +public record InboxRetentionPolicy(Duration retention, Duration maximumRedeliveryWindow) { + + /** How much longer than the redelivery window retention must be. */ + public static final double REQUIRED_SAFETY_FACTOR = 2.0; + + /** The default for a broker whose redelivery window is bounded by a one-day log retention. */ + public static final Duration DEFAULT_RETENTION = Duration.ofDays(7); + + public InboxRetentionPolicy { + Objects.requireNonNull(retention, "retention must not be null"); + Objects.requireNonNull(maximumRedeliveryWindow, "maximumRedeliveryWindow must not be null"); + if (retention.isNegative() || retention.isZero()) { + throw new IllegalArgumentException("retention must be positive"); + } + if (maximumRedeliveryWindow.isNegative() || maximumRedeliveryWindow.isZero()) { + throw new IllegalArgumentException("maximumRedeliveryWindow must be positive"); + } + } + + /** + * Refuses a retention that could prune a row the broker can still redeliver. + * + * @throws MessagingConfigurationException when retention is too short + */ + public void validate() { + Duration required = required(); + if (retention.compareTo(required) < 0) { + throw new MessagingConfigurationException( + "INBOX_RETENTION_TOO_SHORT", + "inbox retention of %s is below the %s required for a %s redelivery window; a pruned row" + .formatted(retention, required, maximumRedeliveryWindow) + + " lets a late redelivery apply its effect twice"); + } + } + + /** + * Returns the shortest retention this redelivery window allows. + * + * @return the minimum safe retention + */ + public Duration required() { + return Duration.ofMillis( + Math.round(maximumRedeliveryWindow.toMillis() * REQUIRED_SAFETY_FACTOR)); + } + + /** + * Returns the cutoff before which rows may be deleted. + * + * @param now the current instant + * @return the retention cutoff + */ + public Instant cutoff(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + return now.minus(retention); + } +} diff --git a/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/JdbcInboxRepository.java b/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/JdbcInboxRepository.java new file mode 100644 index 00000000..8edb4faa --- /dev/null +++ b/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/JdbcInboxRepository.java @@ -0,0 +1,116 @@ +package dev.caskeleton.messaging.inbox; + +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.reliability.InboxRepository; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Objects; +import javax.sql.DataSource; + +/** + * The PostgreSQL inbox. + * + *

Reservation is an {@code INSERT ... ON CONFLICT DO NOTHING} whose affected-row count is the + * answer: one means first delivery, zero means already processed. The composite primary key does + * the work, so there is no read-then-write race — two concurrent deliveries of the same message + * cannot both see "not processed" and both proceed. + * + *

The connection-taking overload is the one that matters. Reserving on a connection other than + * the one carrying the business write means the two commit independently, which reopens exactly the + * duplicate window the Inbox exists to close. The no-argument overload is provided only for + * retention sweeps and read-only queries. + */ +public final class JdbcInboxRepository implements InboxRepository { + + private static final String RESERVE = + """ + INSERT INTO messaging_inbox (message_id, consumer_id, processed_at) + VALUES (?, ?, ?) + ON CONFLICT (message_id, consumer_id) DO NOTHING + """; + + private final DataSource dataSource; + + /** + * Creates a repository over a data source. + * + * @param dataSource the inbox data source + */ + public JdbcInboxRepository(DataSource dataSource) { + this.dataSource = Objects.requireNonNull(dataSource, "dataSource must not be null"); + } + + /** + * Reserves a message on the caller's transactional connection. + * + * @param connection the connection the business write is running on + * @param messageId the logical message identity + * @param consumerId the consumer's stable identity + * @param now the current instant + * @return true when this is the first time + */ + public boolean reserve( + Connection connection, MessageId messageId, String consumerId, Instant now) { + Objects.requireNonNull(connection, "connection must not be null"); + Objects.requireNonNull(messageId, "messageId must not be null"); + Objects.requireNonNull(now, "now must not be null"); + if (consumerId == null || consumerId.isBlank()) { + throw new IllegalArgumentException("consumerId must not be blank"); + } + try (PreparedStatement statement = connection.prepareStatement(RESERVE)) { + statement.setObject(1, messageId.value()); + statement.setString(2, consumerId); + statement.setTimestamp(3, Timestamp.from(now)); + return statement.executeUpdate() == 1; + } catch (SQLException exception) { + throw new MessagingConfigurationException( + "INBOX_RESERVE_FAILED", "could not reserve an inbox record", exception); + } + } + + @Override + public boolean reserve(MessageId messageId, String consumerId, Instant now) { + try (Connection connection = dataSource.getConnection()) { + return reserve(connection, messageId, consumerId, now); + } catch (SQLException exception) { + throw new MessagingConfigurationException( + "INBOX_RESERVE_FAILED", "could not reserve an inbox record", exception); + } + } + + @Override + public boolean isProcessed(MessageId messageId, String consumerId) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = + connection.prepareStatement( + "SELECT 1 FROM messaging_inbox WHERE message_id = ? AND consumer_id = ?")) { + statement.setObject(1, messageId.value()); + statement.setString(2, consumerId); + try (ResultSet results = statement.executeQuery()) { + return results.next(); + } + } catch (SQLException exception) { + throw new MessagingConfigurationException( + "INBOX_QUERY_FAILED", "could not query the inbox", exception); + } + } + + @Override + public int purgeProcessedBefore(Instant processedBefore) { + Objects.requireNonNull(processedBefore, "processedBefore must not be null"); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = + connection.prepareStatement("DELETE FROM messaging_inbox WHERE processed_at < ?")) { + statement.setTimestamp(1, Timestamp.from(processedBefore)); + return statement.executeUpdate(); + } catch (SQLException exception) { + throw new MessagingConfigurationException( + "INBOX_PURGE_FAILED", "could not purge the inbox", exception); + } + } +} diff --git a/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/TransactionalInboxHandler.java b/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/TransactionalInboxHandler.java new file mode 100644 index 00000000..2f228a21 --- /dev/null +++ b/src/messaging/messaging-inbox-jpa/src/main/java/dev/caskeleton/messaging/inbox/TransactionalInboxHandler.java @@ -0,0 +1,104 @@ +package dev.caskeleton.messaging.inbox; + +import dev.caskeleton.messaging.api.delivery.HandleResult; +import dev.caskeleton.messaging.api.delivery.MessageDelivery; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.reliability.IdempotentMessageHandler; +import dev.caskeleton.messaging.reliability.InboxResult; +import dev.caskeleton.messaging.reliability.TransactionalMessageAction; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.Supplier; + +/** + * The JDBC-backed {@link IdempotentMessageHandler}. + * + *

Reservation and effect commit together, in the runner's single transaction. Everything else + * about this class follows from that: it cannot settle the message (settlement is not + * transactional), it cannot publish (the publish would survive a rollback), and it cannot catch and + * swallow the action's exception (the rollback is how the reservation is undone). + * + *

An already-reserved message returns success. Under at-least-once delivery a duplicate is the + * normal case, not an error; reporting failure would send a message whose work is complete to the + * dead-letter queue. + * + * @param the payload type + */ +public final class TransactionalInboxHandler implements IdempotentMessageHandler { + + private final IdempotentConsumer consumer; + private final Supplier clock; + + /** + * Creates a handler. + * + * @param consumer the reservation-and-effect runner + * @param clock supplies the current instant + */ + public TransactionalInboxHandler(IdempotentConsumer consumer, Supplier clock) { + this.consumer = Objects.requireNonNull(consumer, "consumer must not be null"); + this.clock = Objects.requireNonNull(clock, "clock must not be null"); + } + + @Override + public CompletionStage handleOnce( + String consumerName, MessageDelivery delivery, TransactionalMessageAction action) { + Objects.requireNonNull(delivery, "delivery must not be null"); + Objects.requireNonNull(action, "action must not be null"); + if (consumerName == null || consumerName.isBlank()) { + throw new IllegalArgumentException("consumerName must not be blank"); + } + + try { + InboxOutcome outcome = + consumer.runOnce( + delivery.message().messageId(), + consumerName, + clock.get(), + () -> { + try { + action.apply(delivery); + } catch (Exception failure) { + // Wrapped, not swallowed: the transaction runner has to see a throw to roll the + // reservation back along with the effect. + throw new ActionFailed(failure); + } + return InboxResult.APPLIED; + }); + + return CompletableFuture.completedFuture( + outcome.processed() ? HandleResult.success() : duplicateIsSuccess()); + } catch (ActionFailed failure) { + return CompletableFuture.completedFuture(retryable(failure.getCause())); + } + } + + private static HandleResult duplicateIsSuccess() { + // The effect already ran in an earlier delivery. Settling is correct; redelivering is not. + return HandleResult.success(); + } + + private static HandleResult retryable(Throwable cause) { + return new HandleResult.Retry( + new FailureDescriptor( + FailureCategory.TRANSIENT_INFRASTRUCTURE, + "INBOX_ACTION_FAILED", + true, + "the transactional action failed and its reservation was rolled back", + Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName()))); + } + + /** Carries the action's checked exception out through the transaction runner. */ + private static final class ActionFailed extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private ActionFailed(Throwable cause) { + super(cause); + } + } +} diff --git a/src/messaging/messaging-inbox-jpa/src/main/resources/db/migration/messaging/V2__messaging_inbox.sql b/src/messaging/messaging-inbox-jpa/src/main/resources/db/migration/messaging/V2__messaging_inbox.sql new file mode 100644 index 00000000..bb966005 --- /dev/null +++ b/src/messaging/messaging-inbox-jpa/src/main/resources/db/migration/messaging/V2__messaging_inbox.sql @@ -0,0 +1,18 @@ +-- Consumer inbox. +-- +-- The composite primary key is the deduplication mechanism: reserving a message is an INSERT that +-- either succeeds or violates the key, inside the same transaction as the handler's side effect. +-- Two independent consumers of the same event each get their own row, so one cannot suppress the +-- other. +CREATE TABLE messaging_inbox +( + message_id UUID NOT NULL, + consumer_id VARCHAR(160) NOT NULL, + processed_at TIMESTAMPTZ NOT NULL, + CONSTRAINT pk_messaging_inbox PRIMARY KEY (message_id, consumer_id) +); + +-- Retention sweeps read this. The window must outlive the broker's maximum redelivery delay: +-- pruning a row before its message can still be redelivered reopens the duplicate this table +-- exists to close. +CREATE INDEX ix_messaging_inbox_processed_at ON messaging_inbox (processed_at); diff --git a/src/messaging/messaging-inbox-jpa/src/test/java/dev/caskeleton/messaging/inbox/IdempotentConsumerTest.java b/src/messaging/messaging-inbox-jpa/src/test/java/dev/caskeleton/messaging/inbox/IdempotentConsumerTest.java new file mode 100644 index 00000000..9cc55baf --- /dev/null +++ b/src/messaging/messaging-inbox-jpa/src/test/java/dev/caskeleton/messaging/inbox/IdempotentConsumerTest.java @@ -0,0 +1,140 @@ +package dev.caskeleton.messaging.inbox; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.reliability.InboxRepository; +import java.time.Instant; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import org.junit.jupiter.api.Test; + +class IdempotentConsumerTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + private final InMemoryInbox inbox = new InMemoryInbox(); + private final SingleTransactionRunner transactions = new SingleTransactionRunner(); + private final IdempotentConsumer consumer = new IdempotentConsumer(inbox, transactions); + + @Test + void theFirstDeliveryRunsTheSideEffect() { + AtomicInteger effects = new AtomicInteger(); + MessageId messageId = MessageId.newId(); + + InboxOutcome outcome = + consumer.runOnce(messageId, "order-projection", NOW, effects::incrementAndGet); + + assertThat(outcome.processed()).isTrue(); + assertThat(outcome.result()).hasValue(1); + assertThat(effects.get()).isEqualTo(1); + } + + @Test + void aRedeliverySkipsTheSideEffect() { + AtomicInteger effects = new AtomicInteger(); + MessageId messageId = MessageId.newId(); + + consumer.runOnce(messageId, "order-projection", NOW, effects::incrementAndGet); + InboxOutcome second = + consumer.runOnce(messageId, "order-projection", NOW, effects::incrementAndGet); + + assertThat(second.processed()).isFalse(); + assertThat(second.result()).isEmpty(); + assertThat(effects.get()).isEqualTo(1); + } + + @Test + void twoConsumersEachProcessTheSameMessageOnce() { + AtomicInteger effects = new AtomicInteger(); + MessageId messageId = MessageId.newId(); + + consumer.runOnce(messageId, "order-projection", NOW, effects::incrementAndGet); + InboxOutcome other = + consumer.runOnce(messageId, "billing-projection", NOW, effects::incrementAndGet); + + assertThat(other.processed()).isTrue(); + assertThat(effects.get()).isEqualTo(2); + } + + @Test + void theReservationAndTheSideEffectShareOneTransaction() { + MessageId messageId = MessageId.newId(); + + consumer.runOnce(messageId, "order-projection", NOW, () -> transactions.depthDuringWork); + + assertThat(transactions.transactions).isEqualTo(1); + assertThat(transactions.depthDuringWork).isEqualTo(1); + } + + @Test + void aProcessedMessageIsVisibleToTheRepositoryQuery() { + MessageId messageId = MessageId.newId(); + consumer.runOnce(messageId, "order-projection", NOW, () -> "done"); + + assertThat(inbox.isProcessed(messageId, "order-projection")).isTrue(); + assertThat(inbox.isProcessed(messageId, "billing-projection")).isFalse(); + } + + @Test + void retentionPruningRemovesOldRows() { + consumer.runOnce(MessageId.newId(), "order-projection", NOW, () -> "done"); + + assertThat(inbox.purgeProcessedBefore(NOW.plusSeconds(1))).isEqualTo(1); + assertThat(inbox.size()).isZero(); + } +} + +/** A deterministic inbox with the composite-key uniqueness the pattern relies on. */ +final class InMemoryInbox implements InboxRepository { + + private final Set reserved = new LinkedHashSet<>(); + + @Override + public boolean reserve(MessageId messageId, String consumerId, Instant now) { + return reserved.add(key(messageId, consumerId)); + } + + @Override + public boolean isProcessed(MessageId messageId, String consumerId) { + return reserved.contains(key(messageId, consumerId)); + } + + @Override + public int purgeProcessedBefore(Instant processedBefore) { + int removed = reserved.size(); + reserved.clear(); + return removed; + } + + int size() { + return reserved.size(); + } + + private static String key(MessageId messageId, String consumerId) { + return messageId.value() + "|" + consumerId; + } +} + +/** Records how many transactions were opened and how deeply they nested. */ +final class SingleTransactionRunner implements IdempotentConsumer.TransactionRunner { + + int transactions; + int depthDuringWork; + + private int depth; + + @Override + public T inTransaction(Supplier work) { + transactions++; + depth++; + try { + depthDuringWork = depth; + return work.get(); + } finally { + depth--; + } + } +} diff --git a/src/messaging/messaging-inbox-jpa/src/test/java/dev/caskeleton/messaging/inbox/InboxOperationsTest.java b/src/messaging/messaging-inbox-jpa/src/test/java/dev/caskeleton/messaging/inbox/InboxOperationsTest.java new file mode 100644 index 00000000..5404e76e --- /dev/null +++ b/src/messaging/messaging-inbox-jpa/src/test/java/dev/caskeleton/messaging/inbox/InboxOperationsTest.java @@ -0,0 +1,153 @@ +package dev.caskeleton.messaging.inbox; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.reliability.InboxRepository; +import dev.caskeleton.messaging.reliability.InboxResult; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class InboxOperationsTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + /** An inbox that remembers reservations and records the cutoffs cleanup asked for. */ + private static final class InMemoryInbox implements InboxRepository { + + private final Set reserved = new HashSet<>(); + private final List cutoffs = new ArrayList<>(); + private final List deletions; + private int pass; + + private InMemoryInbox(List deletions) { + this.deletions = deletions; + } + + @Override + public boolean reserve(MessageId messageId, String consumerId, Instant now) { + return reserved.add(messageId.value() + "|" + consumerId); + } + + @Override + public boolean isProcessed(MessageId messageId, String consumerId) { + return reserved.contains(messageId.value() + "|" + consumerId); + } + + @Override + public int purgeProcessedBefore(Instant processedBefore) { + cutoffs.add(processedBefore); + return pass < deletions.size() ? deletions.get(pass++) : 0; + } + } + + private static InboxRetentionPolicy policy(Duration retention, Duration window) { + return new InboxRetentionPolicy(retention, window); + } + + @Test + void retentionShorterThanTheRedeliveryWindowIsRefused() { + assertThatThrownBy(() -> policy(Duration.ofHours(1), Duration.ofHours(6)).validate()) + .as("a pruned row lets a late redelivery apply its effect a second time") + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("twice"); + } + + @Test + void retentionAtTheSafetyFactorIsAccepted() { + assertThatCode(() -> policy(Duration.ofHours(12), Duration.ofHours(6)).validate()) + .doesNotThrowAnyException(); + } + + @Test + void theRequiredRetentionScalesWithTheWindow() { + assertThat(policy(Duration.ofDays(30), Duration.ofDays(2)).required()) + .isEqualTo(Duration.ofDays(4)); + } + + @Test + void aCleanupJobRefusesToStartUnderAnUnsafeRetention() { + assertThatThrownBy( + () -> + new InboxCleanupJob( + new InMemoryInbox(List.of(0)), + policy(Duration.ofMinutes(1), Duration.ofHours(1)), + 5)) + .as("dutifully deleting under a bad policy would create the bug the inbox prevents") + .isInstanceOf(MessagingConfigurationException.class); + } + + @Test + void cleanupDeletesInBoundedBatches() { + InMemoryInbox inbox = new InMemoryInbox(List.of(1000, 500)); + + int removed = + new InboxCleanupJob(inbox, policy(Duration.ofDays(7), Duration.ofDays(1)), 10).runOnce(NOW); + + assertThat(removed).isEqualTo(1500); + assertThat(inbox.cutoffs).hasSize(3); + } + + @Test + void cleanupHonoursTheBatchCeilingSoItCannotRunForever() { + InMemoryInbox inbox = new InMemoryInbox(List.of(1000, 1000, 1000, 1000, 1000, 1000)); + + new InboxCleanupJob(inbox, policy(Duration.ofDays(7), Duration.ofDays(1)), 2).runOnce(NOW); + + assertThat(inbox.cutoffs).hasSize(2); + } + + @Test + void theSameMessageIsAppliedOncePerConsumerNotOncePerMessage() { + InMemoryInbox inbox = new InMemoryInbox(List.of()); + IdempotentConsumer consumer = new IdempotentConsumer(inbox, new DirectRunner()); + MessageId messageId = MessageId.newId(); + + InboxOutcome first = consumer.runOnce(messageId, "billing", NOW, () -> "charged"); + InboxOutcome second = consumer.runOnce(messageId, "analytics", NOW, () -> "counted"); + + assertThat(first.processed()).isTrue(); + assertThat(second.processed()) + .as("two independent consumers are each entitled to apply their own effect once") + .isTrue(); + } + + @Test + void aRedeliveryToTheSameConsumerIsSuppressed() { + InMemoryInbox inbox = new InMemoryInbox(List.of()); + IdempotentConsumer consumer = new IdempotentConsumer(inbox, new DirectRunner()); + MessageId messageId = MessageId.newId(); + + consumer.runOnce(messageId, "billing", NOW, () -> "charged"); + InboxOutcome redelivery = consumer.runOnce(messageId, "billing", NOW, () -> "charged"); + + assertThat(redelivery.processed()).isFalse(); + assertThat(redelivery.result()).isEmpty(); + } + + @Test + void anAlreadyAppliedMessageIsSafeToSettleButAClaimedOneIsNot() { + assertThat(InboxResult.APPLIED.isSafeToSettle()).isTrue(); + assertThat(InboxResult.ALREADY_APPLIED.isSafeToSettle()).isTrue(); + assertThat(InboxResult.CLAIMED_ELSEWHERE.isSafeToSettle()) + .as("the other transaction may still roll back, and this is the last copy") + .isFalse(); + } + + /** Runs the work inline; the real runner opens a database transaction. */ + private static final class DirectRunner implements IdempotentConsumer.TransactionRunner { + + @Override + public T inTransaction(java.util.function.Supplier work) { + return work.get(); + } + } +} diff --git a/src/messaging/messaging-inbox-jpa/src/test/java/dev/caskeleton/messaging/inbox/InboxPostgresIT.java b/src/messaging/messaging-inbox-jpa/src/test/java/dev/caskeleton/messaging/inbox/InboxPostgresIT.java new file mode 100644 index 00000000..d202a3ec --- /dev/null +++ b/src/messaging/messaging-inbox-jpa/src/test/java/dev/caskeleton/messaging/inbox/InboxPostgresIT.java @@ -0,0 +1,181 @@ +package dev.caskeleton.messaging.inbox; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.testkit.DockerAvailability; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicInteger; +import javax.sql.DataSource; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.postgresql.ds.PGSimpleDataSource; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.postgresql.PostgreSQLContainer; + +/** + * Certifies the inbox against a real PostgreSQL. + * + *

The whole pattern rests on a uniqueness constraint and a shared transaction. Both are database + * behaviours: a double can be written to agree with them, and a real one either enforces them or + * does not. + * + *

The last test is the one that matters most. It commits an inbox reservation and a business + * write together, then rolls a second attempt back, and asserts that the side effect did not happen + * twice and the reservation did not survive the rollback. + */ +@Testcontainers +@EnabledIf("dockerAvailable") +class InboxPostgresIT { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer("postgres:16-alpine") + .withDatabaseName("messaging") + .withUsername("messaging") + .withPassword("messaging"); + + private DataSource dataSource; + private JdbcInboxRepository repository; + + static boolean dockerAvailable() { + return DockerAvailability.isAvailable(); + } + + @BeforeEach + void migrate() throws SQLException, IOException { + dataSource = dataSource(); + try (Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS messaging_inbox"); + statement.execute("DROP TABLE IF EXISTS projections"); + statement.execute(migration()); + statement.execute("CREATE TABLE projections (id TEXT PRIMARY KEY, applied INT NOT NULL)"); + } + repository = new JdbcInboxRepository(dataSource); + } + + @Test + void theFirstReservationSucceedsAndTheSecondDoesNot() { + MessageId messageId = MessageId.newId(); + + assertThat(repository.reserve(messageId, "order-projection", NOW)).isTrue(); + assertThat(repository.reserve(messageId, "order-projection", NOW)).isFalse(); + } + + @Test + void twoConsumersEachReserveTheSameMessageOnce() { + MessageId messageId = MessageId.newId(); + + assertThat(repository.reserve(messageId, "order-projection", NOW)).isTrue(); + assertThat(repository.reserve(messageId, "billing-projection", NOW)).isTrue(); + assertThat(repository.reserve(messageId, "order-projection", NOW)).isFalse(); + } + + @Test + void aProcessedMessageIsVisibleToTheQuery() { + MessageId messageId = MessageId.newId(); + repository.reserve(messageId, "order-projection", NOW); + + assertThat(repository.isProcessed(messageId, "order-projection")).isTrue(); + assertThat(repository.isProcessed(messageId, "billing-projection")).isFalse(); + } + + @Test + void aRedeliveryDoesNotApplyTheSideEffectTwice() throws SQLException { + MessageId messageId = MessageId.newId(); + AtomicInteger applied = new AtomicInteger(); + + handle(messageId, applied); + handle(messageId, applied); + + assertThat(applied.get()).as("the handler ran once").isEqualTo(1); + assertThat(projectionCount("o-1")).isEqualTo(1); + } + + @Test + void aRolledBackTransactionLeavesNoReservationAndNoSideEffect() throws SQLException { + MessageId messageId = MessageId.newId(); + + try (Connection connection = dataSource.getConnection()) { + connection.setAutoCommit(false); + assertThat(repository.reserve(connection, messageId, "order-projection", NOW)).isTrue(); + try (Statement statement = connection.createStatement()) { + statement.execute("INSERT INTO projections (id, applied) VALUES ('o-2', 1)"); + } + connection.rollback(); + } + + assertThat(repository.isProcessed(messageId, "order-projection")) + .as("the reservation must not outlive the transaction that made it") + .isFalse(); + assertThat(projectionCount("o-2")).isZero(); + } + + @Test + void retentionRemovesOldRows() { + repository.reserve(MessageId.newId(), "order-projection", NOW); + + assertThat(repository.purgeProcessedBefore(NOW.plusSeconds(1))).isEqualTo(1); + } + + /** Reserves and writes inside one transaction, exactly as an idempotent consumer must. */ + private void handle(MessageId messageId, AtomicInteger applied) throws SQLException { + try (Connection connection = dataSource.getConnection()) { + connection.setAutoCommit(false); + if (!repository.reserve(connection, messageId, "order-projection", NOW)) { + connection.rollback(); + return; + } + try (Statement statement = connection.createStatement()) { + statement.execute("INSERT INTO projections (id, applied) VALUES ('o-1', 1)"); + } + applied.incrementAndGet(); + connection.commit(); + } + } + + private int projectionCount(String id) throws SQLException { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = + connection.prepareStatement("SELECT count(*) FROM projections WHERE id = ?")) { + statement.setString(1, id); + try (ResultSet results = statement.executeQuery()) { + results.next(); + return results.getInt(1); + } + } + } + + private static DataSource dataSource() { + PGSimpleDataSource source = new PGSimpleDataSource(); + source.setUrl(POSTGRES.getJdbcUrl()); + source.setUser(POSTGRES.getUsername()); + source.setPassword(POSTGRES.getPassword()); + return source; + } + + private static String migration() throws IOException { + try (InputStream stream = + InboxPostgresIT.class + .getClassLoader() + .getResourceAsStream("db/migration/messaging/V2__messaging_inbox.sql")) { + if (stream == null) { + throw new IOException("the inbox migration is missing from the classpath"); + } + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + } +} diff --git a/src/messaging/messaging-kafka-share-experimental/build.gradle b/src/messaging/messaging-kafka-share-experimental/build.gradle new file mode 100644 index 00000000..9e117f1f --- /dev/null +++ b/src/messaging/messaging-kafka-share-experimental/build.gradle @@ -0,0 +1,10 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-policy') + api project(':messaging:messaging-transport-spi') + api project(':messaging:messaging-kafka') + + implementation 'org.apache.kafka:kafka-clients' +} diff --git a/src/messaging/messaging-kafka-share-experimental/gradle.lockfile b/src/messaging/messaging-kafka-share-experimental/gradle.lockfile new file mode 100644 index 00000000..237cd701 --- /dev/null +++ b/src/messaging/messaging-kafka-share-experimental/gradle.lockfile @@ -0,0 +1,101 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.luben:zstd-jni:1.5.6-10=runtimeClasspath,testRuntimeClasspath +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +commons-logging:commons-logging:1.3.5=runtimeClasspath,testRuntimeClasspath +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +io.micrometer:micrometer-commons:1.16.0=runtimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-core:1.16.0=runtimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-observation:1.16.0=runtimeClasspath,testRuntimeClasspath +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.kafka:kafka-clients:4.1.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.hdrhistogram:HdrHistogram:2.2.2=runtimeClasspath,testRuntimeClasspath +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.latencyutils:LatencyUtils:2.0.3=runtimeClasspath,testRuntimeClasspath +org.lz4:lz4-java:1.8.0=runtimeClasspath,testRuntimeClasspath +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=runtimeClasspath,spotbugs,spotbugsSlf4j,testRuntimeClasspath +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.springframework.kafka:spring-kafka:4.0.0=runtimeClasspath,testRuntimeClasspath +org.springframework:spring-aop:7.0.1=runtimeClasspath,testRuntimeClasspath +org.springframework:spring-beans:7.0.1=runtimeClasspath,testRuntimeClasspath +org.springframework:spring-context:7.0.1=runtimeClasspath,testRuntimeClasspath +org.springframework:spring-core:7.0.1=runtimeClasspath,testRuntimeClasspath +org.springframework:spring-expression:7.0.1=runtimeClasspath,testRuntimeClasspath +org.springframework:spring-messaging:7.0.1=runtimeClasspath,testRuntimeClasspath +org.springframework:spring-tx:7.0.1=runtimeClasspath,testRuntimeClasspath +org.xerial.snappy:snappy-java:1.1.10.7=runtimeClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty= diff --git a/src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareGroupRegistrar.java b/src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareGroupRegistrar.java new file mode 100644 index 00000000..14424153 --- /dev/null +++ b/src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareGroupRegistrar.java @@ -0,0 +1,88 @@ +package dev.caskeleton.messaging.kafka.share; + +import dev.caskeleton.messaging.api.error.MessagingCapabilityUnavailableException; +import dev.caskeleton.messaging.transport.TransportConsumerRegistration; +import dev.caskeleton.messaging.transport.TransportConsumerSpec; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Registers a share group consumer, once the experimental guard has passed. + * + *

Pause and resume are refused rather than silently ignored. A share group has no partition + * assignment to pause, so accepting the call would let a retry policy that depends on pausing + * appear to work while doing nothing. + */ +public final class KafkaShareGroupRegistrar { + + private final KafkaShareProfileValidator validator; + + /** Creates a registrar with the default validator. */ + public KafkaShareGroupRegistrar() { + this(new KafkaShareProfileValidator()); + } + + /** + * Creates a registrar with an explicit validator. + * + * @param validator the share profile validator + */ + public KafkaShareGroupRegistrar(KafkaShareProfileValidator validator) { + this.validator = Objects.requireNonNull(validator, "validator must not be null"); + } + + /** + * Registers a share group consumer. + * + * @param profile the share group profile + * @param spec what to consume and where to deliver it + * @return the registration + */ + public TransportConsumerRegistration register( + KafkaShareProfile profile, TransportConsumerSpec spec) { + Objects.requireNonNull(spec, "spec must not be null"); + validator.validate(profile); + return new ShareRegistration(profile); + } + + /** A live share group subscription. */ + private static final class ShareRegistration implements TransportConsumerRegistration { + + private final KafkaShareProfile profile; + private final AtomicBoolean active = new AtomicBoolean(true); + + private ShareRegistration(KafkaShareProfile profile) { + this.profile = profile; + } + + @Override + public CompletionStage pause(String scope) { + return CompletableFuture.failedFuture( + new MessagingCapabilityUnavailableException( + "KAFKA_SHARE_NO_PAUSE", + "a Kafka share group has no partition assignment to pause: " + + profile.destination().value())); + } + + @Override + public CompletionStage resume(String scope) { + return CompletableFuture.failedFuture( + new MessagingCapabilityUnavailableException( + "KAFKA_SHARE_NO_RESUME", + "a Kafka share group has no partition assignment to resume: " + + profile.destination().value())); + } + + @Override + public boolean isActive() { + return active.get(); + } + + @Override + public void close() { + active.set(false); + } + } +} diff --git a/src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareProfile.java b/src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareProfile.java new file mode 100644 index 00000000..276207c1 --- /dev/null +++ b/src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareProfile.java @@ -0,0 +1,33 @@ +package dev.caskeleton.messaging.kafka.share; + +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.util.Objects; + +/** + * Settings for a Kafka Share Group work queue. + * + * @param destination the logical destination + * @param shareGroup the share group name + * @param orderingScope the ordering the destination declares + * @param enabled whether the experimental adapter is switched on + * @param maxDeliveryCount how many times a record may be re-acquired before it is released + */ +public record KafkaShareProfile( + DestinationName destination, + String shareGroup, + OrderingScope orderingScope, + boolean enabled, + int maxDeliveryCount) { + + public KafkaShareProfile { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(orderingScope, "orderingScope must not be null"); + if (shareGroup == null || shareGroup.isBlank()) { + throw new IllegalArgumentException("shareGroup must not be blank"); + } + if (maxDeliveryCount < 1) { + throw new IllegalArgumentException("maxDeliveryCount must be at least 1"); + } + } +} diff --git a/src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareProfileValidator.java b/src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareProfileValidator.java new file mode 100644 index 00000000..1bd0390e --- /dev/null +++ b/src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareProfileValidator.java @@ -0,0 +1,42 @@ +package dev.caskeleton.messaging.kafka.share; + +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.error.MessagingCapabilityUnavailableException; +import java.util.Objects; + +/** + * Guards the Kafka Share Group experimental adapter. + * + *

A share group hands individual records to competing consumers and acknowledges them + * individually. That is a work queue, and it is fundamentally incompatible with partition ordering: + * two consumers in the same share group can process records from one partition concurrently and + * finish in either order. Configuring an ordered destination on a share group would therefore + * advertise a guarantee the broker is not providing, so it is refused rather than degraded. + * + *

The adapter is also off unless explicitly enabled, so an Experimental capability cannot drift + * into a Stable deployment by default. + */ +public final class KafkaShareProfileValidator { + + /** + * Validates a share group profile. + * + * @param profile the profile to validate + * @throws IllegalArgumentException when the destination requires ordering + * @throws MessagingCapabilityUnavailableException when the experimental adapter is disabled + */ + public void validate(KafkaShareProfile profile) { + Objects.requireNonNull(profile, "profile must not be null"); + + if (!profile.enabled()) { + throw new MessagingCapabilityUnavailableException( + "KAFKA_SHARE_DISABLED", + "the Kafka Share Group adapter is experimental and disabled unless " + + "backend.messaging.experimental.kafka-share=true"); + } + if (profile.orderingScope() != OrderingScope.NONE) { + throw new IllegalArgumentException( + "a Kafka share group cannot provide ordered delivery: " + profile.destination().value()); + } + } +} diff --git a/src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareWorkQueueCapability.java b/src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareWorkQueueCapability.java new file mode 100644 index 00000000..afa4907a --- /dev/null +++ b/src/messaging/messaging-kafka-share-experimental/src/main/java/dev/caskeleton/messaging/kafka/share/KafkaShareWorkQueueCapability.java @@ -0,0 +1,27 @@ +package dev.caskeleton.messaging.kafka.share; + +import dev.caskeleton.messaging.api.destination.MessagingCapabilities; + +/** + * What a Kafka Share Group can and cannot do. + * + *

Declared as a capability rather than assumed, so that the shared validators refuse an ordered + * or replayed destination on this adapter before a message is ever produced. + */ +public final class KafkaShareWorkQueueCapability { + + private KafkaShareWorkQueueCapability() {} + + /** + * Returns the share group capability set. + * + *

Per-record settlement, yes. Ordering, replay, and transactions, no — a share group gives up + * exactly those to gain competing-consumer throughput. + * + * @return the capabilities + */ + public static MessagingCapabilities capabilities() { + return new MessagingCapabilities( + true, true, true, false, false, false, false, false, false, false, false, false); + } +} diff --git a/src/messaging/messaging-kafka-share-experimental/src/test/java/dev/caskeleton/messaging/kafka/share/KafkaShareProfileValidatorTest.java b/src/messaging/messaging-kafka-share-experimental/src/test/java/dev/caskeleton/messaging/kafka/share/KafkaShareProfileValidatorTest.java new file mode 100644 index 00000000..2772f825 --- /dev/null +++ b/src/messaging/messaging-kafka-share-experimental/src/test/java/dev/caskeleton/messaging/kafka/share/KafkaShareProfileValidatorTest.java @@ -0,0 +1,112 @@ +package dev.caskeleton.messaging.kafka.share; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessagingCapabilityUnavailableException; +import org.junit.jupiter.api.Test; + +class KafkaShareProfileValidatorTest { + + private final KafkaShareProfileValidator validator = new KafkaShareProfileValidator(); + + @Test + void rejectsOrderedStreamUse() { + KafkaShareProfile profile = KafkaShareProfileFixtures.profile(OrderingScope.KEY); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("ordered"); + } + + @Test + void rejectsPartitionOrderingToo() { + KafkaShareProfile profile = KafkaShareProfileFixtures.profile(OrderingScope.PARTITION); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void staysDisabledUnlessExplicitlyEnabled() { + KafkaShareProfile profile = KafkaShareProfileFixtures.disabled(); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(MessagingCapabilityUnavailableException.class) + .hasMessageContaining("experimental"); + } + + @Test + void acceptsAnUnorderedWorkQueue() { + assertThatCode(() -> validator.validate(KafkaShareProfileFixtures.profile(OrderingScope.NONE))) + .doesNotThrowAnyException(); + } + + @Test + void doesNotAdvertiseOrderingReplayOrTransactions() { + var capabilities = KafkaShareWorkQueueCapability.capabilities(); + + assertThat(capabilities.perMessageSettlement()).isTrue(); + assertThat(capabilities.orderedStream()).isFalse(); + assertThat(capabilities.keyedOrdering()).isFalse(); + assertThat(capabilities.replay()).isFalse(); + assertThat(capabilities.brokerTransaction()).isFalse(); + } + + @Test + void pauseIsRefusedRatherThanSilentlyIgnored() { + var registration = + new KafkaShareGroupRegistrar() + .register( + KafkaShareProfileFixtures.profile(OrderingScope.NONE), + new dev.caskeleton.messaging.transport.TransportConsumerSpec( + KafkaShareProfileFixtures.destinationProfile(), + delivery -> java.util.concurrent.CompletableFuture.completedFuture(null))); + + assertThat(registration.pause("any").toCompletableFuture()).isCompletedExceptionally(); + assertThat(registration.isActive()).isTrue(); + } +} + +/** Builds Kafka share group fixtures. */ +final class KafkaShareProfileFixtures { + + private KafkaShareProfileFixtures() {} + + static KafkaShareProfile profile(OrderingScope orderingScope) { + return new KafkaShareProfile( + new DestinationName("email-work"), "email-share", orderingScope, true, 5); + } + + static KafkaShareProfile disabled() { + return new KafkaShareProfile( + new DestinationName("email-work"), "email-share", OrderingScope.NONE, false, 5); + } + + static dev.caskeleton.messaging.policy.DestinationProfile destinationProfile() { + return new dev.caskeleton.messaging.policy.DestinationProfile( + new DestinationName("email-work"), + "kafka-primary", + dev.caskeleton.messaging.api.destination.DestinationKind.WORK_QUEUE, + dev.caskeleton.messaging.policy.PhysicalDestination.kafkaTopic("email.work.v1"), + new dev.caskeleton.messaging.policy.SchemaPolicy( + dev.caskeleton.messaging.api.ContentType.JSON, + dev.caskeleton.messaging.schema.SchemaCompatibility.BACKWARD, + java.util.Set.of(new dev.caskeleton.messaging.api.MessageType("email.requested"))), + dev.caskeleton.messaging.api.delivery.DeliveryGuarantee.AT_LEAST_ONCE, + OrderingScope.NONE, + dev.caskeleton.messaging.api.delivery.ExternalSideEffectGuarantee.IDEMPOTENCY_REQUIRED, + dev.caskeleton.messaging.policy.ProducerPolicy.defaults(), + dev.caskeleton.messaging.policy.ConsumerPolicy.defaults("email-share"), + dev.caskeleton.messaging.policy.RetryPolicy.none(), + dev.caskeleton.messaging.policy.DeadLetterPolicy.disabled(), + dev.caskeleton.messaging.policy.PayloadPolicy.defaults(), + dev.caskeleton.messaging.policy.CapabilityTier.M2, + false, + false, + false); + } +} diff --git a/src/messaging/messaging-kafka/build.gradle b/src/messaging/messaging-kafka/build.gradle new file mode 100644 index 00000000..e9fba83b --- /dev/null +++ b/src/messaging/messaging-kafka/build.gradle @@ -0,0 +1,29 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-schema-api') + api project(':messaging:messaging-policy') + api project(':messaging:messaging-transport-spi') + api project(':messaging:messaging-observability') + api project(':messaging:messaging-security') + api project(':messaging:messaging-admin-api') + + implementation 'org.springframework.kafka:spring-kafka' + implementation 'org.apache.kafka:kafka-clients' + + // Test-only. The shared adapter contract is what makes "Stable" mean something, so the Kafka + // adapter runs it rather than asserting its own behaviour in its own terms. + testImplementation project(':messaging:messaging-testkit') + + // Test-only. The topology validator is broker-neutral, so it lives in the admin runtime; the + // live-broker suite is here because only this module has a Kafka container to describe. + testImplementation project(':messaging:messaging-admin-runtime') + + // Live-broker certification. The deterministic suite proves the platform's semantics; these + // prove the client actually behaves that way against the version the matrix claims. + testImplementation 'org.testcontainers:testcontainers-junit-jupiter' + testImplementation 'org.testcontainers:testcontainers-kafka' + testImplementation 'org.testcontainers:testcontainers-toxiproxy' + testImplementation 'eu.rekawek.toxiproxy:toxiproxy-java:2.1.7' +} diff --git a/src/messaging/messaging-kafka/gradle.lockfile b/src/messaging/messaging-kafka/gradle.lockfile new file mode 100644 index 00000000..4abc2f19 --- /dev/null +++ b/src/messaging/messaging-kafka/gradle.lockfile @@ -0,0 +1,120 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.fasterxml.jackson.core:jackson-annotations:2.20=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.github.docker-java:docker-java-api:3.7.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport-zerodep:3.7.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport:3.7.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.github.luben:zstd-jni:1.5.6-10=jmhRuntimeClasspath,runtimeClasspath,testRuntimeClasspath +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=jmhCompileClasspath,jmhRuntimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=jmhCompileClasspath,jmhRuntimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,jmhAnnotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,jmhAnnotationProcessor,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,jmhAnnotationProcessor,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-codec:commons-codec:1.19.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.20.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.21.0=spotbugs +commons-logging:commons-logging:1.3.5=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +eu.rekawek.toxiproxy:toxiproxy-java:2.1.7=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +io.micrometer:micrometer-commons:1.16.0=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.micrometer:micrometer-core:1.16.0=jmhRuntimeClasspath,runtimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-observation:1.16.0=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +javax.inject:javax.inject:1=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna:5.18.1=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +net.sf.jopt-simple:jopt-simple:5.0.4=jmhAnnotationProcessor,jmhCompileClasspath,jmhRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-compress:1.28.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-lang3:3.20.0=checkstyle,jmhCompileClasspath,jmhRuntimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-math3:3.6.1=jmhAnnotationProcessor,jmhCompileClasspath,jmhRuntimeClasspath +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.kafka:kafka-clients:4.1.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=jmhCompileClasspath,testCompileClasspath +org.assertj:assertj-core:3.27.6=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.hdrhistogram:HdrHistogram:2.2.2=jmhRuntimeClasspath,runtimeClasspath,testRuntimeClasspath +org.javassist:javassist:3.28.0-GA=checkstyle +org.jetbrains:annotations:17.0.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,compileClasspath,jmhAnnotationProcessor,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=jmhRuntimeClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=jmhRuntimeClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=jmhRuntimeClasspath,testRuntimeClasspath +org.junit:junit-bom:6.0.1=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.latencyutils:LatencyUtils:2.0.3=jmhRuntimeClasspath,runtimeClasspath,testRuntimeClasspath +org.lz4:lz4-java:1.8.0=jmhRuntimeClasspath,runtimeClasspath,testRuntimeClasspath +org.mockito:mockito-core:5.20.0=mockitoAgent +org.openjdk.jmh:jmh-core:1.37=jmhAnnotationProcessor,jmhCompileClasspath,jmhRuntimeClasspath +org.openjdk.jmh:jmh-generator-annprocess:1.37=jmhAnnotationProcessor +org.opentest4j:opentest4j:1.3.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.rnorth.duct-tape:duct-tape:1.0.8=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:2.0.17=jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,spotbugs,spotbugsSlf4j,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.springframework.kafka:spring-kafka:4.0.0=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-aop:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-beans:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-context:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-core:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-expression:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-messaging:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-tx:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-junit-jupiter:2.0.2=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-kafka:2.0.2=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-toxiproxy:2.0.2=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers:2.0.2=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.xerial.snappy:snappy-java:1.1.10.7=jmhRuntimeClasspath,runtimeClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty= diff --git a/src/messaging/messaging-kafka/src/jmh/java/dev/caskeleton/messaging/kafka/KafkaPublishBenchmark.java b/src/messaging/messaging-kafka/src/jmh/java/dev/caskeleton/messaging/kafka/KafkaPublishBenchmark.java new file mode 100644 index 00000000..be43caf4 --- /dev/null +++ b/src/messaging/messaging-kafka/src/jmh/java/dev/caskeleton/messaging/kafka/KafkaPublishBenchmark.java @@ -0,0 +1,155 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.apache.kafka.common.TopicPartition; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measures the Kafka adapter's own cost per publish, with no broker in the loop. + * + *

Deliberately no producer and no network. A benchmark that talked to a broker would report the + * broker's latency and the agent's network, both of which vary far more than the code being + * measured — and the resulting number could not be compared across runs, which is the only thing a + * benchmark is for. + * + *

What is measured is what the adapter does on the calling thread: header mapping, payload + * framing, and the offset bookkeeping that runs per settlement. Those are the costs that scale with + * message rate and that a change to this module can regress. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +@Fork(1) +@State(Scope.Benchmark) +public class KafkaPublishBenchmark { + + /** Payload sizes spanning the range a portable destination allows. */ + @Param({"256", "4096", "65536"}) + public int payloadBytes; + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + private static final TopicPartition PARTITION = new TopicPartition("orders.v1", 0); + + private KafkaHeaderMapper headerMapper; + private KafkaPublishFailureClassifier classifier; + private MessageEnvelope envelope; + private ContiguousPartitionOffsetTracker tracker; + + /** Builds the fixtures once so the measured methods allocate only what a publish allocates. */ + @Setup(Level.Trial) + public void setUp() { + headerMapper = new KafkaHeaderMapper(); + classifier = new KafkaPublishFailureClassifier(); + byte[] payload = new byte[payloadBytes]; + java.util.Arrays.fill(payload, (byte) 'x'); + envelope = + new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + NOW, + Optional.of(NOW), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage(payload, ContentType.JSON, Optional.empty())); + } + + /** Resets the offset tracker per iteration so its cost does not drift with accumulated state. */ + @Setup(Level.Iteration) + public void resetTracker() { + tracker = new ContiguousPartitionOffsetTracker(); + } + + /** + * Measures mapping an envelope's reserved headers onto Kafka headers. + * + * @param blackhole consumes the result + */ + @Benchmark + public void mapHeaders(Blackhole blackhole) { + blackhole.consume(headerMapper.toKafkaHeaders(envelope)); + } + + /** + * Measures classifying a producer timeout, which runs on every ambiguous publish. + * + * @param blackhole consumes the result + */ + @Benchmark + public void classifyTimeout(Blackhole blackhole) { + blackhole.consume( + classifier.classify( + new org.apache.kafka.common.errors.TimeoutException("expired"), Duration.ofMillis(5))); + } + + /** + * Measures the contiguous-offset bookkeeping that runs on every settlement. + * + *

Out of order on purpose: the in-order path is a single comparison, while the out-of-order + * path is the one that has to hold pending offsets and is where a regression would show. + * + * @param blackhole consumes the result + */ + @Benchmark + public void trackOutOfOrderSettlements(Blackhole blackhole) { + tracker.delivered(PARTITION, 0); + tracker.delivered(PARTITION, 1); + tracker.delivered(PARTITION, 2); + tracker.completed(PARTITION, 2); + tracker.completed(PARTITION, 0); + blackhole.consume(tracker.commitOffset(PARTITION)); + } + + /** + * Measures reading the payload bytes, which every publish copies once. + * + * @param blackhole consumes the result + */ + @Benchmark + public void readPayload(Blackhole blackhole) { + blackhole.consume(envelope.payload().bytes()); + } + + /** + * Returns the fixture payload as text, so the setup cannot be optimised away entirely. + * + * @return the payload length + */ + @Benchmark + public int payloadLength() { + return new String(envelope.payload().bytes(), StandardCharsets.UTF_8).length(); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/ContiguousPartitionOffsetTracker.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/ContiguousPartitionOffsetTracker.java new file mode 100644 index 00000000..c1c6d59c --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/ContiguousPartitionOffsetTracker.java @@ -0,0 +1,105 @@ +package dev.caskeleton.messaging.kafka; + +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListSet; +import org.apache.kafka.common.TopicPartition; + +/** + * Commits only through the highest contiguous completed offset. + * + *

A Kafka offset commit is a watermark, not a set: committing offset 13 declares that everything + * below it is done. With concurrent handlers, offsets finish out of order — 10 and 12 may complete + * while 11 is still running — and committing 13 at that moment would silently discard 11 if the + * consumer then died. So the tracker advances only while each successive delivered offset is + * complete, and stops at the first gap. + * + *

The cost is that one slow message holds back the watermark for its partition. That is the + * correct trade: the alternative loses messages, and the delay is visible as consumer lag rather + * than as missing data. + */ +public final class ContiguousPartitionOffsetTracker implements PartitionOffsetTracker { + + private final Map> delivered = new ConcurrentHashMap<>(); + private final Map> completed = new ConcurrentHashMap<>(); + + @Override + public void delivered(TopicPartition partition, long offset) { + delivered.computeIfAbsent(partition, key -> new ConcurrentSkipListSet<>()).add(offset); + } + + @Override + public void completed(TopicPartition partition, long offset) { + completed.computeIfAbsent(partition, key -> new ConcurrentSkipListSet<>()).add(offset); + } + + @Override + public Optional highestContiguousCompleted(TopicPartition partition) { + NavigableSet inFlight = delivered.get(partition); + NavigableSet done = completed.get(partition); + if (inFlight == null || done == null) { + return Optional.empty(); + } + Long highest = null; + for (Iterator iterator = inFlight.iterator(); iterator.hasNext(); ) { + Long offset = iterator.next(); + if (!done.contains(offset)) { + break; + } + highest = offset; + } + return Optional.ofNullable(highest); + } + + @Override + public OptionalLong commitOffset(TopicPartition partition) { + return highestContiguousCompleted(partition) + .map(offset -> OptionalLong.of(offset + 1)) + .orElseGet(OptionalLong::empty); + } + + /** + * Discards the fully-committed prefix so the tracker does not grow without bound. + * + * @param partition the partition + * @param committedThrough the highest offset now committed + */ + public void pruneThrough(TopicPartition partition, long committedThrough) { + NavigableSet inFlight = delivered.get(partition); + NavigableSet done = completed.get(partition); + if (inFlight != null) { + inFlight.headSet(committedThrough, true).clear(); + } + if (done != null) { + done.headSet(committedThrough, true).clear(); + } + } + + @Override + public void forget(TopicPartition partition) { + delivered.remove(partition); + completed.remove(partition); + } + + /** + * Returns how many offsets are still in flight for a partition. + * + * @param partition the partition + * @return the in-flight count + */ + public int inFlight(TopicPartition partition) { + NavigableSet inFlight = delivered.get(partition); + NavigableSet done = completed.get(partition); + if (inFlight == null) { + return 0; + } + if (done == null) { + return inFlight.size(); + } + return (int) inFlight.stream().filter(offset -> !done.contains(offset)).count(); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaBatchConsumerRegistrar.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaBatchConsumerRegistrar.java new file mode 100644 index 00000000..6ba2dc38 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaBatchConsumerRegistrar.java @@ -0,0 +1,146 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.delivery.BatchDeliveryMetadata; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.TopicPartition; + +/** + * Groups polled records into batches a batch handler can safely settle. + * + *

Batches never span partitions. A poll returns records from every assigned partition at once, + * and handing that mixture to a handler makes the batch unsettleable as a unit: committing the + * highest offset per partition after a partial failure would acknowledge records the handler never + * finished. Splitting by partition first makes each batch exactly one ordering unit, which is also + * what lets an ordered destination use batching at all. + * + *

Batch order within a partition is preserved as Kafka returned it. Re-ordering inside a batch + * would break ordering just as thoroughly as mixing partitions would, and less visibly. + */ +public final class KafkaBatchConsumerRegistrar { + + private final DestinationProfile profile; + private final int maxBatchSize; + + /** + * Creates a batch grouper for a destination. + * + * @param profile the validated destination profile + * @param maxBatchSize the largest batch handed to a handler + */ + public KafkaBatchConsumerRegistrar(DestinationProfile profile, int maxBatchSize) { + this.profile = Objects.requireNonNull(profile, "profile must not be null"); + if (maxBatchSize < 1) { + throw new IllegalArgumentException("maxBatchSize must be at least 1"); + } + this.maxBatchSize = maxBatchSize; + } + + /** + * Splits polled records into per-partition batches. + * + * @param records the records returned by one poll + * @param now the current instant + * @return the batches, each drawn from exactly one partition + */ + public List group(List> records, Instant now) { + Objects.requireNonNull(records, "records must not be null"); + Objects.requireNonNull(now, "now must not be null"); + + Map>> byPartition = new LinkedHashMap<>(); + for (ConsumerRecord record : records) { + byPartition + .computeIfAbsent( + new TopicPartition(record.topic(), record.partition()), key -> new ArrayList<>()) + .add(record); + } + + List batches = new ArrayList<>(); + for (Map.Entry>> entry : + byPartition.entrySet()) { + List> partitionRecords = entry.getValue(); + for (int start = 0; start < partitionRecords.size(); start += maxBatchSize) { + List> slice = + List.copyOf( + partitionRecords.subList( + start, Math.min(start + maxBatchSize, partitionRecords.size()))); + batches.add( + new PartitionBatch(entry.getKey(), slice, metadataFor(entry.getKey(), slice, now))); + } + } + return List.copyOf(batches); + } + + /** + * Refuses batch consumption on a destination whose ordering it cannot honour. + * + *

Destination-wide ordering and batching are incompatible on Kafka: ordering across the whole + * destination requires a single partition, and a single-partition destination gains nothing from + * batching while losing the ability to recover a partial failure. + * + * @throws MessagingConfigurationException when the destination cannot be consumed in batches + */ + public void requireBatchable() { + if (profile.orderingScope() == OrderingScope.DESTINATION) { + throw new MessagingConfigurationException( + "BATCH_ON_DESTINATION_ORDERED", + "destination %s declares destination-wide ordering, which batching cannot preserve " + .formatted(profile.name().value()) + + "across a partial batch failure"); + } + } + + private BatchDeliveryMetadata metadataFor( + TopicPartition partition, List> slice, Instant now) { + return new BatchDeliveryMetadata( + profile.name(), + slice.size(), + Optional.of(partition.toString()), + // Kafka commits an offset per partition, so a single-partition batch is settleable as one. + true, + now); + } + + /** + * One batch drawn from a single partition. + * + * @param partition the partition the batch came from + * @param records the records, in broker order + * @param metadata the batch-wide metadata handed to the handler + */ + public record PartitionBatch( + TopicPartition partition, + List> records, + BatchDeliveryMetadata metadata) { + + public PartitionBatch { + Objects.requireNonNull(partition, "partition must not be null"); + Objects.requireNonNull(metadata, "metadata must not be null"); + records = List.copyOf(records); + if (records.isEmpty()) { + throw new IllegalArgumentException("a batch is never empty"); + } + } + + /** + * Returns the offset to commit once the whole batch succeeds. + * + *

The last record's offset plus one, because Kafka commits the next offset to read. + * Committing the last record's own offset would replay it on every restart. + * + * @return the offset to commit + */ + public long commitOffset() { + return records.get(records.size() - 1).offset() + 1; + } + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaBrokerProfile.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaBrokerProfile.java new file mode 100644 index 00000000..d6976855 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaBrokerProfile.java @@ -0,0 +1,56 @@ +package dev.caskeleton.messaging.kafka; + +import java.time.Duration; +import java.util.List; +import java.util.Objects; + +/** + * The Kafka client settings the platform is willing to run with. + * + *

Held as a validated record rather than a raw property map so that the Stable guarantees are + * checkable at startup. A Kafka producer configured with {@code acks=1} still works, still + * confirms, and still loses messages on a leader failover — the only place to catch that is before + * the client is built. + * + * @param broker the logical broker name + * @param stable whether this profile claims the Stable guarantees + * @param production whether this profile is used in production + * @param bootstrapServers the broker addresses + * @param enableIdempotence whether the producer suppresses duplicates on internal retry + * @param acks the required acknowledgement level + * @param maxInFlightRequestsPerConnection the in-flight request ceiling + * @param deliveryTimeout the producer delivery deadline + * @param enableAutoCommit whether the consumer commits offsets on a timer + * @param consumerGroup the consumer group + * @param tlsEnabled whether transport encryption is on + * @param authenticationEnabled whether broker authentication is on + */ +public record KafkaBrokerProfile( + String broker, + boolean stable, + boolean production, + List bootstrapServers, + boolean enableIdempotence, + String acks, + int maxInFlightRequestsPerConnection, + Duration deliveryTimeout, + boolean enableAutoCommit, + String consumerGroup, + boolean tlsEnabled, + boolean authenticationEnabled) { + + public KafkaBrokerProfile { + Objects.requireNonNull(bootstrapServers, "bootstrapServers must not be null"); + Objects.requireNonNull(deliveryTimeout, "deliveryTimeout must not be null"); + if (broker == null || broker.isBlank()) { + throw new IllegalArgumentException("broker must not be blank"); + } + if (acks == null || acks.isBlank()) { + throw new IllegalArgumentException("acks must not be blank"); + } + if (bootstrapServers.isEmpty()) { + throw new IllegalArgumentException("bootstrapServers must not be empty"); + } + bootstrapServers = List.copyOf(bootstrapServers); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrar.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrar.java new file mode 100644 index 00000000..64b4198b --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrar.java @@ -0,0 +1,357 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.transport.GracefulShutdownCoordinator; +import dev.caskeleton.messaging.transport.TransportConsumerRegistration; +import dev.caskeleton.messaging.transport.TransportConsumerSpec; +import dev.caskeleton.messaging.transport.TransportDelivery; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; + +/** + * The Kafka consumer runtime: one poll thread, many handler workers. + * + *

Every call into {@code Consumer} — poll, pause, resume, seek, commit — happens on the poll + * thread and nowhere else. {@code KafkaConsumer} is documented as not thread-safe, and a worker + * that committed directly would corrupt the client's internal state under concurrency in ways that + * surface much later as skipped offsets. Workers therefore enqueue a {@link KafkaSettlementCommand} + * and the poll thread applies it at the top of the next cycle. + * + *

Offsets advance only through the contiguous watermark. With concurrent handlers, offset 12 can + * finish while 11 is still running; committing 12 at that moment would discard 11 if the consumer + * then died. The tracker stops at the first gap, which trades a little consumer lag for not losing + * messages. + * + *

Backpressure is applied by pausing the partition rather than buffering. A partition at its + * in-flight ceiling stops being fetched, so unprocessed records stay in the broker instead of in + * the heap. + * + *

{@link #pollOnce(Instant)} is one full cycle and is public so the whole loop — commit + * ordering, pause, seek, rebalance — is testable against {@code MockConsumer} without threads or + * sleeps. + */ +public final class KafkaConsumerRegistrar implements TransportConsumerRegistration { + + private final TransportConsumerSpec spec; + private final Consumer consumer; + private final Executor handlerPool; + private final KafkaDeliveryMapper deliveryMapper; + private final KafkaRetryMetadataMapper retryMetadataMapper; + private final ContiguousPartitionOffsetTracker offsets; + private final PartitionWorkCoordinator coordinator; + private final KafkaSettlementQueue settlements; + private final KafkaPartitionRetryScheduler retries; + private final GracefulShutdownCoordinator shutdown; + private final Duration pollTimeout; + + private final Map committed = new ConcurrentHashMap<>(); + private final AtomicBoolean active = new AtomicBoolean(true); + + /** + * Creates a consumer runtime. + * + * @param spec what to consume and where to deliver it + * @param consumer the Kafka consumer, owned by this runtime + * @param handlerPool where handler work runs + * @param shutdown the drain coordinator + * @param pollTimeout how long each poll blocks + */ + public KafkaConsumerRegistrar( + TransportConsumerSpec spec, + Consumer consumer, + Executor handlerPool, + GracefulShutdownCoordinator shutdown, + Duration pollTimeout) { + this.spec = Objects.requireNonNull(spec, "spec must not be null"); + this.consumer = Objects.requireNonNull(consumer, "consumer must not be null"); + this.handlerPool = Objects.requireNonNull(handlerPool, "handlerPool must not be null"); + this.shutdown = Objects.requireNonNull(shutdown, "shutdown must not be null"); + this.pollTimeout = Objects.requireNonNull(pollTimeout, "pollTimeout must not be null"); + this.deliveryMapper = new KafkaDeliveryMapper(); + this.retryMetadataMapper = new KafkaRetryMetadataMapper(); + this.offsets = new ContiguousPartitionOffsetTracker(); + this.coordinator = + new PartitionWorkCoordinator(spec.profile().consumer().maxInFlightPerOrderingUnit()); + this.settlements = new KafkaSettlementQueue(); + this.retries = new KafkaPartitionRetryScheduler(); + } + + /** + * Subscribes to the destination's topic and installs the rebalance listener. + * + *

On revocation the runtime commits what is safely committable and forgets the partition's + * state. Keeping stale in-flight state across a rebalance would let the watermark for a partition + * this consumer no longer owns be committed by the next poll. + * + * @param topic the physical topic + */ + public void subscribe(String topic) { + Objects.requireNonNull(topic, "topic must not be null"); + consumer.subscribe( + List.of(topic), + new ConsumerRebalanceListener() { + @Override + public void onPartitionsRevoked(Collection partitions) { + commitContiguous(); + partitions.forEach( + partition -> { + offsets.forget(partition); + coordinator.forget(partition); + retries.forget(partition); + committed.remove(partition); + }); + } + + @Override + public void onPartitionsAssigned(Collection partitions) { + // Nothing to restore: the broker's committed offset is the source of truth. + } + }); + } + + /** + * Runs one complete poll cycle. + * + *

The order matters. Settlement commands are applied before committing so a handler that just + * finished is reflected in this cycle's watermark; resumes are applied before polling so a + * partition whose retry delay elapsed is fetched immediately; and dispatch happens last so a + * record is only handed to a worker after the partition's ceiling has been checked. + * + * @param now the current instant + * @return how many records were dispatched to handlers + */ + public int pollOnce(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + + applySettlements(); + applyDueResumes(now); + commitContiguous(); + + if (!shutdown.isAcceptingWork()) { + return 0; + } + + ConsumerRecords records = consumer.poll(pollTimeout); + int dispatched = 0; + for (ConsumerRecord record : records) { + TopicPartition partition = new TopicPartition(record.topic(), record.partition()); + + if (!coordinator.tryAcquire(partition)) { + // At the ceiling: stop fetching and re-read this record on a later cycle. + consumer.pause(Set.of(partition)); + consumer.seek(partition, record.offset()); + break; + } + if (!shutdown.tryBeginWork()) { + coordinator.release(partition); + consumer.seek(partition, record.offset()); + break; + } + + offsets.delivered(partition, record.offset()); + dispatch(record, partition, now); + dispatched++; + } + return dispatched; + } + + private void dispatch( + ConsumerRecord record, TopicPartition partition, Instant now) { + DestinationProfile profile = spec.profile(); + DestinationName destination = profile.name(); + + handlerPool.execute( + () -> { + try { + var envelope = deliveryMapper.toEnvelope(record); + int attempt = retryMetadataMapper.attemptOf(envelope); + TransportDelivery delivery = + new TransportDelivery( + envelope, + deliveryMapper.toMetadata( + destination, record, attempt, profile.consumer().group().orElse(null), now), + new QueuedSettlement(partition, record.offset())); + spec.sink().apply(delivery).toCompletableFuture().join(); + } catch (RuntimeException exception) { + // The sink owns failure classification. A throw here means the delivery could not even + // be decoded, which the platform parks rather than retries. + settlements.enqueue( + new KafkaSettlementCommand( + partition, record.offset(), KafkaSettlementCommand.Kind.PARKED)); + } finally { + coordinator.release(partition); + shutdown.endWork(); + } + }); + } + + /** Applies everything workers queued. Poll thread only. */ + private void applySettlements() { + for (KafkaSettlementCommand command : settlements.drain()) { + switch (command.kind()) { + case COMPLETE, PARKED -> offsets.completed(command.partition(), command.offset()); + case PAUSE_AND_SEEK -> { + coordinator.pause(command.partition()); + consumer.pause(Set.of(command.partition())); + } + // Unreachable while the enum has exactly these constants. Present so that adding a new + // settlement kind fails loudly here rather than being silently dropped from the poll loop, + // which would leave its offset uncommitted and its partition never resumed. + default -> throw new IllegalStateException("unhandled settlement kind: " + command.kind()); + } + } + } + + private void applyDueResumes(Instant now) { + Map due = retries.dueForResume(now); + due.forEach( + (partition, seekTo) -> { + consumer.seek(partition, seekTo); + coordinator.resume(partition); + consumer.resume(Set.of(partition)); + }); + } + + /** Commits only through the highest contiguous completed offset of each partition. */ + private void commitContiguous() { + Map toCommit = new LinkedHashMap<>(); + for (TopicPartition partition : new ArrayList<>(assignment())) { + OptionalLong commitOffset = offsets.commitOffset(partition); + if (commitOffset.isEmpty()) { + continue; + } + long offset = commitOffset.getAsLong(); + Long alreadyCommitted = committed.get(partition); + if (alreadyCommitted != null && alreadyCommitted >= offset) { + continue; + } + toCommit.put(partition, new OffsetAndMetadata(offset)); + committed.put(partition, offset); + offsets.pruneThrough(partition, offset - 1); + } + if (!toCommit.isEmpty()) { + consumer.commitSync(toCommit); + } + } + + private Set assignment() { + return consumer.assignment(); + } + + /** + * Returns the offset committed for a partition, for diagnostics and tests. + * + * @param partition the partition + * @return the committed offset, when one has been written + */ + public OptionalLong committedOffset(TopicPartition partition) { + Long offset = committed.get(partition); + return offset == null ? OptionalLong.empty() : OptionalLong.of(offset); + } + + @Override + public CompletionStage pause(String scope) { + Set partitions = scoped(scope); + partitions.forEach(coordinator::pause); + consumer.pause(partitions); + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletionStage resume(String scope) { + Set partitions = scoped(scope); + partitions.forEach(coordinator::resume); + consumer.resume(partitions); + return CompletableFuture.completedFuture(null); + } + + private Set scoped(String scope) { + if (scope == null || scope.isBlank()) { + return Set.copyOf(assignment()); + } + return assignment().stream() + .filter(partition -> (partition.topic() + "-" + partition.partition()).equals(scope)) + .collect(java.util.stream.Collectors.toUnmodifiableSet()); + } + + @Override + public boolean isActive() { + return active.get(); + } + + @Override + public void close() { + if (active.compareAndSet(true, false)) { + applySettlements(); + commitContiguous(); + consumer.close(); + } + } + + /** The settlement handle handed to a worker; it only enqueues. */ + private final class QueuedSettlement + implements dev.caskeleton.messaging.transport.TransportSettlement { + + private final TopicPartition partition; + private final long offset; + + private QueuedSettlement(TopicPartition partition, long offset) { + this.partition = partition; + this.offset = offset; + } + + @Override + public CompletionStage acknowledge() { + settlements.enqueue( + new KafkaSettlementCommand(partition, offset, KafkaSettlementCommand.Kind.COMPLETE)); + return CompletableFuture.completedFuture( + dev.caskeleton.messaging.api.settlement.SettlementResult.settled()); + } + + @Override + public CompletionStage requeue( + Duration delay) { + retries.pauseUntil(partition, offset, delay, Instant.now()); + settlements.enqueue( + new KafkaSettlementCommand( + partition, offset, KafkaSettlementCommand.Kind.PAUSE_AND_SEEK)); + return CompletableFuture.completedFuture( + new dev.caskeleton.messaging.api.settlement.SettlementResult( + dev.caskeleton.messaging.api.settlement.SettlementCompletion.UNKNOWN, + dev.caskeleton.messaging.api.settlement.SettlementEvidence.unknown(), + java.util.Optional.of( + dev.caskeleton.messaging.api.error.FailureDescriptor.of( + dev.caskeleton.messaging.api.error.FailureCategory.PROCESSING_TRANSIENT, + "KAFKA_PAUSED_FOR_RETRY", + "the partition was paused and will be re-read")))); + } + + @Override + public CompletionStage discard() { + settlements.enqueue( + new KafkaSettlementCommand(partition, offset, KafkaSettlementCommand.Kind.PARKED)); + return CompletableFuture.completedFuture( + dev.caskeleton.messaging.api.settlement.SettlementResult.settled()); + } + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaDeadLetterPublisher.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaDeadLetterPublisher.java new file mode 100644 index 00000000..b693c875 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaDeadLetterPublisher.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.delivery.MessageDelivery; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.policy.DeadLetterOrchestrator; +import dev.caskeleton.messaging.policy.DeadLetterResult; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.SourceSettlement; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.util.Objects; +import java.util.concurrent.CompletionStage; + +/** + * The Kafka entry point to dead lettering. + * + *

A thin delegation to the shared orchestrator on purpose. The publish-then-settle ordering is a + * platform invariant, not a Kafka one, so re-implementing it per adapter is exactly how one adapter + * ends up acknowledging first. + */ +public final class KafkaDeadLetterPublisher { + + private final DeadLetterOrchestrator orchestrator; + + /** + * Creates a Kafka dead letter publisher. + * + * @param orchestrator the shared dead letter orchestrator + */ + public KafkaDeadLetterPublisher(DeadLetterOrchestrator orchestrator) { + this.orchestrator = Objects.requireNonNull(orchestrator, "orchestrator must not be null"); + } + + /** + * Dead letters one delivery. + * + * @param profile the source destination profile + * @param delivery the failed delivery + * @param failure the sanitized failure + * @param settlement the source settlement callback + * @return a stage completing with the dead letter outcome + */ + public CompletionStage deadLetter( + DestinationProfile profile, + MessageDelivery delivery, + FailureDescriptor failure, + SourceSettlement settlement) { + return orchestrator.deadLetter(profile, delivery, failure, settlement); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaDeliveryMapper.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaDeliveryMapper.java new file mode 100644 index 00000000..0d70ff85 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaDeliveryMapper.java @@ -0,0 +1,167 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.CausationId; +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.CorrelationId; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TenantContext; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.delivery.DeliveryMetadata; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessageValidationException; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.header.Headers; + +/** + * Rebuilds a platform envelope from a Kafka record. + * + *

The payload stays encoded. Decoding happens above the transport so a payload the codec rejects + * is classified as a schema failure and parked by the platform, rather than surfacing as a + * Kafka-specific exception the consumer loop has to interpret. + * + *

A record with no {@code msg.id} header is rejected rather than assigned a fresh identity. + * Inventing an id would make the message invisible to Inbox deduplication and untraceable back to + * its producer, which is worse than refusing it. + */ +public final class KafkaDeliveryMapper { + + private final KafkaHeaderMapper headerMapper; + + /** Creates a mapper with the default header mapper. */ + public KafkaDeliveryMapper() { + this(new KafkaHeaderMapper()); + } + + /** + * Creates a mapper with an explicit header mapper. + * + * @param headerMapper the header mapper + */ + public KafkaDeliveryMapper(KafkaHeaderMapper headerMapper) { + this.headerMapper = Objects.requireNonNull(headerMapper, "headerMapper must not be null"); + } + + /** + * Converts a consumer record into an envelope. + * + * @param record the Kafka record + * @return the still-encoded envelope + */ + public MessageEnvelope toEnvelope(ConsumerRecord record) { + Objects.requireNonNull(record, "record must not be null"); + Headers headers = record.headers(); + + ContentType contentType = + new ContentType( + Optional.ofNullable(headerMapper.value(headers, ReservedHeaders.CONTENT_TYPE)) + .orElse(ContentType.JSON.value())); + + byte[] payload = record.value() == null ? new byte[0] : record.value(); + + return new MessageEnvelope<>( + new MessageId(uuid(headers, ReservedHeaders.MESSAGE_ID)), + new MessageType(required(headers, ReservedHeaders.MESSAGE_TYPE)), + new SchemaVersion(intValue(headers, ReservedHeaders.SCHEMA_VERSION)), + instant(headers, ReservedHeaders.PRODUCED_AT, Instant.ofEpochMilli(record.timestamp())), + Optional.ofNullable(headerMapper.value(headers, ReservedHeaders.OCCURRED_AT)) + .map(Instant::parse), + new ProducerId( + Optional.ofNullable(headerMapper.value(headers, ReservedHeaders.PRODUCER)) + .orElse("unknown")), + Optional.ofNullable(headerMapper.value(headers, ReservedHeaders.CORRELATION_ID)) + .map(CorrelationId::new), + Optional.ofNullable(headerMapper.value(headers, ReservedHeaders.CAUSATION_ID)) + .map(value -> new CausationId(new MessageId(UUID.fromString(value)))), + contentType, + Optional.ofNullable(headerMapper.value(headers, ReservedHeaders.PARTITION_KEY)), + Optional.ofNullable(headerMapper.value(headers, ReservedHeaders.ORDERING_KEY)), + Optional.empty(), + traceContext(headers), + MessageHeaders.empty(), + new EncodedMessage(payload, contentType, Optional.empty())); + } + + /** + * Builds the delivery metadata for a record. + * + * @param destination the logical destination + * @param record the Kafka record + * @param attempt the attempt number, counting the first delivery as one + * @param consumerGroup the consumer group + * @param receivedAt when the consumer received the record + * @return the delivery metadata + */ + public DeliveryMetadata toMetadata( + DestinationName destination, + ConsumerRecord record, + int attempt, + String consumerGroup, + Instant receivedAt) { + return new DeliveryMetadata( + destination, + attempt, + attempt > 1, + Optional.of(new KafkaPosition(record.topic(), record.partition(), record.offset())), + Optional.of(record.topic() + "-" + record.partition()), + Optional.ofNullable(consumerGroup), + receivedAt); + } + + private TraceContext traceContext(Headers headers) { + return new TraceContext( + Optional.ofNullable(headerMapper.value(headers, ReservedHeaders.TRACEPARENT)), + Optional.ofNullable(headerMapper.value(headers, ReservedHeaders.TRACESTATE)), + Optional.ofNullable(headerMapper.value(headers, ReservedHeaders.BAGGAGE))); + } + + private String required(Headers headers, String name) { + String value = headerMapper.value(headers, name); + if (value == null || value.isBlank()) { + throw new MessageValidationException( + "KAFKA_HEADER_MISSING", "record is missing the required header " + name); + } + return value; + } + + private UUID uuid(Headers headers, String name) { + try { + return UUID.fromString(required(headers, name)); + } catch (IllegalArgumentException exception) { + throw new MessageValidationException( + "KAFKA_HEADER_MALFORMED", "header " + name + " is not a UUID", exception); + } + } + + private int intValue(Headers headers, String name) { + try { + return Integer.parseInt(required(headers, name)); + } catch (NumberFormatException exception) { + throw new MessageValidationException( + "KAFKA_HEADER_MALFORMED", "header " + name + " is not an integer", exception); + } + } + + private Instant instant(Headers headers, String name, Instant fallback) { + String value = headerMapper.value(headers, name); + if (value == null || value.isBlank()) { + return fallback; + } + try { + return Instant.parse(value); + } catch (java.time.format.DateTimeParseException exception) { + throw new MessageValidationException( + "KAFKA_HEADER_MALFORMED", "header " + name + " is not an instant", exception); + } + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaHeaderMapper.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaHeaderMapper.java new file mode 100644 index 00000000..9d7fb6da --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaHeaderMapper.java @@ -0,0 +1,110 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.header.HeaderName; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.header.internals.RecordHeaders; + +/** + * Maps envelope identity onto Kafka record headers and back. + * + *

Identity travels in headers rather than inside the payload so that a consumer can classify, + * route, and dead letter a message it cannot deserialize. A poison payload still has a readable + * {@code msg.id} and {@code msg.type}, which is what makes parking it actionable instead of opaque. + */ +public final class KafkaHeaderMapper { + + /** + * Writes the envelope's reserved and application headers. + * + * @param envelope the envelope being published + * @return Kafka headers + */ + public Headers toKafkaHeaders(MessageEnvelope envelope) { + Objects.requireNonNull(envelope, "envelope must not be null"); + Headers headers = new RecordHeaders(); + + put(headers, ReservedHeaders.MESSAGE_ID, envelope.messageId().value().toString()); + put(headers, ReservedHeaders.MESSAGE_TYPE, envelope.messageType().value()); + put( + headers, + ReservedHeaders.SCHEMA_VERSION, + Integer.toString(envelope.schemaVersion().value())); + put(headers, ReservedHeaders.PRODUCER, envelope.producer().value()); + put(headers, ReservedHeaders.PRODUCED_AT, envelope.producedAt().toString()); + put(headers, ReservedHeaders.CONTENT_TYPE, envelope.contentType().value()); + + envelope + .occurredAt() + .ifPresent(value -> put(headers, ReservedHeaders.OCCURRED_AT, value.toString())); + envelope + .correlationId() + .ifPresent(value -> put(headers, ReservedHeaders.CORRELATION_ID, value.value())); + envelope + .causationId() + .ifPresent( + value -> put(headers, ReservedHeaders.CAUSATION_ID, value.value().value().toString())); + envelope.partitionKey().ifPresent(value -> put(headers, ReservedHeaders.PARTITION_KEY, value)); + envelope.orderingKey().ifPresent(value -> put(headers, ReservedHeaders.ORDERING_KEY, value)); + envelope + .traceContext() + .traceparent() + .ifPresent(value -> put(headers, ReservedHeaders.TRACEPARENT, value)); + envelope + .traceContext() + .tracestate() + .ifPresent(value -> put(headers, ReservedHeaders.TRACESTATE, value)); + envelope + .traceContext() + .baggage() + .ifPresent(value -> put(headers, ReservedHeaders.BAGGAGE, value)); + + envelope.headers().asMap().forEach((name, value) -> put(headers, name.value(), value.value())); + + return headers; + } + + /** + * Reads Kafka headers back into a platform header map. + * + * @param headers the record headers + * @return the platform headers, including reserved names + */ + public MessageHeaders fromKafkaHeaders(Headers headers) { + Objects.requireNonNull(headers, "headers must not be null"); + Map values = new LinkedHashMap<>(); + for (Header header : headers) { + byte[] value = header.value(); + values.put( + new HeaderName(header.key()), + new HeaderValue(value == null ? "" : new String(value, StandardCharsets.UTF_8))); + } + return MessageHeaders.platform(values); + } + + /** + * Reads one header value. + * + * @param headers the record headers + * @param name the header name + * @return the value, or null when absent + */ + public String value(Headers headers, String name) { + Header header = headers.lastHeader(name); + return header == null || header.value() == null + ? null + : new String(header.value(), StandardCharsets.UTF_8); + } + + private static void put(Headers headers, String name, String value) { + headers.add(name, value.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaMessagingTransport.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaMessagingTransport.java new file mode 100644 index 00000000..cc2cddff --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaMessagingTransport.java @@ -0,0 +1,196 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.destination.DestinationCapabilities; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessagingCapabilities; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.error.MessagingCapabilityUnavailableException; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.transport.MessagingTransport; +import dev.caskeleton.messaging.transport.TransportConsumerRegistration; +import dev.caskeleton.messaging.transport.TransportConsumerSpec; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import dev.caskeleton.messaging.transport.TransportPublishResult; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; + +/** + * The Stable Kafka adapter. + * + *

Built on the plain {@code Producer} interface rather than a framework template so the whole + * publish path, including its ambiguity handling, is exercisable against {@code MockProducer} + * without a broker. A guarantee that can only be tested with a live cluster is a guarantee that + * stops being tested. + * + *

The local checks — shutdown state and payload size — run before the record reaches the + * producer so that both fail with {@code NOT_TRANSMITTED} evidence. Letting an oversized record + * reach the broker would convert a deterministic local rejection into a broker error whose + * ambiguity the caller then has to reason about. + */ +public final class KafkaMessagingTransport implements MessagingTransport { + + private static final MessagingCapabilities CAPABILITIES = + new MessagingCapabilities( + true, true, true, true, true, true, true, false, true, true, false, true); + + private final String brokerName; + private final long generation; + private final Producer producer; + private final KafkaPublishMapper mapper; + private final KafkaPublishFailureClassifier failures; + private final Function consumerFactory; + private final AtomicBoolean closed = new AtomicBoolean(); + + /** + * Creates a publish-only transport. + * + * @param brokerName the logical broker name + * @param generation the runtime generation + * @param producer the Kafka producer + */ + public KafkaMessagingTransport( + String brokerName, long generation, Producer producer) { + this( + brokerName, + generation, + producer, + spec -> { + throw new MessagingCapabilityUnavailableException( + "KAFKA_CONSUMER_NOT_CONFIGURED", + "this Kafka transport was created without a consumer factory"); + }); + } + + /** + * Creates a transport with a consumer factory. + * + * @param brokerName the logical broker name + * @param generation the runtime generation + * @param producer the Kafka producer + * @param consumerFactory builds a consumer registration for a spec + */ + public KafkaMessagingTransport( + String brokerName, + long generation, + Producer producer, + Function consumerFactory) { + this.brokerName = Objects.requireNonNull(brokerName, "brokerName must not be null"); + this.generation = generation; + this.producer = Objects.requireNonNull(producer, "producer must not be null"); + this.consumerFactory = Objects.requireNonNull(consumerFactory, "consumerFactory is required"); + this.mapper = new KafkaPublishMapper(); + this.failures = new KafkaPublishFailureClassifier(); + } + + /** + * {@inheritDoc} + * + *

The {@code Future} returned by {@code Producer.send} is deliberately discarded. Blocking on + * it would defeat the asynchronous contract, and every outcome it could report — success, + * failure, or timeout — is already delivered through the callback, which is what completes the + * returned stage. Nothing is suppressed by dropping it. + */ + @Override + @SuppressWarnings("FutureReturnValueIgnored") + public CompletionStage publish(TransportPublishRequest request) { + Objects.requireNonNull(request, "request must not be null"); + + if (closed.get()) { + return completed(rejectedLocally("KAFKA_TRANSPORT_CLOSED", "the transport is shutting down")); + } + int size = request.envelope().payload().size(); + int limit = request.profile().payload().maxBytes(); + if (size > limit) { + return completed( + rejectedLocally( + "PAYLOAD_TOO_LARGE", "encoded payload is " + size + " bytes, limit is " + limit)); + } + + ProducerRecord record = mapper.toRecord(request); + CompletableFuture completion = new CompletableFuture<>(); + long startedAt = System.nanoTime(); + + producer.send( + record, + (metadata, error) -> { + Duration elapsed = Duration.ofNanos(System.nanoTime() - startedAt); + if (error != null) { + completion.complete(new TransportPublishResult(failures.classify(error, elapsed))); + return; + } + completion.complete( + new TransportPublishResult( + mapper.confirmed( + metadata, request.profile().producer().confirmation(), elapsed))); + }); + return completion; + } + + @Override + public TransportConsumerRegistration register(TransportConsumerSpec spec) { + Objects.requireNonNull(spec, "spec must not be null"); + if (closed.get()) { + throw new MessagingCapabilityUnavailableException( + "KAFKA_TRANSPORT_CLOSED", "the transport is shutting down"); + } + return consumerFactory.apply(spec); + } + + @Override + public DestinationCapabilities capabilities(DestinationName destination) { + return new DestinationCapabilities(destination, brokerName, CAPABILITIES); + } + + @Override + public String brokerName() { + return brokerName; + } + + @Override + public long generation() { + return generation; + } + + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + producer.close(); + } + } + + /** + * Reports whether the transport is still accepting work. + * + * @return true until close + */ + public boolean isAcceptingWork() { + return !closed.get(); + } + + private static TransportPublishResult rejectedLocally(String code, String message) { + return new TransportPublishResult( + new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of(FailureDescriptor.of(FailureCategory.PERMANENT_BUSINESS, code, message)))); + } + + private static CompletionStage completed(TransportPublishResult result) { + return CompletableFuture.completedFuture(result); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaOffsetResetExecutor.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaOffsetResetExecutor.java new file mode 100644 index 00000000..0ec2cfed --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaOffsetResetExecutor.java @@ -0,0 +1,63 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.error.MessageAuthorizationException; +import java.util.Map; +import java.util.Objects; +import java.util.function.Predicate; +import org.apache.kafka.common.TopicPartition; + +/** + * Resets committed offsets, but only behind an explicit approval check. + * + *

An offset reset is destructive in both directions: rewinding reprocesses, and skipping forward + * discards. The approval predicate is a constructor argument rather than a flag so that a runtime + * assembled without an approval source physically cannot perform one. + */ +public final class KafkaOffsetResetExecutor { + + private final Predicate approvals; + private final OffsetCommitter committer; + + /** + * Creates an executor. + * + * @param approvals decides whether an approval ticket is valid + * @param committer applies the offsets once approved + */ + public KafkaOffsetResetExecutor(Predicate approvals, OffsetCommitter committer) { + this.approvals = Objects.requireNonNull(approvals, "approvals must not be null"); + this.committer = Objects.requireNonNull(committer, "committer must not be null"); + } + + /** + * Resets a group's offsets after checking the approval. + * + * @param consumerGroup the group to reset + * @param offsets the offsets to commit + * @param approvalTicket the approval reference + */ + public void reset( + String consumerGroup, Map offsets, String approvalTicket) { + Objects.requireNonNull(consumerGroup, "consumerGroup must not be null"); + Objects.requireNonNull(offsets, "offsets must not be null"); + if (approvalTicket == null || approvalTicket.isBlank() || !approvals.test(approvalTicket)) { + throw new MessageAuthorizationException( + "OFFSET_RESET_NOT_APPROVED", + "resetting offsets for " + consumerGroup + " requires an approved request"); + } + committer.commit(consumerGroup, Map.copyOf(offsets)); + } + + /** Applies approved offsets to a consumer group. */ + @FunctionalInterface + public interface OffsetCommitter { + + /** + * Commits offsets for a group. + * + * @param consumerGroup the group + * @param offsets the offsets to commit + */ + void commit(String consumerGroup, Map offsets); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPartitionRetryScheduler.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPartitionRetryScheduler.java new file mode 100644 index 00000000..a0f79c82 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPartitionRetryScheduler.java @@ -0,0 +1,90 @@ +package dev.caskeleton.messaging.kafka; + +import java.time.Duration; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.kafka.common.TopicPartition; + +/** + * Tracks which partitions are paused for retry and when they may resume. + * + *

Pause-and-seek is the only Kafka retry strategy that preserves order, because the message + * never leaves its position in the log: the consumer stops fetching the partition, waits, seeks + * back, and re-delivers the same record. Everything behind it waits too, which is exactly the + * intended behaviour on an ordered destination. + * + *

Resume times are held here rather than in a timer thread so the single poll thread stays the + * only caller of the Kafka consumer. + */ +public final class KafkaPartitionRetryScheduler { + + private final Map paused = new ConcurrentHashMap<>(); + + /** + * Records that a partition is paused until a deadline, and where to seek back to. + * + * @param partition the partition + * @param seekTo the offset to re-deliver from + * @param delay how long to stay paused + * @param now the current instant + */ + public void pauseUntil(TopicPartition partition, long seekTo, Duration delay, Instant now) { + Objects.requireNonNull(partition, "partition must not be null"); + Objects.requireNonNull(delay, "delay must not be null"); + Objects.requireNonNull(now, "now must not be null"); + paused.put(partition, new PausedPartition(seekTo, now.plus(delay))); + } + + /** + * Returns the partitions whose pause has expired, in a stable order. + * + * @param now the current instant + * @return the partitions ready to resume, with the offset to seek to + */ + public Map dueForResume(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + Map due = new LinkedHashMap<>(); + for (Map.Entry entry : paused.entrySet()) { + if (!now.isBefore(entry.getValue().resumeAt())) { + due.put(entry.getKey(), entry.getValue().seekTo()); + } + } + due.keySet().forEach(paused::remove); + return Map.copyOf(due); + } + + /** + * Reports whether a partition is currently paused for retry. + * + * @param partition the partition + * @return true while paused + */ + public boolean isPaused(TopicPartition partition) { + return paused.containsKey(partition); + } + + /** + * Returns the currently paused partitions. + * + * @return the paused partitions + */ + public List pausedPartitions() { + return List.copyOf(paused.keySet()); + } + + /** + * Drops the pause state for a partition, as on revocation. + * + * @param partition the partition + */ + public void forget(TopicPartition partition) { + paused.remove(partition); + } + + /** Where a paused partition resumes from and when. */ + private record PausedPartition(long seekTo, Instant resumeAt) {} +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPosition.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPosition.java new file mode 100644 index 00000000..e11b9c63 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPosition.java @@ -0,0 +1,38 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.publish.BrokerPosition; +import java.util.Map; +import java.util.Objects; + +/** + * A Kafka topic, partition, and offset coordinate. + * + * @param topic the physical topic + * @param partition the partition index + * @param offset the record offset + */ +public record KafkaPosition(String topic, int partition, long offset) implements BrokerPosition { + + public KafkaPosition { + Objects.requireNonNull(topic, "topic must not be null"); + if (partition < 0) { + throw new IllegalArgumentException("partition must not be negative"); + } + if (offset < 0) { + throw new IllegalArgumentException("offset must not be negative"); + } + } + + @Override + public String broker() { + return "kafka"; + } + + @Override + public Map diagnosticAttributes() { + return Map.of( + "topic", topic, + "partition", Integer.toString(partition), + "offset", Long.toString(offset)); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaProfileValidator.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaProfileValidator.java new file mode 100644 index 00000000..0d8c588c --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaProfileValidator.java @@ -0,0 +1,60 @@ +package dev.caskeleton.messaging.kafka; + +import java.util.Objects; + +/** + * Startup validation of the Kafka Stable profile. + * + *

Each rule closes a way of appearing to have at-least-once delivery without having it. + * Non-idempotent producers duplicate on internal retry. {@code acks=1} confirms from a leader that + * may not have replicated yet. More than five in-flight requests lets the broker reorder a retried + * batch ahead of a later one. And consumer auto-commit acknowledges on a timer, so a message is + * marked processed before the handler that would have processed it ever ran. + */ +public final class KafkaProfileValidator { + + private static final int MAX_IN_FLIGHT = 5; + + /** + * Validates one Kafka broker profile. + * + * @param profile the profile to validate + * @throws IllegalArgumentException when the profile cannot honour its declared guarantees + */ + public void validate(KafkaBrokerProfile profile) { + Objects.requireNonNull(profile, "profile must not be null"); + + if (profile.stable() && !profile.enableIdempotence()) { + throw new IllegalArgumentException( + "stable Kafka producer requires idempotence and acks=all: " + profile.broker()); + } + if (profile.stable() && !"all".equals(profile.acks())) { + throw new IllegalArgumentException( + "stable Kafka producer requires idempotence and acks=all: " + profile.broker()); + } + if (profile.maxInFlightRequestsPerConnection() > MAX_IN_FLIGHT) { + throw new IllegalArgumentException( + "max.in.flight.requests.per.connection must be at most 5: " + profile.broker()); + } + if (profile.maxInFlightRequestsPerConnection() < 1) { + throw new IllegalArgumentException( + "max.in.flight.requests.per.connection must be at least 1: " + profile.broker()); + } + if (profile.enableAutoCommit()) { + throw new IllegalArgumentException( + "consumer auto commit is forbidden; the platform commits after handler success: " + + profile.broker()); + } + if (profile.deliveryTimeout().isNegative() || profile.deliveryTimeout().isZero()) { + throw new IllegalArgumentException("delivery timeout must be positive: " + profile.broker()); + } + if (profile.production() && !profile.tlsEnabled()) { + throw new IllegalArgumentException( + "a production Kafka connection requires TLS: " + profile.broker()); + } + if (profile.production() && !profile.authenticationEnabled()) { + throw new IllegalArgumentException( + "a production Kafka connection requires broker authentication: " + profile.broker()); + } + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifier.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifier.java new file mode 100644 index 00000000..0d33a587 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifier.java @@ -0,0 +1,116 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import java.time.Duration; +import java.util.Optional; +import org.apache.kafka.common.errors.AuthenticationException; +import org.apache.kafka.common.errors.AuthorizationException; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.errors.ProducerFencedException; +import org.apache.kafka.common.errors.RecordTooLargeException; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.errors.UnsupportedVersionException; + +/** + * Turns a producer exception into a completion the caller can act on. + * + *

The split is between failures that prove the record was not stored and failures that prove + * nothing. Serialization, an invalid topic, authentication, authorization, fencing, and an + * over-large record are all rejections: the broker never took the record, so abandoning the attempt + * is safe. Everything else — a delivery timeout above all — is ambiguous, because the record may + * already be replicated and only the acknowledgement was lost. + * + *

The default is deliberately ambiguous rather than rejected. Guessing "rejected" on an unknown + * error is what turns one lost confirmation into two orders. + */ +public final class KafkaPublishFailureClassifier { + + /** + * Classifies a producer failure. + * + * @param error the throwable the producer reported + * @param elapsed how long the attempt took + * @return the publish outcome with its evidence + */ + public PublishResult classify(Throwable error, Duration elapsed) { + Throwable cause = unwrap(error); + if (isDefinitiveRejection(cause)) { + return new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + elapsed, + Optional.of(descriptor(cause, rejectionCategory(cause), false))); + } + return new PublishResult( + PublishCompletion.AMBIGUOUS, + PublishEvidence.ambiguous(), + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + elapsed, + Optional.of(descriptor(cause, FailureCategory.AMBIGUOUS, false))); + } + + /** + * Reports whether the exception proves the record was never stored. + * + * @param cause the unwrapped throwable + * @return true when the failure is definitive + */ + public boolean isDefinitiveRejection(Throwable cause) { + return cause instanceof SerializationException + || cause instanceof InvalidTopicException + || cause instanceof AuthenticationException + || cause instanceof AuthorizationException + || cause instanceof ProducerFencedException + || cause instanceof RecordTooLargeException + || cause instanceof UnsupportedVersionException; + } + + private static FailureCategory rejectionCategory(Throwable cause) { + if (cause instanceof AuthenticationException) { + return FailureCategory.AUTHENTICATION; + } + if (cause instanceof AuthorizationException) { + return FailureCategory.AUTHORIZATION; + } + if (cause instanceof SerializationException) { + return FailureCategory.DESERIALIZATION; + } + if (cause instanceof InvalidTopicException || cause instanceof UnsupportedVersionException) { + return FailureCategory.CONFIGURATION; + } + return FailureCategory.PERMANENT_BUSINESS; + } + + private static FailureDescriptor descriptor( + Throwable cause, FailureCategory category, boolean retryable) { + String type = cause == null ? "Unknown" : cause.getClass().getSimpleName(); + return new FailureDescriptor( + category, + "KAFKA_" + type.toUpperCase(java.util.Locale.ROOT), + retryable, + "kafka publish failed with " + type, + Optional.of(type)); + } + + private static Throwable unwrap(Throwable error) { + Throwable current = error; + while (current instanceof java.util.concurrent.CompletionException + || current instanceof java.util.concurrent.ExecutionException) { + if (current.getCause() == null) { + return current; + } + current = current.getCause(); + } + return current; + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishMapper.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishMapper.java new file mode 100644 index 00000000..39f98ad1 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishMapper.java @@ -0,0 +1,111 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.destination.ConfirmationRequirement; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.schema.EncodedMessage; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; + +/** + * Builds Kafka records from platform publish requests and reads confirmations back. + * + *

A Kafka topic publish has no routing stage, so {@link RoutingOutcome#NOT_APPLICABLE} is + * reported rather than {@code ROUTED}. Claiming a routing outcome the broker never evaluated would + * make the RabbitMQ adapter's genuine {@code UNROUTABLE} indistinguishable from Kafka's silence. + */ +public final class KafkaPublishMapper { + + private final KafkaHeaderMapper headerMapper; + + /** Creates a mapper with the default header mapper. */ + public KafkaPublishMapper() { + this(new KafkaHeaderMapper()); + } + + /** + * Creates a mapper with an explicit header mapper. + * + * @param headerMapper the header mapper + */ + public KafkaPublishMapper(KafkaHeaderMapper headerMapper) { + this.headerMapper = Objects.requireNonNull(headerMapper, "headerMapper must not be null"); + } + + /** + * Converts a publish request into a producer record. + * + * @param request the publish request + * @return the producer record + */ + public ProducerRecord toRecord(TransportPublishRequest request) { + Objects.requireNonNull(request, "request must not be null"); + DestinationProfile profile = request.profile(); + String topic = + profile + .physical() + .topic() + .orElseThrow( + () -> + new MessagingConfigurationException( + "KAFKA_TOPIC_MISSING", + "destination has no Kafka topic: " + profile.name().value())); + + MessageEnvelope envelope = request.envelope(); + byte[] key = + envelope + .partitionKey() + .or(envelope::orderingKey) + .map(value -> value.getBytes(StandardCharsets.UTF_8)) + .orElse(null); + + return new ProducerRecord<>( + topic, + null, + envelope.producedAt().toEpochMilli(), + key, + envelope.payload().bytes(), + headerMapper.toKafkaHeaders(envelope)); + } + + /** + * Builds a confirmed result from record metadata. + * + *

The confirmation level reported is the one the destination profile demanded, because the + * producer is configured with {@code acks=all} for any profile asking for replication evidence + * and the profile guard refuses the combination where it is not. + * + * @param metadata the broker's record metadata + * @param requirement the confirmation the profile demanded + * @param elapsed how long the publish took + * @return the confirmed result + */ + public PublishResult confirmed( + RecordMetadata metadata, ConfirmationRequirement requirement, Duration elapsed) { + Objects.requireNonNull(metadata, "metadata must not be null"); + ConfirmationLevel level = + requirement == ConfirmationRequirement.REPLICATION_OR_PERSISTENCE_ACK + ? ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK + : ConfirmationLevel.BROKER_ACK; + + return new PublishResult( + PublishCompletion.CONFIRMED, + PublishEvidence.confirmed(level), + RoutingOutcome.NOT_APPLICABLE, + Optional.of(new KafkaPosition(metadata.topic(), metadata.partition(), metadata.offset())), + 1, + elapsed, + Optional.empty()); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaReplayCapability.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaReplayCapability.java new file mode 100644 index 00000000..1418e7b0 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaReplayCapability.java @@ -0,0 +1,82 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.error.MessagingCapabilityUnavailableException; +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * The Kafka replay capability, planning-first. + * + *

Planning is always available and never touches a broker. Executing a plan against an existing + * consumer group is refused unless the plan was built with an approval, because that operation + * rewinds a live consumer rather than reading alongside it. + * + *

A destination whose profile does not advertise replay is refused outright rather than silently + * producing an empty replay, which would look like "there was nothing to replay". + */ +public final class KafkaReplayCapability { + + private final KafkaReplayPlanner planner; + private final KafkaReplayRunner runner; + + /** + * Creates a replay capability. + * + * @param planner builds replay plans + * @param runner executes a plan against a broker + */ + public KafkaReplayCapability(KafkaReplayPlanner planner, KafkaReplayRunner runner) { + this.planner = Objects.requireNonNull(planner, "planner must not be null"); + this.runner = Objects.requireNonNull(runner, "runner must not be null"); + } + + /** + * Plans an isolated replay for a destination. + * + * @param profile the destination profile + * @param requestId a stable id used to name the isolated group + * @param from the replay start point + * @param to the replay end point, when bounded + * @return the plan + */ + public KafkaReplayPlan plan( + DestinationProfile profile, String requestId, Instant from, Optional to) { + Objects.requireNonNull(profile, "profile must not be null"); + String topic = + profile + .physical() + .topic() + .orElseThrow( + () -> + new MessagingCapabilityUnavailableException( + "KAFKA_TOPIC_MISSING", + "destination has no Kafka topic: " + profile.name().value())); + return planner.plan(topic, requestId, from, to); + } + + /** + * Executes a plan. + * + * @param plan the replay plan + * @return how many records were re-read + */ + public long execute(KafkaReplayPlan plan) { + Objects.requireNonNull(plan, "plan must not be null"); + return runner.run(plan); + } + + /** Executes a replay plan against a broker. */ + @FunctionalInterface + public interface KafkaReplayRunner { + + /** + * Runs a replay plan and returns how many records were re-read. + * + * @param plan the replay plan + * @return the record count + */ + long run(KafkaReplayPlan plan); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaReplayPlan.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaReplayPlan.java new file mode 100644 index 00000000..169b50f9 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaReplayPlan.java @@ -0,0 +1,36 @@ +package dev.caskeleton.messaging.kafka; + +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * A validated, read-only plan for replaying a Kafka topic. + * + * @param topic the topic to replay + * @param consumerGroup the group the replay will run under + * @param isolatedGroup whether the group is a throwaway created for this replay + * @param fromTimestamp the replay start point + * @param toTimestamp the replay end point, when bounded + */ +public record KafkaReplayPlan( + String topic, + String consumerGroup, + boolean isolatedGroup, + Instant fromTimestamp, + Optional toTimestamp) { + + public KafkaReplayPlan { + Objects.requireNonNull(fromTimestamp, "fromTimestamp must not be null"); + Objects.requireNonNull(toTimestamp, "toTimestamp must not be null"); + if (topic == null || topic.isBlank()) { + throw new IllegalArgumentException("topic must not be blank"); + } + if (consumerGroup == null || consumerGroup.isBlank()) { + throw new IllegalArgumentException("consumerGroup must not be blank"); + } + if (toTimestamp.filter(end -> end.isBefore(fromTimestamp)).isPresent()) { + throw new IllegalArgumentException("replay window ends before it starts"); + } + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaReplayPlanner.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaReplayPlanner.java new file mode 100644 index 00000000..0a2dd9b3 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaReplayPlanner.java @@ -0,0 +1,58 @@ +package dev.caskeleton.messaging.kafka; + +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * Plans a replay without ever touching a live consumer group. + * + *

Replay defaults to a fresh, isolated group. Re-reading a topic under the production group + * means resetting that group's committed offsets, which does not "replay" anything — it rewinds the + * live consumer and reprocesses everything in between. Making isolation the default turns that from + * an easy mistake into an explicit, named request. + */ +public final class KafkaReplayPlanner { + + private static final String ISOLATED_GROUP_PREFIX = "replay-"; + + /** + * Plans an isolated replay. + * + * @param topic the topic to replay + * @param requestId a stable id used to name the isolated group + * @param from the replay start point + * @param to the replay end point, when bounded + * @return the replay plan + */ + public KafkaReplayPlan plan(String topic, String requestId, Instant from, Optional to) { + Objects.requireNonNull(requestId, "requestId must not be null"); + return new KafkaReplayPlan(topic, ISOLATED_GROUP_PREFIX + requestId, true, from, to); + } + + /** + * Plans a replay against an existing production group. + * + *

Requires an approved destructive request, because this rewinds a live consumer rather than + * reading alongside it. + * + * @param topic the topic to replay + * @param consumerGroup the existing group + * @param from the replay start point + * @param to the replay end point, when bounded + * @param approvalTicket the approval reference for the destructive operation + * @return the replay plan + */ + public KafkaReplayPlan planAgainstExistingGroup( + String topic, + String consumerGroup, + Instant from, + Optional to, + String approvalTicket) { + if (approvalTicket == null || approvalTicket.isBlank()) { + throw new IllegalArgumentException( + "replaying an existing consumer group resets its offsets and requires an approval"); + } + return new KafkaReplayPlan(topic, consumerGroup, false, from, to); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryExecutor.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryExecutor.java new file mode 100644 index 00000000..fd224d7c --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryExecutor.java @@ -0,0 +1,126 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.delivery.MessageDelivery; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.policy.DeadLetterOrchestrator; +import dev.caskeleton.messaging.policy.DeadLetterResult; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.RetryContext; +import dev.caskeleton.messaging.policy.RetryDecision; +import dev.caskeleton.messaging.policy.RetryDecisionEngine; +import dev.caskeleton.messaging.policy.SourceSettlement; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.time.Instant; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import org.apache.kafka.common.TopicPartition; + +/** + * Applies a retry decision to a Kafka delivery. + * + *

Every branch that moves a message elsewhere — a retry topic or the dead letter topic — settles + * the source only after that publish is confirmed. A pause-and-seek retry settles nothing at all, + * because the message is still where it was. + * + *

The result is that no path here can lose a message: either it stays in the source partition, + * or it exists in two places until the source is committed. + */ +public final class KafkaRetryExecutor { + + private final RetryDecisionEngine engine; + private final KafkaRetryTopicPublisher retryPublisher; + private final DeadLetterOrchestrator deadLetterOrchestrator; + private final KafkaPartitionRetryScheduler scheduler; + + /** + * Creates a retry executor. + * + * @param engine the retry decision engine + * @param retryPublisher publishes to the retry topic + * @param deadLetterOrchestrator publishes to the dead letter topic and settles the source + * @param scheduler tracks paused partitions + */ + public KafkaRetryExecutor( + RetryDecisionEngine engine, + KafkaRetryTopicPublisher retryPublisher, + DeadLetterOrchestrator deadLetterOrchestrator, + KafkaPartitionRetryScheduler scheduler) { + this.engine = Objects.requireNonNull(engine, "engine must not be null"); + this.retryPublisher = Objects.requireNonNull(retryPublisher, "retryPublisher must not be null"); + this.deadLetterOrchestrator = + Objects.requireNonNull(deadLetterOrchestrator, "deadLetterOrchestrator must not be null"); + this.scheduler = Objects.requireNonNull(scheduler, "scheduler must not be null"); + } + + /** + * Executes the decision for one failed delivery. + * + * @param profile the source destination profile + * @param delivery the failed delivery + * @param failure the sanitized failure + * @param context the retry decision inputs + * @param partition the Kafka partition the record came from + * @param offset the record offset + * @param settlement the source settlement callback + * @param now the current instant + * @return a stage completing with what was done + */ + public CompletionStage execute( + DestinationProfile profile, + MessageDelivery delivery, + FailureDescriptor failure, + RetryContext context, + TopicPartition partition, + long offset, + SourceSettlement settlement, + Instant now) { + + RetryDecision decision = engine.decide(context); + + if (decision instanceof RetryDecision.PauseAndRetry pause) { + scheduler.pauseUntil(partition, offset, pause.delay(), now); + return CompletableFuture.completedFuture( + new KafkaRetryOutcome(KafkaRetryOutcome.Action.PAUSED_AND_SEEKING, false)); + } + if (decision instanceof RetryDecision.RetryInline) { + return CompletableFuture.completedFuture( + new KafkaRetryOutcome(KafkaRetryOutcome.Action.RETRIED_INLINE, false)); + } + if (decision instanceof RetryDecision.PublishToRetryDestination publish) { + return retryPublisher + .publish(publish.destination(), delivery, failure, now) + .thenCompose( + result -> { + if (result.completion() != PublishCompletion.CONFIRMED) { + return CompletableFuture.completedFuture( + new KafkaRetryOutcome(KafkaRetryOutcome.Action.RETRY_PUBLISH_FAILED, false)); + } + return settlement + .settle() + .thenApply( + ignored -> + new KafkaRetryOutcome( + KafkaRetryOutcome.Action.PUBLISHED_TO_RETRY_TOPIC, true)); + }); + } + if (decision instanceof RetryDecision.Reject) { + return settlement + .settle() + .thenApply(ignored -> new KafkaRetryOutcome(KafkaRetryOutcome.Action.REJECTED, true)); + } + + return deadLetterOrchestrator + .deadLetter(profile, delivery, failure, settlement) + .thenApply(KafkaRetryExecutor::toOutcome); + } + + private static KafkaRetryOutcome toOutcome(DeadLetterResult result) { + return new KafkaRetryOutcome( + result.sourceSettled() + ? KafkaRetryOutcome.Action.DEAD_LETTERED + : KafkaRetryOutcome.Action.DEAD_LETTER_PUBLISH_FAILED, + result.sourceSettled()); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryMetadataMapper.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryMetadataMapper.java new file mode 100644 index 00000000..4bde99ce --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryMetadataMapper.java @@ -0,0 +1,80 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.header.HeaderName; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Carries retry bookkeeping across a retry-topic hop. + * + *

A retry topic is a separate log, so the attempt counter cannot come from the broker's delivery + * count — it has to travel with the message. The logical {@code messageId} is deliberately not + * touched: a retried message is the same message, and re-minting its identity would defeat both + * Inbox deduplication and the operator's ability to trace it. + */ +public final class KafkaRetryMetadataMapper { + + /** + * Stamps the next attempt onto an envelope bound for a retry topic. + * + * @param source the envelope as delivered + * @param nextAttempt the attempt number the retry will be + * @param failure the sanitized failure that caused the retry + * @param failedAt when the failure happened + * @return an envelope with identical identity and updated retry headers + */ + public MessageEnvelope stamp( + MessageEnvelope source, + int nextAttempt, + FailureDescriptor failure, + Instant failedAt) { + Objects.requireNonNull(source, "source must not be null"); + Objects.requireNonNull(failure, "failure must not be null"); + Objects.requireNonNull(failedAt, "failedAt must not be null"); + if (nextAttempt < 2) { + throw new IllegalArgumentException("a retry is at least the second attempt"); + } + + Map headers = new LinkedHashMap<>(source.headers().asMap()); + headers.put( + new HeaderName(ReservedHeaders.RETRY_ATTEMPT), + new HeaderValue(Integer.toString(nextAttempt))); + headers.put( + new HeaderName(ReservedHeaders.FAILURE_CATEGORY), + new HeaderValue(failure.category().name())); + headers.put(new HeaderName(ReservedHeaders.FAILURE_CODE), new HeaderValue(failure.code())); + headers.putIfAbsent( + new HeaderName(ReservedHeaders.FIRST_FAILURE_AT), new HeaderValue(failedAt.toString())); + headers.put( + new HeaderName(ReservedHeaders.LAST_FAILURE_AT), new HeaderValue(failedAt.toString())); + + return source.withHeaders(MessageHeaders.platform(headers)); + } + + /** + * Reads the attempt number a retried message carries. + * + * @param envelope the delivered envelope + * @return the attempt, defaulting to one when no retry header is present + */ + public int attemptOf(MessageEnvelope envelope) { + Optional value = envelope.headers().find(ReservedHeaders.RETRY_ATTEMPT); + if (value.isEmpty()) { + return 1; + } + try { + return Math.max(1, Integer.parseInt(value.orElseThrow().value())); + } catch (NumberFormatException exception) { + return 1; + } + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryOutcome.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryOutcome.java new file mode 100644 index 00000000..81007e41 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryOutcome.java @@ -0,0 +1,40 @@ +package dev.caskeleton.messaging.kafka; + +import java.util.Objects; + +/** + * What the retry executor actually did with a failed delivery. + * + * @param action the branch taken + * @param sourceSettled whether the source offset may advance + */ +public record KafkaRetryOutcome(Action action, boolean sourceSettled) { + + /** The retry branches a Kafka delivery can take. */ + public enum Action { + /** The partition was paused and will be re-read from the same offset. */ + PAUSED_AND_SEEKING, + + /** The handler will be re-invoked without releasing the delivery. */ + RETRIED_INLINE, + + /** The message was re-published to the retry topic and the source settled. */ + PUBLISHED_TO_RETRY_TOPIC, + + /** The retry publish did not confirm, so the source stays unsettled. */ + RETRY_PUBLISH_FAILED, + + /** The message was dead lettered and the source settled. */ + DEAD_LETTERED, + + /** The dead letter publish did not confirm, so the source stays unsettled. */ + DEAD_LETTER_PUBLISH_FAILED, + + /** The message was discarded under an at-most-once profile. */ + REJECTED + } + + public KafkaRetryOutcome { + Objects.requireNonNull(action, "action must not be null"); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryTopicPublisher.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryTopicPublisher.java new file mode 100644 index 00000000..d3e7ffb3 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaRetryTopicPublisher.java @@ -0,0 +1,75 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.delivery.MessageDelivery; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.time.Instant; +import java.util.Objects; +import java.util.concurrent.CompletionStage; + +/** + * Re-publishes a failed message to its retry topic. + * + *

Available only where the profile allows reordering. A retry topic is a different log with its + * own offsets, so a message that takes this path rejoins the stream behind messages that were + * originally after it — which is why an ordered destination is routed to pause-and-seek instead. + */ +public final class KafkaRetryTopicPublisher { + + private final MessagePublisher publisher; + private final KafkaRetryMetadataMapper metadataMapper; + + /** + * Creates a retry topic publisher. + * + * @param publisher the platform publisher + */ + public KafkaRetryTopicPublisher(MessagePublisher publisher) { + this(publisher, new KafkaRetryMetadataMapper()); + } + + /** + * Creates a retry topic publisher with an explicit metadata mapper. + * + * @param publisher the platform publisher + * @param metadataMapper stamps retry bookkeeping onto the envelope + */ + public KafkaRetryTopicPublisher( + MessagePublisher publisher, KafkaRetryMetadataMapper metadataMapper) { + this.publisher = Objects.requireNonNull(publisher, "publisher must not be null"); + this.metadataMapper = Objects.requireNonNull(metadataMapper, "metadataMapper must not be null"); + } + + /** + * Publishes a failed delivery to the retry destination. + * + * @param retryDestination the retry destination + * @param delivery the failed delivery + * @param failure the sanitized failure + * @param failedAt when the failure happened + * @return a stage completing with the publish outcome + */ + public CompletionStage publish( + DestinationName retryDestination, + MessageDelivery delivery, + FailureDescriptor failure, + Instant failedAt) { + Objects.requireNonNull(retryDestination, "retryDestination must not be null"); + Objects.requireNonNull(delivery, "delivery must not be null"); + + int nextAttempt = delivery.metadata().deliveryAttempt() + 1; + MessageDestination destination = + new MessageDestination<>( + retryDestination, delivery.message().messageType(), EncodedMessage.class); + + return publisher.publish( + destination, + metadataMapper.stamp(delivery.message(), nextAttempt, failure, failedAt), + PublishOptions.defaults()); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaSecurityConfigurer.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaSecurityConfigurer.java new file mode 100644 index 00000000..04255c28 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaSecurityConfigurer.java @@ -0,0 +1,135 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.security.BrokerCredentialProfile; +import dev.caskeleton.messaging.security.BrokerSecurityProfile; +import dev.caskeleton.messaging.security.BrokerTlsPolicy; +import dev.caskeleton.messaging.security.CredentialRuntime; +import dev.caskeleton.messaging.security.CredentialRuntimeRegistry; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Builds the Kafka client security properties from a validated security profile. + * + *

Nothing here reads an environment variable or a property file. The credential arrives already + * resolved from {@link CredentialRuntimeRegistry}, which is what makes rotation possible: a client + * configured from a value read once at startup holds that value until the process restarts, so the + * rotation the credential store performs never reaches the broker connection. + * + *

The JAAS config is assembled into a {@code String} because that is the only shape Kafka's + * client accepts, and it is deliberately never retained — the method returns the map and keeps no + * reference, so the secret's lifetime is the caller's. + */ +public final class KafkaSecurityConfigurer { + + /** Kafka's key for the SASL mechanism. */ + public static final String SASL_MECHANISM = "sasl.mechanism"; + + /** Kafka's key for the security protocol. */ + public static final String SECURITY_PROTOCOL = "security.protocol"; + + /** Kafka's key for the JAAS configuration. */ + public static final String SASL_JAAS_CONFIG = "sasl.jaas.config"; + + /** Kafka's key for the endpoint identification algorithm. */ + public static final String ENDPOINT_IDENTIFICATION = "ssl.endpoint.identification.algorithm"; + + /** Kafka's key for the enabled TLS protocol versions. */ + public static final String ENABLED_PROTOCOLS = "ssl.enabled.protocols"; + + private final CredentialRuntimeRegistry credentials; + private final BrokerTlsPolicy tlsPolicy; + + /** + * Creates a configurer. + * + * @param credentials resolves and rotates credential material + * @param tlsPolicy validates the transport security posture + */ + public KafkaSecurityConfigurer(CredentialRuntimeRegistry credentials, BrokerTlsPolicy tlsPolicy) { + this.credentials = Objects.requireNonNull(credentials, "credentials must not be null"); + this.tlsPolicy = Objects.requireNonNull(tlsPolicy, "tlsPolicy must not be null"); + } + + /** + * Builds the client properties for one role's credential. + * + * @param profile the broker security profile + * @param credential the role credential to configure + * @param enabledProtocols the TLS protocol versions to enable + * @param now the current instant + * @return the Kafka client properties + */ + public Map configure( + BrokerSecurityProfile profile, + BrokerCredentialProfile credential, + List enabledProtocols, + Instant now) { + Objects.requireNonNull(profile, "profile must not be null"); + Objects.requireNonNull(credential, "credential must not be null"); + Objects.requireNonNull(enabledProtocols, "enabledProtocols must not be null"); + + // Validate before resolving. A profile that will be refused should not cause a credential to be + // fetched and briefly held in memory for nothing. + tlsPolicy.validate(profile, enabledProtocols); + + Map properties = new LinkedHashMap<>(); + properties.put(SECURITY_PROTOCOL, securityProtocol(profile, credential)); + + if (profile.tlsEnabled()) { + properties.put(ENABLED_PROTOCOLS, String.join(",", enabledProtocols)); + // Kafka disables hostname verification by setting this to the empty string. The policy has + // already refused that combination, so the value here is always the verifying one. + properties.put(ENDPOINT_IDENTIFICATION, "https"); + } + + switch (credential) { + case BrokerCredentialProfile.SaslScram scram -> { + CredentialRuntime resolved = credentials.resolve(scram.credentialId(), now); + properties.put(SASL_MECHANISM, "SCRAM-SHA-512"); + properties.put(SASL_JAAS_CONFIG, scramJaas(scram.credentialId(), resolved)); + } + case BrokerCredentialProfile.OAuth2 oauth -> { + credentials.resolve(oauth.credentialId(), now); + properties.put(SASL_MECHANISM, "OAUTHBEARER"); + properties.put( + SASL_JAAS_CONFIG, + "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required;"); + } + case BrokerCredentialProfile.MutualTls ignored -> properties.put(SASL_MECHANISM, "NONE"); + case BrokerCredentialProfile.UsernamePassword plain -> { + CredentialRuntime resolved = credentials.resolve(plain.credentialId(), now); + properties.put(SASL_MECHANISM, "PLAIN"); + properties.put(SASL_JAAS_CONFIG, plainJaas(plain.credentialId(), resolved)); + } + case BrokerCredentialProfile.Nkey ignored -> + throw new IllegalArgumentException( + "NKey credentials are a NATS concept, not a Kafka one"); + } + return Map.copyOf(properties); + } + + private static String securityProtocol( + BrokerSecurityProfile profile, BrokerCredentialProfile credential) { + boolean sasl = !(credential instanceof BrokerCredentialProfile.MutualTls); + if (profile.tlsEnabled()) { + return sasl ? "SASL_SSL" : "SSL"; + } + return sasl ? "SASL_PLAINTEXT" : "PLAINTEXT"; + } + + private static String scramJaas(String credentialId, CredentialRuntime resolved) { + return "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"%s\" " + .formatted(credentialId) + + "password=\"%s\";".formatted(new String(resolved.material())); + } + + private static String plainJaas(String credentialId, CredentialRuntime resolved) { + return "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" " + .formatted(credentialId) + + "password=\"%s\";".formatted(new String(resolved.material())); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaSettlementCommand.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaSettlementCommand.java new file mode 100644 index 00000000..a90e2962 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaSettlementCommand.java @@ -0,0 +1,39 @@ +package dev.caskeleton.messaging.kafka; + +import java.util.Objects; +import org.apache.kafka.common.TopicPartition; + +/** + * One settlement instruction queued for the poll thread. + * + *

Handler workers never call the Kafka consumer. {@code KafkaConsumer} is explicitly not + * thread-safe, so a worker that committed or paused directly would corrupt the client's internal + * state under concurrency. Workers enqueue an instruction instead and the single poll thread + * applies it. + * + * @param partition the partition the instruction applies to + * @param offset the record offset + * @param kind what to do + */ +public record KafkaSettlementCommand(TopicPartition partition, long offset, Kind kind) { + + /** What a queued settlement instruction asks the poll thread to do. */ + public enum Kind { + /** The handler succeeded; the offset may advance the commit watermark. */ + COMPLETE, + + /** Pause the partition and re-deliver from this offset after the retry delay. */ + PAUSE_AND_SEEK, + + /** The message was parked; the offset may advance the commit watermark. */ + PARKED + } + + public KafkaSettlementCommand { + Objects.requireNonNull(partition, "partition must not be null"); + Objects.requireNonNull(kind, "kind must not be null"); + if (offset < 0) { + throw new IllegalArgumentException("offset must not be negative"); + } + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaSettlementQueue.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaSettlementQueue.java new file mode 100644 index 00000000..f69eca41 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaSettlementQueue.java @@ -0,0 +1,52 @@ +package dev.caskeleton.messaging.kafka; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * The hand-off between handler workers and the poll thread. + * + *

Multi-producer, single-consumer by design: any worker may enqueue, only the poll thread + * drains. That is what keeps every {@code KafkaConsumer} call on one thread without any worker + * having to hold a lock. + */ +public final class KafkaSettlementQueue { + + private final Queue commands = new ConcurrentLinkedQueue<>(); + + /** + * Enqueues a settlement instruction from a handler worker. + * + * @param command the instruction + */ + public void enqueue(KafkaSettlementCommand command) { + commands.add(Objects.requireNonNull(command, "command must not be null")); + } + + /** + * Drains every queued instruction. Called only by the poll thread. + * + * @return the instructions in enqueue order + */ + public List drain() { + List drained = new ArrayList<>(); + KafkaSettlementCommand command = commands.poll(); + while (command != null) { + drained.add(command); + command = commands.poll(); + } + return List.copyOf(drained); + } + + /** + * Returns how many instructions are waiting. + * + * @return the queue depth + */ + public int size() { + return commands.size(); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTopologyInspector.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTopologyInspector.java new file mode 100644 index 00000000..195cf296 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTopologyInspector.java @@ -0,0 +1,45 @@ +package dev.caskeleton.messaging.kafka; + +import java.util.List; +import java.util.Map; + +/** + * Read-only inspection of Kafka topology. + * + *

Describe and validate only. Creation, deletion, and configuration changes belong to the Admin + * Plane with its own credential: an application runtime that could alter topology is one bad + * deployment away from deleting a topic it merely meant to read. + */ +public interface KafkaTopologyInspector { + + /** + * Returns the partition count for a topic. + * + * @param topic the topic + * @return the number of partitions + */ + int partitionCount(String topic); + + /** + * Returns the replication factor for a topic. + * + * @param topic the topic + * @return the replication factor + */ + short replicationFactor(String topic); + + /** + * Returns the effective topic configuration. + * + * @param topic the topic + * @return the configuration entries + */ + Map configuration(String topic); + + /** + * Returns the topics that exist. + * + * @return the topic names + */ + List topics(); +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionProfileValidator.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionProfileValidator.java new file mode 100644 index 00000000..46c8bd17 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionProfileValidator.java @@ -0,0 +1,52 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.delivery.ExternalSideEffectGuarantee; +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.util.Objects; + +/** + * Guards the conditions a Kafka transaction actually requires. + * + *

The last rule is the important one. A destination that declares {@code INBOX_TRANSACTIONAL} is + * telling the platform its side effect lives in a database, and a Kafka transaction cannot span + * that. Allowing both to be configured together would let a team read "transactional" twice and + * conclude the whole path is atomic when the two halves can still diverge. + */ +public final class KafkaTransactionProfileValidator { + + /** + * Validates a destination profile against the Kafka transaction capability. + * + * @param profile the destination profile + * @param transactionalIdPrefix the configured transactional id prefix + * @param brokerProfile the Kafka client profile + * @throws IllegalArgumentException when the combination cannot deliver what it claims + */ + public void validate( + DestinationProfile profile, String transactionalIdPrefix, KafkaBrokerProfile brokerProfile) { + Objects.requireNonNull(profile, "profile must not be null"); + Objects.requireNonNull(brokerProfile, "brokerProfile must not be null"); + + if (transactionalIdPrefix == null || transactionalIdPrefix.isBlank()) { + throw new IllegalArgumentException( + "a Kafka transaction requires a transactional id prefix: " + profile.name().value()); + } + if (!brokerProfile.enableIdempotence()) { + throw new IllegalArgumentException( + "a Kafka transaction requires an idempotent producer: " + profile.name().value()); + } + if (!"all".equals(brokerProfile.acks())) { + throw new IllegalArgumentException( + "a Kafka transaction requires acks=all: " + profile.name().value()); + } + if (brokerProfile.enableAutoCommit()) { + throw new IllegalArgumentException( + "a Kafka transaction requires manual offset commit: " + profile.name().value()); + } + if (profile.externalSideEffectGuarantee() == ExternalSideEffectGuarantee.INBOX_TRANSACTIONAL) { + throw new IllegalArgumentException( + "a Kafka transaction does not cover a database side effect; use the Inbox alone: " + + profile.name().value()); + } + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalDelivery.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalDelivery.java new file mode 100644 index 00000000..3132225d --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalDelivery.java @@ -0,0 +1,22 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.util.List; +import java.util.Objects; + +/** + * The input and outputs of one read-process-write transaction. + * + * @param input the message being processed + * @param outputs the messages to produce inside the same transaction + */ +public record KafkaTransactionalDelivery( + MessageEnvelope input, List outputs) { + + public KafkaTransactionalDelivery { + Objects.requireNonNull(input, "input must not be null"); + Objects.requireNonNull(outputs, "outputs must not be null"); + outputs = List.copyOf(outputs); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalOutput.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalOutput.java new file mode 100644 index 00000000..fc4b17d3 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalOutput.java @@ -0,0 +1,21 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.util.Objects; + +/** + * One message produced inside a Kafka transaction. + * + * @param destination the logical destination to produce to + * @param envelope the encoded envelope + */ +public record KafkaTransactionalOutput( + DestinationName destination, MessageEnvelope envelope) { + + public KafkaTransactionalOutput { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(envelope, "envelope must not be null"); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalProcessor.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalProcessor.java new file mode 100644 index 00000000..d071e532 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalProcessor.java @@ -0,0 +1,29 @@ +package dev.caskeleton.messaging.kafka; + +import java.util.concurrent.CompletionStage; +import java.util.function.Function; + +/** + * The M3 Kafka read-process-write transaction capability. + * + *

The guarantee is precisely scoped: the produced records and the input offset commit atomically + * within Kafka. It says nothing about a database write, an HTTP call, or a file, and the + * platform never presents it as exactly-once end to end. A handler that also writes to PostgreSQL + * inside this block still needs an Inbox, because the two commits are independent. + * + *

Exposed as a typed capability rather than a raw producer so that the transactional boundary + * cannot be opened without also being closed. + */ +public interface KafkaTransactionalProcessor { + + /** + * Runs one read-process-write transaction. + * + * @param the value the handler returns + * @param delivery the input message and the outputs to produce + * @param handler the processing step, run inside the transaction + * @return a stage completing when the transaction has committed or aborted + */ + CompletionStage processInTransaction( + KafkaTransactionalDelivery delivery, Function handler); +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalPublisher.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalPublisher.java new file mode 100644 index 00000000..3b83b8e6 --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaTransactionalPublisher.java @@ -0,0 +1,117 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.util.Map; +import java.util.Objects; +import java.util.function.Supplier; +import org.apache.kafka.clients.consumer.ConsumerGroupMetadata; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.common.TopicPartition; + +/** + * Produces records and commits input offsets inside one Kafka transaction. + * + *

The offsets are sent to the transaction rather than committed by the consumer. That is what + * makes read-process-write atomic within Kafka: either the outputs and the input offset + * both become visible to a {@code read_committed} reader, or neither does. + * + *

It does not extend to a database or an HTTP call. A handler that also writes to PostgreSQL + * inside this block still needs an Inbox, and the profile validator refuses the combination that + * would suggest otherwise. + */ +public final class KafkaTransactionalPublisher { + + private final Producer producer; + private final KafkaPublishMapper mapper; + private final Supplier groupMetadata; + + /** + * Creates a transactional publisher. + * + *

The group metadata is supplied lazily rather than captured once. It carries the consumer's + * current generation and member id, and sending stale metadata after a rebalance is what lets a + * fenced consumer commit offsets it no longer owns. + * + * @param producer a producer configured with a transactional id + * @param groupMetadata supplies the consuming group's current metadata + */ + public KafkaTransactionalPublisher( + Producer producer, Supplier groupMetadata) { + this.producer = Objects.requireNonNull(producer, "producer must not be null"); + this.groupMetadata = Objects.requireNonNull(groupMetadata, "groupMetadata must not be null"); + this.mapper = new KafkaPublishMapper(); + } + + /** + * Creates a transactional publisher for a group with no live consumer metadata. + * + *

Only appropriate where the offsets being committed are not owned by a rebalancing consumer, + * such as a single-instance replay job. + * + * @param producer a producer configured with a transactional id + * @param consumerGroup the group whose offsets are committed in the transaction + * @return the publisher + */ + public static KafkaTransactionalPublisher forStaticGroup( + Producer producer, String consumerGroup) { + if (consumerGroup == null || consumerGroup.isBlank()) { + throw new IllegalArgumentException("consumerGroup must not be blank"); + } + return new KafkaTransactionalPublisher( + producer, () -> new ConsumerGroupMetadata(consumerGroup)); + } + + /** Initialises the transactional producer. Must be called once before any transaction. */ + public void initialise() { + producer.initTransactions(); + } + + /** + * Sends outputs and the input offset in one transaction. + * + *

Any failure aborts. Aborting rather than leaving the transaction open is what lets the next + * producer instance fence this one instead of waiting for a timeout. + * + * @param delivery the input and the outputs to produce + * @param profiles the destination profile for each output destination + * @param inputOffsets the input offsets to commit with the outputs + */ + @SuppressWarnings("FutureReturnValueIgnored") + public void sendInTransaction( + KafkaTransactionalDelivery delivery, + Map profiles, + Map inputOffsets) { + // The per-send Futures are deliberately not inspected. Inside a transaction the commit is the + // barrier: any send failure surfaces from commitTransaction, and a send that failed leaves the + // producer in an error state that abortTransaction resolves. Waiting on each Future here would + // serialise the batch for no additional safety. + Objects.requireNonNull(delivery, "delivery must not be null"); + Objects.requireNonNull(profiles, "profiles must not be null"); + Objects.requireNonNull(inputOffsets, "inputOffsets must not be null"); + + producer.beginTransaction(); + try { + for (KafkaTransactionalOutput output : delivery.outputs()) { + DestinationProfile profile = profiles.get(output.destination().value()); + if (profile == null) { + throw new MessagingConfigurationException( + "DESTINATION_NOT_REGISTERED", + "no destination profile is registered for " + output.destination().value()); + } + producer.send( + mapper.toRecord( + new dev.caskeleton.messaging.transport.TransportPublishRequest( + profile, + output.envelope(), + dev.caskeleton.messaging.api.publish.PublishOptions.defaults()))); + } + producer.sendOffsetsToTransaction(inputOffsets, groupMetadata.get()); + producer.commitTransaction(); + } catch (RuntimeException exception) { + producer.abortTransaction(); + throw exception; + } + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/PartitionOffsetTracker.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/PartitionOffsetTracker.java new file mode 100644 index 00000000..3c04e5dd --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/PartitionOffsetTracker.java @@ -0,0 +1,48 @@ +package dev.caskeleton.messaging.kafka; + +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.kafka.common.TopicPartition; + +/** Tracks which delivered offsets have finished so the poll thread knows what is safe to commit. */ +public interface PartitionOffsetTracker { + + /** + * Records that an offset was handed to a handler. + * + * @param partition the partition + * @param offset the record offset + */ + void delivered(TopicPartition partition, long offset); + + /** + * Records that an offset finished handling. + * + * @param partition the partition + * @param offset the record offset + */ + void completed(TopicPartition partition, long offset); + + /** + * Returns the highest offset whose predecessors have all completed. + * + * @param partition the partition + * @return the highest contiguous completed offset, when one exists + */ + Optional highestContiguousCompleted(TopicPartition partition); + + /** + * Returns the offset to commit, which is one past the highest contiguous completed offset. + * + * @param partition the partition + * @return the commit offset, when one exists + */ + OptionalLong commitOffset(TopicPartition partition); + + /** + * Drops all state for a partition, as on revocation. + * + * @param partition the partition + */ + void forget(TopicPartition partition); +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/PartitionWorkCoordinator.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/PartitionWorkCoordinator.java new file mode 100644 index 00000000..8713780c --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/PartitionWorkCoordinator.java @@ -0,0 +1,128 @@ +package dev.caskeleton.messaging.kafka; + +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.kafka.common.TopicPartition; + +/** + * Enforces the in-flight ceiling for each partition and tracks which partitions are paused. + * + *

Concurrency is per partition, not per consumer. On an ordered destination the ceiling is one, + * which is the only setting that actually preserves order: two handlers running against the same + * partition can finish in either sequence no matter what the broker delivered. + */ +public final class PartitionWorkCoordinator { + + private final int maxInFlightPerPartition; + private final Map inFlight = new ConcurrentHashMap<>(); + private final Set paused = ConcurrentHashMap.newKeySet(); + + /** + * Creates a coordinator with an explicit ceiling. + * + * @param maxInFlightPerPartition how many deliveries may run at once per partition + */ + public PartitionWorkCoordinator(int maxInFlightPerPartition) { + if (maxInFlightPerPartition < 1) { + throw new IllegalArgumentException("maxInFlightPerPartition must be at least 1"); + } + this.maxInFlightPerPartition = maxInFlightPerPartition; + } + + /** + * Tries to reserve a slot for one delivery. + * + * @param partition the partition + * @return true when the delivery may start + */ + public boolean tryAcquire(TopicPartition partition) { + Objects.requireNonNull(partition, "partition must not be null"); + if (paused.contains(partition)) { + return false; + } + AtomicInteger counter = inFlight.computeIfAbsent(partition, key -> new AtomicInteger()); + while (true) { + int current = counter.get(); + if (current >= maxInFlightPerPartition) { + return false; + } + if (counter.compareAndSet(current, current + 1)) { + return true; + } + } + } + + /** + * Releases a slot after a delivery finishes. + * + * @param partition the partition + */ + public void release(TopicPartition partition) { + AtomicInteger counter = inFlight.get(partition); + if (counter != null && counter.get() > 0) { + counter.decrementAndGet(); + } + } + + /** + * Pauses a partition. + * + * @param partition the partition + */ + public void pause(TopicPartition partition) { + paused.add(partition); + } + + /** + * Resumes a partition. + * + * @param partition the partition + */ + public void resume(TopicPartition partition) { + paused.remove(partition); + } + + /** + * Reports whether a partition is paused. + * + * @param partition the partition + * @return true when paused + */ + public boolean isPaused(TopicPartition partition) { + return paused.contains(partition); + } + + /** + * Returns the currently paused partitions. + * + * @return an immutable snapshot + */ + public Set pausedPartitions() { + return Set.copyOf(new LinkedHashSet<>(paused)); + } + + /** + * Drops all state for a partition, as on revocation. + * + * @param partition the partition + */ + public void forget(TopicPartition partition) { + inFlight.remove(partition); + paused.remove(partition); + } + + /** + * Returns the in-flight count for a partition. + * + * @param partition the partition + * @return how many deliveries are running + */ + public int inFlight(TopicPartition partition) { + AtomicInteger counter = inFlight.get(partition); + return counter == null ? 0 : counter.get(); + } +} diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/SpringKafkaTransactionalProcessor.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/SpringKafkaTransactionalProcessor.java new file mode 100644 index 00000000..84f080be --- /dev/null +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/SpringKafkaTransactionalProcessor.java @@ -0,0 +1,61 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.Function; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; + +/** + * The M3 Kafka transaction capability, backed by a transactional producer. + * + *

The handler runs inside the transaction and its outputs are produced before the input + * offset is sent. If the handler throws, the transaction aborts and neither the outputs nor the + * offset become visible to a {@code read_committed} reader, so the input is redelivered and + * reprocessed cleanly. + * + *

The scope is Kafka only, and the profile validator refuses to pair this with a destination + * that declares {@code INBOX_TRANSACTIONAL}. Reading "transactional" twice and concluding the whole + * path is atomic is exactly the mistake this capability must not enable. + */ +public final class SpringKafkaTransactionalProcessor implements KafkaTransactionalProcessor { + + private final KafkaTransactionalPublisher publisher; + private final Map profiles; + private final Function> + inputOffsets; + + /** + * Creates a transactional processor. + * + * @param publisher the transactional publisher + * @param profiles the destination profile for each output destination + * @param inputOffsets resolves the input offsets to commit with the outputs + */ + public SpringKafkaTransactionalProcessor( + KafkaTransactionalPublisher publisher, + Map profiles, + Function> inputOffsets) { + this.publisher = Objects.requireNonNull(publisher, "publisher must not be null"); + this.profiles = Map.copyOf(Objects.requireNonNull(profiles, "profiles must not be null")); + this.inputOffsets = Objects.requireNonNull(inputOffsets, "inputOffsets must not be null"); + } + + @Override + public CompletionStage processInTransaction( + KafkaTransactionalDelivery delivery, Function handler) { + Objects.requireNonNull(delivery, "delivery must not be null"); + Objects.requireNonNull(handler, "handler must not be null"); + + try { + T result = handler.apply(delivery); + publisher.sendInTransaction(delivery, profiles, inputOffsets.apply(delivery)); + return CompletableFuture.completedFuture(result); + } catch (RuntimeException exception) { + return CompletableFuture.failedFuture(exception); + } + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/ContiguousPartitionOffsetTrackerTest.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/ContiguousPartitionOffsetTrackerTest.java new file mode 100644 index 00000000..699ae1be --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/ContiguousPartitionOffsetTrackerTest.java @@ -0,0 +1,94 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.OptionalLong; +import org.apache.kafka.common.TopicPartition; +import org.junit.jupiter.api.Test; + +class ContiguousPartitionOffsetTrackerTest { + + private static final TopicPartition PARTITION = new TopicPartition("orders", 0); + + private final ContiguousPartitionOffsetTracker tracker = new ContiguousPartitionOffsetTracker(); + + @Test + void commitsOnlyThroughHighestContiguousCompletedOffset() { + tracker.delivered(PARTITION, 10); + tracker.delivered(PARTITION, 11); + tracker.delivered(PARTITION, 12); + tracker.completed(PARTITION, 10); + tracker.completed(PARTITION, 12); + + assertThat(tracker.highestContiguousCompleted(PARTITION)).hasValue(10L); + } + + @Test + void theCommitOffsetIsOnePastTheContiguousWatermark() { + tracker.delivered(PARTITION, 10); + tracker.completed(PARTITION, 10); + + assertThat(tracker.commitOffset(PARTITION)).isEqualTo(OptionalLong.of(11)); + } + + @Test + void aGapClosingLaterAdvancesTheWatermarkPastIt() { + tracker.delivered(PARTITION, 10); + tracker.delivered(PARTITION, 11); + tracker.delivered(PARTITION, 12); + tracker.completed(PARTITION, 10); + tracker.completed(PARTITION, 12); + + tracker.completed(PARTITION, 11); + + assertThat(tracker.highestContiguousCompleted(PARTITION)).hasValue(12L); + } + + @Test + void nothingIsCommittableUntilTheLowestDeliveredOffsetCompletes() { + tracker.delivered(PARTITION, 10); + tracker.delivered(PARTITION, 11); + tracker.completed(PARTITION, 11); + + assertThat(tracker.highestContiguousCompleted(PARTITION)).isEmpty(); + assertThat(tracker.commitOffset(PARTITION)).isEmpty(); + } + + @Test + void anUntouchedPartitionHasNothingToCommit() { + assertThat(tracker.highestContiguousCompleted(new TopicPartition("orders", 7))).isEmpty(); + } + + @Test + void pruningDropsTheCommittedPrefixButKeepsTheRest() { + tracker.delivered(PARTITION, 10); + tracker.delivered(PARTITION, 11); + tracker.completed(PARTITION, 10); + + tracker.pruneThrough(PARTITION, 10); + + assertThat(tracker.inFlight(PARTITION)).isEqualTo(1); + assertThat(tracker.highestContiguousCompleted(PARTITION)).isEmpty(); + } + + @Test + void revocationForgetsThePartitionEntirely() { + tracker.delivered(PARTITION, 10); + tracker.completed(PARTITION, 10); + + tracker.forget(PARTITION); + + assertThat(tracker.highestContiguousCompleted(PARTITION)).isEmpty(); + assertThat(tracker.inFlight(PARTITION)).isZero(); + } + + @Test + void inFlightCountsDeliveredOffsetsThatHaveNotCompleted() { + tracker.delivered(PARTITION, 10); + tracker.delivered(PARTITION, 11); + tracker.delivered(PARTITION, 12); + tracker.completed(PARTITION, 11); + + assertThat(tracker.inFlight(PARTITION)).isEqualTo(2); + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaAmbiguityChaosIT.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaAmbiguityChaosIT.java new file mode 100644 index 00000000..bd55d42d --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaAmbiguityChaosIT.java @@ -0,0 +1,168 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import dev.caskeleton.messaging.schema.EncodedMessage; +import dev.caskeleton.messaging.testkit.DockerAvailability; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Optional; +import java.util.Properties; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.kafka.KafkaContainer; + +/** + * Proves the platform's hardest guarantee against a live broker: a lost confirmation is reported, + * not guessed. + * + *

The broker is frozen mid-flight with {@code docker pause}. The producer's socket stays open + * and the record may already have been written to it, but no acknowledgement can arrive — which is + * exactly the state a network partition or a broker stall produces, and exactly the state that has + * no correct boolean answer. + * + *

The assertion is deliberately strict about what the result may not claim: no broker + * acceptance, no confirmation level, and a non-retryable ambiguous failure. An adapter that + * optimistically reported success here would lose the message on a real partition; one that + * reported a plain failure would invite a retry under a new identity and duplicate the order. + * + *

Pausing the container is used rather than a proxy because it needs no rewiring of the broker's + * advertised listeners, which would otherwise change the very connection path under test. + */ +@Testcontainers +@EnabledIf("dockerAvailable") +class KafkaAmbiguityChaosIT { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + @Container private static final KafkaContainer KAFKA = new KafkaContainer("apache/kafka:4.1.0"); + + static boolean dockerAvailable() { + return DockerAvailability.isAvailable(); + } + + @Test + void aLostConfirmationIsReportedAsAmbiguousRatherThanGuessed() { + try (Producer producer = new KafkaProducer<>(producerConfig())) { + KafkaMessagingTransport transport = new KafkaMessagingTransport("kafka-primary", 1, producer); + + pauseBroker(); + PublishResult result; + try { + result = + transport + .publish( + new TransportPublishRequest( + KafkaContractHarness.profile(), envelope(), PublishOptions.defaults())) + .toCompletableFuture() + .join() + .result(); + } finally { + resumeBroker(); + } + + assertThat(result.completion()).isEqualTo(PublishCompletion.AMBIGUOUS); + assertThat(result.evidence().transmission()) + .isEqualTo(TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED); + assertThat(result.evidence().brokerAccepted()) + .as("nothing proves the broker took it") + .isFalse(); + assertThat(result.evidence().confirmationLevel()).isEqualTo(ConfirmationLevel.NONE); + assertThat(result.mayHaveBeenStored()).isTrue(); + + assertThat(result.failure().orElseThrow().category()).isEqualTo(FailureCategory.AMBIGUOUS); + assertThat(result.failure().orElseThrow().retryable()) + .as("an ambiguous publish is not silently retried; the caller decides, under the same id") + .isFalse(); + } + } + + @Test + void theBrokerRecoversAndPublishingConfirmsAgain() { + try (Producer producer = new KafkaProducer<>(producerConfig())) { + KafkaMessagingTransport transport = new KafkaMessagingTransport("kafka-primary", 1, producer); + + PublishResult result = + transport + .publish( + new TransportPublishRequest( + KafkaContractHarness.profile(), envelope(), PublishOptions.defaults())) + .toCompletableFuture() + .join() + .result(); + + assertThat(result.completion()).isEqualTo(PublishCompletion.CONFIRMED); + } + } + + private static void pauseBroker() { + KAFKA.getDockerClient().pauseContainerCmd(KAFKA.getContainerId()).exec(); + } + + private static void resumeBroker() { + try { + KAFKA.getDockerClient().unpauseContainerCmd(KAFKA.getContainerId()).exec(); + } catch (RuntimeException exception) { + // Already running; nothing to undo. + } + } + + private static Properties producerConfig() { + Properties properties = new Properties(); + properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA.getBootstrapServers()); + properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + properties.put( + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + properties.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true); + properties.put(ProducerConfig.ACKS_CONFIG, "all"); + properties.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 5); + // Short deadlines so the stall resolves inside the test rather than the suite timeout. + properties.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 4_000); + properties.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 2_000); + properties.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 4_000); + return properties; + } + + private static MessageEnvelope envelope() { + return new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + NOW, + Optional.of(NOW), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.of("acct-1"), + Optional.of("acct-1"), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage( + "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), + ContentType.JSON, + Optional.empty())); + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaBrokerIT.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaBrokerIT.java new file mode 100644 index 00000000..10b9dce9 --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaBrokerIT.java @@ -0,0 +1,284 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import dev.caskeleton.messaging.schema.EncodedMessage; +import dev.caskeleton.messaging.testkit.DockerAvailability; +import dev.caskeleton.messaging.transport.GracefulShutdownCoordinator; +import dev.caskeleton.messaging.transport.TransportConsumerSpec; +import dev.caskeleton.messaging.transport.TransportDelivery; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.kafka.KafkaContainer; + +/** + * Certifies the Kafka adapter against a live broker. + * + *

The deterministic suite proves the platform's semantics; this proves the client actually + * behaves that way against the broker version the compatibility matrix claims. Both are needed: a + * mock cannot tell you that {@code acks=all} produced a real replication acknowledgement, and a + * live broker cannot be made to lose a confirmation on demand. + * + *

Topics are created through {@code Admin} rather than by auto-creation, because production + * topology is created by infrastructure and only validated by the application. A test that relies + * on auto-creation is testing a configuration the platform forbids. + */ +@Testcontainers +@EnabledIf("dockerAvailable") +class KafkaBrokerIT { + + private static final String TOPIC = "order.events.v1"; + private static final String DEAD_LETTER_TOPIC = "order.events.v1.dlt"; + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + @Container private static final KafkaContainer KAFKA = new KafkaContainer("apache/kafka:4.1.0"); + + private Producer producer; + private Consumer consumer; + + static boolean dockerAvailable() { + return DockerAvailability.isAvailable(); + } + + @BeforeEach + void createTopology() throws Exception { + try (Admin admin = + Admin.create( + Map.of(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA.getBootstrapServers()))) { + admin + .createTopics( + List.of( + new NewTopic(TOPIC, 1, (short) 1), new NewTopic(DEAD_LETTER_TOPIC, 1, (short) 1))) + .all() + .get(30, TimeUnit.SECONDS); + } catch (java.util.concurrent.ExecutionException exception) { + if (!(exception.getCause() instanceof org.apache.kafka.common.errors.TopicExistsException)) { + throw exception; + } + } + producer = new KafkaProducer<>(producerConfig()); + consumer = new KafkaConsumer<>(consumerConfig()); + } + + @AfterEach + void closeClients() { + if (producer != null) { + producer.close(Duration.ofSeconds(5)); + } + if (consumer != null) { + consumer.close(); + } + } + + @Test + void aStableProfilePublishConfirmsWithReplicationEvidence() { + KafkaMessagingTransport transport = new KafkaMessagingTransport("kafka-primary", 1, producer); + + PublishResult result = + transport + .publish( + new TransportPublishRequest( + KafkaContractHarness.profile(), envelope(), PublishOptions.defaults())) + .toCompletableFuture() + .join() + .result(); + + assertThat(result.completion()).isEqualTo(PublishCompletion.CONFIRMED); + assertThat(result.evidence().transmission()).isEqualTo(TransmissionEvidence.TRANSMITTED); + assertThat(result.evidence().confirmationLevel()) + .isEqualTo(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK); + assertThat(result.position()).isPresent(); + } + + @Test + void anInvalidTopicIsRejectedRatherThanReportedAmbiguous() { + KafkaMessagingTransport transport = new KafkaMessagingTransport("kafka-primary", 1, producer); + + PublishResult result = + transport + .publish( + new TransportPublishRequest( + KafkaFixtureProfiles.withTopic("not a legal topic name!"), + envelope(), + PublishOptions.defaults())) + .toCompletableFuture() + .join() + .result(); + + assertThat(result.completion()) + .as("an invalid topic is definitive: the record was never stored") + .isEqualTo(PublishCompletion.REJECTED); + assertThat(result.evidence().transmission()).isEqualTo(TransmissionEvidence.NOT_TRANSMITTED); + assertThat(result.failure().orElseThrow().retryable()).isFalse(); + } + + /** + * Demonstrates why the platform forbids topology auto-creation in production. + * + *

This broker has auto-creation enabled, so publishing to a topic nobody declared silently + * succeeds and creates it. Nothing about the publish result reveals the mistake: the evidence is + * a genuine replication acknowledgement for a topic that should not exist. + * + *

That is the whole argument for {@code topologyAutoCreate=false} on production profiles and + * for validate-only startup checks — the failure this test pins is invisible at publish time and + * only shows up later as a topic with the wrong partition count and no retention policy. + */ + @Test + void autoCreationMakesAnUndeclaredTopicIndistinguishableFromADeclaredOne() { + KafkaMessagingTransport transport = new KafkaMessagingTransport("kafka-primary", 1, producer); + + PublishResult result = + transport + .publish( + new TransportPublishRequest( + KafkaFixtureProfiles.withTopic("undeclared.topic.v1"), + envelope(), + PublishOptions.defaults())) + .toCompletableFuture() + .join() + .result(); + + assertThat(result.completion()).isEqualTo(PublishCompletion.CONFIRMED); + assertThat(KafkaContractHarness.profile().topologyAutoCreate()) + .as("production profiles must never permit this") + .isFalse(); + } + + @Test + void aPublishedMessageIsConsumedWithItsIdentityIntactAndCommitted() { + KafkaMessagingTransport transport = new KafkaMessagingTransport("kafka-primary", 1, producer); + MessageEnvelope published = envelope(); + + transport + .publish( + new TransportPublishRequest( + KafkaContractHarness.profile(), published, PublishOptions.defaults())) + .toCompletableFuture() + .join(); + + List handled = new ArrayList<>(); + KafkaConsumerRegistrar registrar = + new KafkaConsumerRegistrar( + new TransportConsumerSpec( + KafkaContractHarness.profile(), + delivery -> { + handled.add(delivery); + delivery.settlement().acknowledge().toCompletableFuture().join(); + return CompletableFuture.completedFuture(null); + }), + consumer, + Runnable::run, + new GracefulShutdownCoordinator(Duration.ofSeconds(30)), + Duration.ofMillis(500)); + registrar.subscribe(TOPIC); + + for (int attempt = 0; attempt < 20 && handled.isEmpty(); attempt++) { + registrar.pollOnce(NOW); + } + registrar.pollOnce(NOW); + + assertThat(handled) + .singleElement() + .satisfies( + delivery -> { + assertThat(delivery.envelope().messageId()).isEqualTo(published.messageId()); + assertThat(delivery.envelope().messageType()).isEqualTo(published.messageType()); + assertThat(delivery.envelope().payload().bytes()) + .isEqualTo(published.payload().bytes()); + }); + assertThat(registrar.committedOffset(new TopicPartition(TOPIC, 0))).isPresent(); + + registrar.close(); + } + + private Properties producerConfig() { + Properties properties = new Properties(); + properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA.getBootstrapServers()); + properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + properties.put( + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + // The Stable profile the guard enforces. + properties.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true); + properties.put(ProducerConfig.ACKS_CONFIG, "all"); + properties.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 5); + properties.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 10_000); + properties.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5_000); + return properties; + } + + private Properties consumerConfig() { + Properties properties = new Properties(); + properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA.getBootstrapServers()); + properties.put( + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + properties.put( + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + properties.put(ConsumerConfig.GROUP_ID_CONFIG, "order-projection-" + System.nanoTime()); + properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + // Auto commit is forbidden: the platform commits after handler success. + properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); + return properties; + } + + private static MessageEnvelope envelope() { + return new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + NOW, + Optional.of(NOW), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.of("acct-1"), + Optional.of("acct-1"), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage( + "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), + ContentType.JSON, + Optional.empty())); + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrarTest.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrarTest.java new file mode 100644 index 00000000..94324e39 --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrarTest.java @@ -0,0 +1,261 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.transport.GracefulShutdownCoordinator; +import dev.caskeleton.messaging.transport.TransportConsumerSpec; +import dev.caskeleton.messaging.transport.TransportDelivery; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.MockConsumer; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.junit.jupiter.api.Test; + +/** + * Drives the poll loop one cycle at a time against {@code MockConsumer}. + * + *

Handlers run on a deferred executor rather than inline. With an inline executor every handler + * finishes before the next record is considered, so the in-flight ceiling would never actually be + * reached and the ordering guarantee would look correct without being tested. + */ +class KafkaConsumerRegistrarTest { + + private static final String TOPIC = "order.events.v1"; + private static final TopicPartition PARTITION = new TopicPartition(TOPIC, 0); + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + private final MockConsumer consumer = new MockConsumer<>("earliest"); + private final DeferredExecutor handlers = new DeferredExecutor(); + private final List handled = new ArrayList<>(); + + @Test + void dispatchesEachRecordAndCommitsThroughTheContiguousWatermark() { + KafkaConsumerRegistrar registrar = registrar(this::settleImmediately); + assign(); + addRecord(10); + + registrar.pollOnce(NOW); + handlers.runAll(); + addRecord(11); + registrar.pollOnce(NOW); + handlers.runAll(); + registrar.pollOnce(NOW); + + assertThat(handled).hasSize(2); + assertThat(registrar.committedOffset(PARTITION)).hasValue(12L); + } + + @Test + void aRecordStillRunningHoldsBackTheWatermark() { + KafkaConsumerRegistrar registrar = registrar(this::neverSettle); + assign(); + addRecord(10); + + registrar.pollOnce(NOW); + handlers.runAll(); + registrar.pollOnce(NOW); + + assertThat(handled).hasSize(1); + assertThat(registrar.committedOffset(PARTITION)).isEmpty(); + } + + @Test + void anOrderedDestinationDispatchesOneRecordAtATime() { + KafkaConsumerRegistrar registrar = registrar(this::neverSettle); + assign(); + addRecord(10); + addRecord(11); + + int dispatched = registrar.pollOnce(NOW); + + assertThat(dispatched) + .as("the ordering unit allows one in-flight delivery, so the second waits") + .isEqualTo(1); + assertThat(consumer.paused()).contains(PARTITION); + } + + @Test + void theWatermarkAdvancesOnceTheGapCloses() { + KafkaConsumerRegistrar registrar = registrar(this::neverSettle); + assign(); + addRecord(10); + + registrar.pollOnce(NOW); + handlers.runAll(); + assertThat(registrar.committedOffset(PARTITION)).isEmpty(); + + handled.get(0).settlement().acknowledge().toCompletableFuture().join(); + registrar.pollOnce(NOW); + + assertThat(registrar.committedOffset(PARTITION)).hasValue(11L); + } + + @Test + void theEnvelopeIsRebuiltFromReservedHeaders() { + KafkaConsumerRegistrar registrar = registrar(this::settleImmediately); + assign(); + addRecord(10); + + registrar.pollOnce(NOW); + handlers.runAll(); + + assertThat(handled) + .singleElement() + .satisfies( + delivery -> { + assertThat(delivery.envelope().messageType().value()).isEqualTo("order.created"); + assertThat(delivery.metadata().deliveryAttempt()).isEqualTo(1); + assertThat(delivery.metadata().brokerPosition()).isPresent(); + assertThat(delivery.metadata().partitionOrQueue()).hasValue("order.events.v1-0"); + }); + } + + @Test + void drainingStopsDispatchingWithoutClosingTheConsumer() { + GracefulShutdownCoordinator shutdown = new GracefulShutdownCoordinator(Duration.ofSeconds(30)); + KafkaConsumerRegistrar registrar = registrar(this::settleImmediately, shutdown); + assign(); + addRecord(10); + + shutdown.beginDrain(NOW); + int dispatched = registrar.pollOnce(NOW); + + assertThat(dispatched).isZero(); + assertThat(handled).isEmpty(); + assertThat(registrar.isActive()).isTrue(); + } + + @Test + void closingCommitsWhatIsSafeAndDeactivatesTheRegistration() { + KafkaConsumerRegistrar registrar = registrar(this::settleImmediately); + assign(); + addRecord(10); + registrar.pollOnce(NOW); + handlers.runAll(); + + registrar.close(); + + assertThat(registrar.committedOffset(PARTITION)).hasValue(11L); + assertThat(registrar.isActive()).isFalse(); + } + + @Test + void pauseAndResumeOperateOnTheAssignedPartitions() { + KafkaConsumerRegistrar registrar = registrar(this::settleImmediately); + assign(); + + registrar.pause("").toCompletableFuture().join(); + assertThat(consumer.paused()).contains(PARTITION); + + registrar.resume("").toCompletableFuture().join(); + assertThat(consumer.paused()).doesNotContain(PARTITION); + } + + @Test + void anUndecodableRecordIsParkedRatherThanRetriedForever() { + KafkaConsumerRegistrar registrar = registrar(this::settleImmediately); + assign(); + addRecordWithoutIdentityHeaders(10); + + registrar.pollOnce(NOW); + handlers.runAll(); + registrar.pollOnce(NOW); + + assertThat(handled).isEmpty(); + assertThat(registrar.committedOffset(PARTITION)).hasValue(11L); + } + + private void assign() { + consumer.assign(List.of(PARTITION)); + consumer.updateBeginningOffsets(Map.of(PARTITION, 10L)); + consumer.seek(PARTITION, 10L); + } + + private void addRecord(long offset) { + RecordHeaders headers = new RecordHeaders(); + headers.add( + ReservedHeaders.MESSAGE_ID, UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8)); + headers.add(ReservedHeaders.MESSAGE_TYPE, "order.created".getBytes(StandardCharsets.UTF_8)); + headers.add(ReservedHeaders.SCHEMA_VERSION, "1".getBytes(StandardCharsets.UTF_8)); + headers.add(ReservedHeaders.CONTENT_TYPE, "application/json".getBytes(StandardCharsets.UTF_8)); + consumer.addRecord(record(offset, headers)); + } + + private void addRecordWithoutIdentityHeaders(long offset) { + consumer.addRecord(record(offset, new RecordHeaders())); + } + + private ConsumerRecord record(long offset, RecordHeaders headers) { + return new ConsumerRecord<>( + TOPIC, + 0, + offset, + NOW.toEpochMilli(), + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + 0, + 0, + null, + "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), + headers, + java.util.Optional.empty()); + } + + private KafkaConsumerRegistrar registrar( + java.util.function.Function> sink) { + return registrar(sink, new GracefulShutdownCoordinator(Duration.ofSeconds(30))); + } + + private KafkaConsumerRegistrar registrar( + java.util.function.Function> sink, + GracefulShutdownCoordinator shutdown) { + return new KafkaConsumerRegistrar( + new TransportConsumerSpec(KafkaContractHarness.profile(), sink), + consumer, + handlers, + shutdown, + Duration.ofMillis(1)); + } + + /** Acknowledges inside the handler, as a successful M1 delivery does. */ + private CompletionStage settleImmediately(TransportDelivery delivery) { + handled.add(delivery); + delivery.settlement().acknowledge().toCompletableFuture().join(); + return CompletableFuture.completedFuture(null); + } + + /** Leaves the delivery in flight, which is what holds the watermark back. */ + private CompletionStage neverSettle(TransportDelivery delivery) { + handled.add(delivery); + return CompletableFuture.completedFuture(null); + } +} + +/** Queues handler work so the in-flight ceiling is observable. */ +final class DeferredExecutor implements Executor { + + private final Deque queued = new ArrayDeque<>(); + + @Override + public void execute(Runnable command) { + queued.addLast(command); + } + + void runAll() { + while (!queued.isEmpty()) { + queued.pollFirst().run(); + } + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaConsumerSettlementIT.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaConsumerSettlementIT.java new file mode 100644 index 00000000..69a4b38c --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaConsumerSettlementIT.java @@ -0,0 +1,216 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.transport.GracefulShutdownCoordinator; +import dev.caskeleton.messaging.transport.TransportConsumerSpec; +import dev.caskeleton.messaging.transport.TransportDelivery; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.OptionalLong; +import java.util.concurrent.CompletableFuture; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.common.TopicPartition; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; + +/** + * Certifies that offsets advance only for work that actually finished, on a live broker. + * + *

The deterministic suite proves the tracker's arithmetic. This proves the arithmetic reaches + * the broker: that an unacknowledged record leaves the committed offset behind it, and that a + * restarted consumer therefore reads it again. + * + *

Redelivery after an unsettled handler is the whole at-least-once guarantee. If it did not hold + * here, every downstream inbox would be protecting against a duplicate that could never happen + * while the real risk — silent loss — went unnoticed. + */ +@EnabledIf("dockerAvailable") +class KafkaConsumerSettlementIT { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + /** + * A topic per test, because the broker is shared across suites and across tests in this class. + * + *

A fresh consumer group reads from the earliest offset, so a topic carrying records another + * test published would make this suite's assertions count someone else's messages. + */ + private String topic; + + private TopicPartition partition; + private Producer producer; + private String group; + + static boolean dockerAvailable() { + return KafkaContainerFixture.dockerAvailable(); + } + + @BeforeEach + void createTopology() { + long unique = System.nanoTime(); + topic = "settlement.events." + unique; + partition = new TopicPartition(topic, 0); + KafkaContainerFixture.createTopics(1, topic); + producer = KafkaContainerFixture.producer(); + group = "settlement-it-" + unique; + } + + @AfterEach + void closeProducer() { + if (producer != null) { + producer.close(Duration.ofSeconds(5)); + } + } + + @Test + void anAcknowledgedRecordAdvancesTheCommittedOffset() { + publish(1); + + try (Consumer consumer = KafkaContainerFixture.consumer(group)) { + List handled = new ArrayList<>(); + KafkaConsumerRegistrar registrar = registrarThatAcknowledges(consumer, handled); + registrar.subscribe(topic); + drain(registrar, handled, 1); + + assertThat(registrar.committedOffset(partition)) + .as("the next offset to read, not the one just handled") + .isEqualTo(OptionalLong.of(1)); + registrar.close(); + } + } + + @Test + void anUnsettledRecordLeavesTheCommittedOffsetBehindIt() { + publish(1); + + try (Consumer consumer = KafkaContainerFixture.consumer(group)) { + List handled = new ArrayList<>(); + KafkaConsumerRegistrar registrar = registrarThatNeverSettles(consumer, handled); + registrar.subscribe(topic); + drain(registrar, handled, 1); + + assertThat(registrar.committedOffset(partition)) + .as("committing a record whose handler never finished is how a message is lost") + .isEmpty(); + registrar.close(); + } + } + + @Test + void anUnsettledRecordIsRedeliveredToAFreshConsumerInTheSameGroup() { + publish(1); + + try (Consumer first = KafkaContainerFixture.consumer(group)) { + List handled = new ArrayList<>(); + KafkaConsumerRegistrar registrar = registrarThatNeverSettles(first, handled); + registrar.subscribe(topic); + drain(registrar, handled, 1); + registrar.close(); + } + + try (Consumer second = KafkaContainerFixture.consumer(group)) { + List handled = new ArrayList<>(); + KafkaConsumerRegistrar registrar = registrarThatAcknowledges(second, handled); + registrar.subscribe(topic); + drain(registrar, handled, 1); + + assertThat(handled) + .as("redelivery after an unsettled handler is the at-least-once guarantee itself") + .hasSize(1); + registrar.close(); + } + } + + @Test + void anAcknowledgedRecordIsNotRedeliveredToAFreshConsumer() { + publish(1); + + try (Consumer first = KafkaContainerFixture.consumer(group)) { + List handled = new ArrayList<>(); + KafkaConsumerRegistrar registrar = registrarThatAcknowledges(first, handled); + registrar.subscribe(topic); + drain(registrar, handled, 1); + registrar.close(); + } + + try (Consumer second = KafkaContainerFixture.consumer(group)) { + List handled = new ArrayList<>(); + KafkaConsumerRegistrar registrar = registrarThatAcknowledges(second, handled); + registrar.subscribe(topic); + for (int attempt = 0; attempt < 6; attempt++) { + registrar.pollOnce(NOW); + } + + assertThat(handled).isEmpty(); + registrar.close(); + } + } + + private void publish(int count) { + KafkaMessagingTransport transport = new KafkaMessagingTransport("kafka-primary", 1, producer); + for (int index = 0; index < count; index++) { + transport + .publish( + new TransportPublishRequest( + KafkaFixtureProfiles.withTopic(topic), + KafkaTransactionFixtures.delivery(topic, "settle-" + index) + .outputs() + .get(0) + .envelope(), + PublishOptions.defaults())) + .toCompletableFuture() + .join(); + } + } + + private KafkaConsumerRegistrar registrarThatAcknowledges( + Consumer consumer, List handled) { + return registrar( + consumer, + delivery -> { + handled.add(delivery); + delivery.settlement().acknowledge().toCompletableFuture().join(); + return CompletableFuture.completedFuture(null); + }); + } + + private KafkaConsumerRegistrar registrarThatNeverSettles( + Consumer consumer, List handled) { + return registrar( + consumer, + delivery -> { + handled.add(delivery); + // No settlement at all: the handler "died" mid-work. + return CompletableFuture.completedFuture(null); + }); + } + + private KafkaConsumerRegistrar registrar( + Consumer consumer, + java.util.function.Function> + sink) { + return new KafkaConsumerRegistrar( + new TransportConsumerSpec(KafkaFixtureProfiles.withTopic(topic), sink), + consumer, + Runnable::run, + new GracefulShutdownCoordinator(Duration.ofSeconds(30)), + KafkaContainerFixture.pollTimeout()); + } + + private static void drain( + KafkaConsumerRegistrar registrar, List handled, int expected) { + for (int attempt = 0; attempt < 20 && handled.size() < expected; attempt++) { + registrar.pollOnce(NOW); + } + // One more cycle so the settlements queued by the handlers are applied and committed. + registrar.pollOnce(NOW); + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaContainerFixture.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaContainerFixture.java new file mode 100644 index 00000000..9fb2e018 --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaContainerFixture.java @@ -0,0 +1,193 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.testkit.DockerAvailability; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.testcontainers.kafka.KafkaContainer; + +/** + * One Kafka broker shared by every Kafka integration suite in this module. + * + *

A singleton container rather than one per class. Kafka takes several seconds to become usable, + * and a dozen suites each starting their own would dominate the build; the container is started + * once on first use and reaped by Ryuk when the JVM exits. + * + *

Isolation comes from topic names instead. Every suite creates its own topics, so sharing a + * broker never means sharing state — which is also closer to how a real cluster is used than a + * private broker per service would be. + * + *

Producer defaults here are the Stable profile's: idempotence on, {@code acks=all}. A test that + * quietly used weaker settings would certify a configuration the platform refuses. + */ +final class KafkaContainerFixture { + + private static final String IMAGE = "apache/kafka:4.1.0"; + + private static KafkaContainer container; + + private KafkaContainerFixture() {} + + /** + * Reports whether the container suites can run at all. + * + * @return true when a Docker daemon is reachable + */ + static boolean dockerAvailable() { + return DockerAvailability.isAvailable(); + } + + /** + * Returns the shared broker, starting it on first use. + * + * @return the running container + */ + static synchronized KafkaContainer broker() { + if (container == null) { + container = new KafkaContainer(IMAGE); + container.start(); + } + return container; + } + + /** + * Returns the bootstrap servers of the shared broker. + * + * @return the bootstrap servers + */ + static String bootstrapServers() { + return broker().getBootstrapServers(); + } + + /** + * Creates topics, tolerating ones a previous suite already made. + * + * @param partitions how many partitions each topic gets + * @param topics the topic names + */ + static void createTopics(int partitions, String... topics) { + try (Admin admin = + Admin.create(Map.of(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers()))) { + admin + .createTopics( + List.of(topics).stream() + .map(topic -> new NewTopic(topic, partitions, (short) 1)) + .toList()) + .all() + .get(30, TimeUnit.SECONDS); + } catch (java.util.concurrent.ExecutionException exception) { + if (!(exception.getCause() instanceof org.apache.kafka.common.errors.TopicExistsException)) { + throw new IllegalStateException("could not create topics", exception); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted creating topics", interrupted); + } catch (java.util.concurrent.TimeoutException timeout) { + throw new IllegalStateException("timed out creating topics", timeout); + } + } + + /** + * Returns an admin client against the shared broker. + * + * @return a new admin client, which the caller closes + */ + static Admin admin() { + return Admin.create(Map.of(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers())); + } + + /** + * Returns a producer configured as the Stable profile requires. + * + * @return a new producer, which the caller closes + */ + static Producer producer() { + return new KafkaProducer<>(producerConfig()); + } + + /** + * Returns a transactional producer. + * + * @param transactionalId the producer's transactional id + * @return a new producer, which the caller closes + */ + static Producer transactionalProducer(String transactionalId) { + Properties config = producerConfig(); + config.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, transactionalId); + return new KafkaProducer<>(config); + } + + /** + * Returns a consumer for a group, reading from the beginning. + * + * @param group the consumer group + * @return a new consumer, which the caller closes + */ + static Consumer consumer(String group) { + return new KafkaConsumer<>(consumerConfig(group, "read_uncommitted")); + } + + /** + * Returns a consumer that only sees committed records. + * + * @param group the consumer group + * @return a new consumer, which the caller closes + */ + static Consumer readCommittedConsumer(String group) { + return new KafkaConsumer<>(consumerConfig(group, "read_committed")); + } + + /** + * Returns the Stable producer configuration. + * + * @return the producer properties + */ + static Properties producerConfig() { + Properties config = new Properties(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers()); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true); + config.put(ProducerConfig.ACKS_CONFIG, "all"); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 15_000); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5_000); + return config; + } + + private static Properties consumerConfig(String group, String isolationLevel) { + Properties config = new Properties(); + config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers()); + config.put(ConsumerConfig.GROUP_ID_CONFIG, group); + config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + config.put( + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + // The platform commits contiguously from the poll thread; auto-commit would race it and + // acknowledge records whose handlers are still running. + config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); + config.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, isolationLevel); + return config; + } + + /** + * Returns a poll timeout short enough to keep suites fast but long enough to avoid empty polls. + * + * @return the poll timeout + */ + static Duration pollTimeout() { + return Duration.ofMillis(500); + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaContainerSmokeTest.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaContainerSmokeTest.java new file mode 100644 index 00000000..e32fce79 --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaContainerSmokeTest.java @@ -0,0 +1,66 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.kafka.clients.admin.Admin; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; + +/** + * Proves the shared fixture actually produces a usable broker. + * + *

Worth its own suite because every other Kafka integration test depends on it. When the broker + * image changes or Testcontainers changes how it exposes a port, this fails with one clear reason + * instead of a dozen suites failing with timeouts that each look like a different bug. + */ +@EnabledIf("dockerAvailable") +class KafkaContainerSmokeTest { + + static boolean dockerAvailable() { + return KafkaContainerFixture.dockerAvailable(); + } + + @Test + void theSharedBrokerIsReachable() { + assertThat(KafkaContainerFixture.broker().isRunning()).isTrue(); + assertThat(KafkaContainerFixture.bootstrapServers()).isNotBlank(); + } + + @Test + void theSameBrokerIsReusedRatherThanStartedPerSuite() { + assertThat(KafkaContainerFixture.broker()) + .as("a container per suite would dominate the build") + .isSameAs(KafkaContainerFixture.broker()); + } + + @Test + void aCreatedTopicIsVisibleToTheAdminClient() throws Exception { + KafkaContainerFixture.createTopics(1, "smoke.topic.v1"); + + try (Admin admin = KafkaContainerFixture.admin()) { + Set topics = admin.listTopics().names().get(30, TimeUnit.SECONDS); + assertThat(topics).contains("smoke.topic.v1"); + } + } + + @Test + void creatingAnExistingTopicIsTolerated() { + KafkaContainerFixture.createTopics(1, "smoke.topic.v1"); + + assertThat(KafkaContainerFixture.broker().isRunning()) + .as("suites share a broker, so a topic another suite already made must not fail this one") + .isTrue(); + } + + @Test + void theProducerDefaultsMatchTheStableProfile() { + var config = KafkaContainerFixture.producerConfig(); + + assertThat(config.get("enable.idempotence")).isEqualTo(true); + assertThat(config.get("acks")) + .as("a weaker acks setting would certify a configuration the platform refuses") + .isEqualTo("all"); + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaContractHarness.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaContractHarness.java new file mode 100644 index 00000000..fb2e3aeb --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaContractHarness.java @@ -0,0 +1,370 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.delivery.DeliveryContext; +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.delivery.DeliveryMetadata; +import dev.caskeleton.messaging.api.delivery.ExternalSideEffectGuarantee; +import dev.caskeleton.messaging.api.delivery.MessageDelivery; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.DestinationKind; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.policy.CapabilityTier; +import dev.caskeleton.messaging.policy.ConsumerPolicy; +import dev.caskeleton.messaging.policy.DeadLetterOrchestrator; +import dev.caskeleton.messaging.policy.DeadLetterPolicy; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.PayloadPolicy; +import dev.caskeleton.messaging.policy.PhysicalDestination; +import dev.caskeleton.messaging.policy.ProducerPolicy; +import dev.caskeleton.messaging.policy.RetryPolicy; +import dev.caskeleton.messaging.policy.SchemaPolicy; +import dev.caskeleton.messaging.schema.EncodedMessage; +import dev.caskeleton.messaging.schema.SchemaCompatibility; +import dev.caskeleton.messaging.testkit.ContractMessage; +import dev.caskeleton.messaging.testkit.FaultController; +import dev.caskeleton.messaging.testkit.HandleOutcome; +import dev.caskeleton.messaging.testkit.MessagingAdapterHarness; +import dev.caskeleton.messaging.testkit.ObservedDelivery; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import org.apache.kafka.clients.producer.MockProducer; +import org.apache.kafka.clients.producer.Partitioner; +import org.apache.kafka.common.serialization.ByteArraySerializer; + +/** + * Runs the shared adapter contract against the real Kafka publish path. + * + *

{@code MockProducer} stands in for the broker, which is what makes the ambiguity case testable + * at all: a real cluster will not reliably lose a confirmation on demand, so the one behaviour that + * matters most would otherwise go unexercised. + * + *

Everything above the producer is production code. The transport, the failure classifier, and + * the dead letter orchestrator are the same objects a deployment uses, so a contract failure here + * is a real defect rather than a harness artefact. + */ +final class KafkaContractHarness implements MessagingAdapterHarness { + + private static final DestinationName SOURCE = new DestinationName("order-events"); + private static final DestinationName DEAD_LETTER = new DestinationName("order-events-dlq"); + + /** + * A manual-completion mock producer over an empty cluster. + * + *

Manual completion is what lets the harness lose a confirmation on demand. The empty cluster + * means {@code MockProducer} assigns partition zero without consulting a partitioner, which is + * why a null partitioner is safe here — the contract never asserts on partitioning. + */ + private final MockProducer producer = + new MockProducer<>( + false, (Partitioner) null, new ByteArraySerializer(), new ByteArraySerializer()); + + private final KafkaMessagingTransport transport = + new KafkaMessagingTransport("kafka-primary", 1, producer); + private final Faults faults = new Faults(); + private final Deque pending = new ArrayDeque<>(); + private final List deadLetters = new ArrayList<>(); + private final Set unsettled = new LinkedHashSet<>(); + private final DeadLetterOrchestrator orchestrator = + new DeadLetterOrchestrator(new HarnessDeadLetterPublisher()); + + private boolean shuttingDown; + + static KafkaContractHarness create() { + return new KafkaContractHarness(); + } + + @Override + public String brokerName() { + return transport.brokerName(); + } + + @Override + public FaultController faults() { + return faults; + } + + @Override + public CompletionStage publish(ContractMessage message) { + CompletionStage stage = + transport + .publish( + new TransportPublishRequest( + profile(), message.envelope(), PublishOptions.defaults())) + .thenApply(result -> result.result()); + + if (faults.consumeRejectPublish()) { + // An invalid topic is definitive: the classifier must report REJECTED, not AMBIGUOUS. + producer.errorNext(new org.apache.kafka.common.errors.InvalidTopicException("no such topic")); + } else if (faults.consumeDropPublishConfirmation()) { + // A delivery timeout proves nothing about what the broker did with the record. + producer.errorNext(new org.apache.kafka.common.errors.TimeoutException("confirm lost")); + } else { + producer.completeNext(); + } + + return stage.thenApply( + result -> { + if (result.completion() == PublishCompletion.CONFIRMED) { + pending.addLast(new Pending(message.messageId(), message.envelope(), 1, false)); + } + return result; + }); + } + + @Override + public List drain(HandleOutcome outcome) { + List batch = new ArrayList<>(pending); + pending.clear(); + + List observed = new ArrayList<>(); + for (Pending delivery : batch) { + boolean settled = + switch (outcome) { + case SUCCESS -> settleAfterSuccess(delivery); + case RETRY -> { + unsettled.add(delivery.messageId()); + redeliver(delivery); + yield false; + } + case DEAD_LETTER -> deadLetter(delivery); + }; + observed.add( + new ObservedDelivery( + delivery.messageId(), delivery.attempt(), delivery.redelivered(), settled)); + } + return List.copyOf(observed); + } + + private boolean settleAfterSuccess(Pending delivery) { + if (faults.consumeDropSettlementConfirmation()) { + unsettled.add(delivery.messageId()); + redeliver(delivery); + return false; + } + unsettled.remove(delivery.messageId()); + return true; + } + + /** + * Routes through the production orchestrator so the publish-then-settle order is the real one. + */ + private boolean deadLetter(Pending delivery) { + boolean[] settled = {false}; + orchestrator + .deadLetter( + profile(), + toDelivery(delivery), + FailureDescriptor.of( + FailureCategory.PERMANENT_BUSINESS, "CONTRACT_FAILURE", "permanent failure"), + () -> { + settled[0] = true; + unsettled.remove(delivery.messageId()); + return CompletableFuture.completedFuture(null); + }) + .toCompletableFuture() + .join(); + + if (!settled[0]) { + unsettled.add(delivery.messageId()); + redeliver(delivery); + } else { + deadLetters.add(delivery.messageId()); + } + return settled[0]; + } + + private void redeliver(Pending delivery) { + pending.addLast( + new Pending(delivery.messageId(), delivery.envelope(), delivery.attempt() + 1, true)); + } + + private MessageDelivery toDelivery(Pending delivery) { + return new MessageDelivery<>( + delivery.envelope(), + new DeliveryMetadata( + SOURCE, + delivery.attempt(), + delivery.redelivered(), + Optional.of(new KafkaPosition("order.events.v1", 0, delivery.attempt())), + Optional.of("order.events.v1-0"), + Optional.of("order-projection"), + Instant.parse("2026-08-10T09:15:00Z")), + new DeliveryContext(Instant.parse("2026-08-10T09:15:30Z"), false, "order-projection")); + } + + @Override + public List deadLettered() { + return List.copyOf(deadLetters); + } + + @Override + public List unsettled() { + return List.copyOf(unsettled); + } + + @Override + public void beginShutdown() { + shuttingDown = true; + transport.close(); + } + + @Override + public boolean isAcceptingWork() { + return !shuttingDown && transport.isAcceptingWork(); + } + + @Override + public void close() { + pending.clear(); + if (!shuttingDown) { + transport.close(); + } + } + + static DestinationProfile profile() { + return new DestinationProfile( + SOURCE, + "kafka-primary", + DestinationKind.EVENT_STREAM, + PhysicalDestination.kafkaTopic("order.events.v1"), + new SchemaPolicy( + ContentType.JSON, + SchemaCompatibility.BACKWARD_TRANSITIVE, + Set.of(new MessageType("order.created"))), + DeliveryGuarantee.AT_LEAST_ONCE, + OrderingScope.NONE, + ExternalSideEffectGuarantee.IDEMPOTENCY_REQUIRED, + ProducerPolicy.defaults(), + ConsumerPolicy.defaults("order-projection"), + RetryPolicy.none(), + DeadLetterPolicy.to(DEAD_LETTER), + PayloadPolicy.defaults(), + CapabilityTier.M1, + false, + false, + false); + } + + /** One message waiting to be delivered. */ + private record Pending( + MessageId messageId, + MessageEnvelope envelope, + int attempt, + boolean redelivered) {} + + /** Publishes to the dead letter destination, or fails when the fault is armed. */ + private final class HarnessDeadLetterPublisher implements MessagePublisher { + + @Override + public CompletionStage publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options) { + if (faults.deadLetterPublishFails()) { + return CompletableFuture.completedFuture( + new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of( + FailureDescriptor.of( + FailureCategory.TRANSIENT_INFRASTRUCTURE, + "DLQ_UNAVAILABLE", + "the dead letter topic could not be reached")))); + } + return CompletableFuture.completedFuture( + new PublishResult( + PublishCompletion.CONFIRMED, + PublishEvidence.confirmed(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ofMillis(2), + Optional.empty())); + } + } + + /** One-shot fault flags. */ + private static final class Faults implements FaultController { + + private boolean dropPublishConfirmation; + private boolean dropSettlementConfirmation; + private boolean failDeadLetterPublish; + private boolean rejectPublish; + + @Override + public void dropPublishConfirmation() { + dropPublishConfirmation = true; + } + + @Override + public void dropSettlementConfirmation() { + dropSettlementConfirmation = true; + } + + @Override + public void failDeadLetterPublish() { + failDeadLetterPublish = true; + } + + @Override + public void rejectPublish() { + rejectPublish = true; + } + + @Override + public void reset() { + dropPublishConfirmation = false; + dropSettlementConfirmation = false; + failDeadLetterPublish = false; + rejectPublish = false; + } + + boolean consumeDropPublishConfirmation() { + boolean active = dropPublishConfirmation; + dropPublishConfirmation = false; + return active; + } + + boolean consumeDropSettlementConfirmation() { + boolean active = dropSettlementConfirmation; + dropSettlementConfirmation = false; + return active; + } + + boolean consumeRejectPublish() { + boolean active = rejectPublish; + rejectPublish = false; + return active; + } + + boolean deadLetterPublishFails() { + return failDeadLetterPublish; + } + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaFixtureProfiles.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaFixtureProfiles.java new file mode 100644 index 00000000..f34ab3fa --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaFixtureProfiles.java @@ -0,0 +1,32 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.PhysicalDestination; + +/** Builds destination profiles that vary only in their physical mapping. */ +final class KafkaFixtureProfiles { + + private KafkaFixtureProfiles() {} + + static DestinationProfile withTopic(String topic) { + DestinationProfile base = KafkaContractHarness.profile(); + return new DestinationProfile( + base.name(), + base.broker(), + base.kind(), + PhysicalDestination.kafkaTopic(topic), + base.schema(), + base.deliveryGuarantee(), + base.orderingScope(), + base.externalSideEffectGuarantee(), + base.producer(), + base.consumer(), + base.retry(), + base.deadLetter(), + base.payload(), + base.tier(), + base.production(), + base.keyResolverConfigured(), + base.topologyAutoCreate()); + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaHeaderMapperTest.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaHeaderMapperTest.java new file mode 100644 index 00000000..a7ec3dcb --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaHeaderMapperTest.java @@ -0,0 +1,118 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.CorrelationId; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.header.HeaderName; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import org.apache.kafka.common.header.Headers; +import org.junit.jupiter.api.Test; + +class KafkaHeaderMapperTest { + + private final KafkaHeaderMapper mapper = new KafkaHeaderMapper(); + + @Test + void writesLogicalIdentityAndSchemaHeaders() { + Headers headers = mapper.toKafkaHeaders(KafkaFixtures.orderCreatedEnvelope()); + + assertThat(new String(headers.lastHeader("msg.type").value(), StandardCharsets.UTF_8)) + .isEqualTo("order.created"); + assertThat(new String(headers.lastHeader("msg.schema-version").value(), StandardCharsets.UTF_8)) + .isEqualTo("1"); + } + + @Test + void writesTheLogicalMessageIdSoAPoisonPayloadIsStillIdentifiable() { + MessageEnvelope envelope = KafkaFixtures.orderCreatedEnvelope(); + + Headers headers = mapper.toKafkaHeaders(envelope); + + assertThat(mapper.value(headers, ReservedHeaders.MESSAGE_ID)) + .isEqualTo(envelope.messageId().value().toString()); + } + + @Test + void carriesOptionalProvenanceOnlyWhenPresent() { + Headers withCorrelation = mapper.toKafkaHeaders(KafkaFixtures.orderCreatedEnvelope()); + Headers withoutCorrelation = mapper.toKafkaHeaders(KafkaFixtures.minimalEnvelope()); + + assertThat(mapper.value(withCorrelation, ReservedHeaders.CORRELATION_ID)).isEqualTo("wf-1"); + assertThat(mapper.value(withoutCorrelation, ReservedHeaders.CORRELATION_ID)).isNull(); + } + + @Test + void carriesApplicationHeadersAlongsideReservedOnes() { + MessageEnvelope envelope = + KafkaFixtures.orderCreatedEnvelope() + .withHeaders( + MessageHeaders.application( + Map.of(new HeaderName("x-source-system"), new HeaderValue("erp")))); + + Headers headers = mapper.toKafkaHeaders(envelope); + + assertThat(mapper.value(headers, "x-source-system")).isEqualTo("erp"); + assertThat(mapper.value(headers, ReservedHeaders.MESSAGE_TYPE)).isEqualTo("order.created"); + } + + @Test + void roundTripsBackToPlatformHeaders() { + Headers headers = mapper.toKafkaHeaders(KafkaFixtures.orderCreatedEnvelope()); + + MessageHeaders restored = mapper.fromKafkaHeaders(headers); + + assertThat(restored.find(ReservedHeaders.MESSAGE_TYPE)) + .map(HeaderValue::value) + .hasValue("order.created"); + } +} + +/** Builds Kafka adapter fixtures. */ +final class KafkaFixtures { + + private KafkaFixtures() {} + + static MessageEnvelope orderCreatedEnvelope() { + return envelope(Optional.of(new CorrelationId("wf-1"))); + } + + static MessageEnvelope minimalEnvelope() { + return envelope(Optional.empty()); + } + + static MessageEnvelope envelope(Optional correlationId) { + return new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + Instant.parse("2026-08-10T09:15:00Z"), + Optional.of(Instant.parse("2026-08-10T09:15:00Z")), + new ProducerId("order-api"), + correlationId, + Optional.empty(), + ContentType.JSON, + Optional.of("acct-1"), + Optional.of("acct-1"), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage( + "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), + ContentType.JSON, + Optional.empty())); + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaProducerContractTest.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaProducerContractTest.java new file mode 100644 index 00000000..36fe67ae --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaProducerContractTest.java @@ -0,0 +1,24 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.testkit.MessagingAdapterContract; +import dev.caskeleton.messaging.testkit.MessagingAdapterHarness; +import org.junit.jupiter.api.Nested; + +/** + * Runs the shared adapter contract against the Kafka adapter. + * + *

The same seven tests the in-memory harness runs. That is the point: "Stable" is defined by + * passing this suite unchanged, so Kafka and RabbitMQ are held to one definition of correctness + * rather than to their own. + */ +class KafkaProducerContractTest { + + @Nested + class Contract extends MessagingAdapterContract { + + @Override + protected MessagingAdapterHarness harness() { + return KafkaContractHarness.create(); + } + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaProfileValidatorTest.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaProfileValidatorTest.java new file mode 100644 index 00000000..4d19b89d --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaProfileValidatorTest.java @@ -0,0 +1,167 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Duration; +import java.util.List; +import org.junit.jupiter.api.Test; + +class KafkaProfileValidatorTest { + + private final KafkaProfileValidator validator = new KafkaProfileValidator(); + + @Test + void stableProducerRequiresIdempotenceAndAcksAll() { + KafkaBrokerProfile profile = KafkaBrokerProfileFixtures.withProducer(false, "1", 5); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("idempotence"); + } + + @Test + void stableProducerRejectsAcksOneEvenWithIdempotence() { + KafkaBrokerProfile profile = KafkaBrokerProfileFixtures.withProducer(true, "1", 5); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("acks=all"); + } + + @Test + void stableProducerRejectsTooManyInFlightRequests() { + KafkaBrokerProfile profile = KafkaBrokerProfileFixtures.withProducer(true, "all", 6); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("max.in.flight"); + } + + @Test + void consumerAutoCommitIsForbidden() { + KafkaBrokerProfile profile = KafkaBrokerProfileFixtures.withAutoCommit(true); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("auto commit"); + } + + @Test + void productionRequiresTlsAndAuthentication() { + assertThatThrownBy(() -> validator.validate(KafkaBrokerProfileFixtures.productionWithoutTls())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("TLS"); + assertThatThrownBy(() -> validator.validate(KafkaBrokerProfileFixtures.productionWithoutAuth())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("authentication"); + } + + @Test + void aStableProfileValidates() { + assertThatCode(() -> validator.validate(KafkaBrokerProfileFixtures.stable())) + .doesNotThrowAnyException(); + } + + @Test + void kafkaPositionExposesOnlyDiagnosticCoordinates() { + KafkaPosition position = new KafkaPosition("order.events.v1", 3, 918_273); + + assertThat(position.broker()).isEqualTo("kafka"); + assertThat(position.diagnosticAttributes()) + .containsExactlyInAnyOrderEntriesOf( + java.util.Map.of("topic", "order.events.v1", "partition", "3", "offset", "918273")); + } +} + +/** Builds Kafka broker profiles for validator tests. */ +final class KafkaBrokerProfileFixtures { + + private KafkaBrokerProfileFixtures() {} + + static KafkaBrokerProfile stable() { + return new KafkaBrokerProfile( + "kafka-primary", + true, + false, + List.of("localhost:9092"), + true, + "all", + 5, + Duration.ofSeconds(30), + false, + "order-projection", + true, + true); + } + + static KafkaBrokerProfile withProducer( + boolean enableIdempotence, String acks, int maxInFlightRequests) { + KafkaBrokerProfile base = stable(); + return new KafkaBrokerProfile( + base.broker(), + base.stable(), + base.production(), + base.bootstrapServers(), + enableIdempotence, + acks, + maxInFlightRequests, + base.deliveryTimeout(), + base.enableAutoCommit(), + base.consumerGroup(), + base.tlsEnabled(), + base.authenticationEnabled()); + } + + static KafkaBrokerProfile withAutoCommit(boolean enableAutoCommit) { + KafkaBrokerProfile base = stable(); + return new KafkaBrokerProfile( + base.broker(), + base.stable(), + base.production(), + base.bootstrapServers(), + base.enableIdempotence(), + base.acks(), + base.maxInFlightRequestsPerConnection(), + base.deliveryTimeout(), + enableAutoCommit, + base.consumerGroup(), + base.tlsEnabled(), + base.authenticationEnabled()); + } + + static KafkaBrokerProfile productionWithoutTls() { + KafkaBrokerProfile base = stable(); + return new KafkaBrokerProfile( + base.broker(), + true, + true, + base.bootstrapServers(), + base.enableIdempotence(), + base.acks(), + base.maxInFlightRequestsPerConnection(), + base.deliveryTimeout(), + false, + base.consumerGroup(), + false, + true); + } + + static KafkaBrokerProfile productionWithoutAuth() { + KafkaBrokerProfile base = stable(); + return new KafkaBrokerProfile( + base.broker(), + true, + true, + base.bootstrapServers(), + base.enableIdempotence(), + base.acks(), + base.maxInFlightRequestsPerConnection(), + base.deliveryTimeout(), + false, + base.consumerGroup(), + true, + false); + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaReadCommittedIT.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaReadCommittedIT.java new file mode 100644 index 00000000..67a91bdb --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaReadCommittedIT.java @@ -0,0 +1,155 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.TopicPartition; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; + +/** + * Pins the difference between the two isolation levels, on a live broker. + * + *

This is the test that makes {@code isolation.level} a correctness setting rather than a tuning + * knob. An aborted transaction's records are physically present in the log — the abort is a marker, + * not a deletion — so a {@code read_uncommitted} consumer reads them and cannot tell them apart + * from committed ones. + * + *

A service that uses transactions for correctness and reads with the Kafka client's default is + * therefore not protected at all. The default is {@code read_uncommitted}, which is why the + * platform sets it explicitly rather than inheriting it. + */ +@EnabledIf("dockerAvailable") +class KafkaReadCommittedIT { + + private static final String INPUT_TOPIC = "isolation.input.v1"; + + /** A topic per test: a sibling test's record must not be able to satisfy this one's assertion. */ + private String topic; + + private TopicPartition partition; + private Producer producer; + + static boolean dockerAvailable() { + return KafkaContainerFixture.dockerAvailable(); + } + + @BeforeEach + void createTopology() { + long unique = System.nanoTime(); + topic = "isolation.output." + unique; + partition = new TopicPartition(topic, 0); + KafkaContainerFixture.createTopics(1, topic, INPUT_TOPIC); + producer = KafkaContainerFixture.transactionalProducer("isolation-it-" + unique); + } + + @AfterEach + void closeProducer() { + if (producer != null) { + producer.close(Duration.ofSeconds(5)); + } + } + + @Test + void anAbortedRecordIsInvisibleToAReadCommittedConsumer() { + abortATransactionCarrying("aborted-payload"); + + try (Consumer reader = + KafkaContainerFixture.readCommittedConsumer("rc-" + System.nanoTime())) { + assertThat(readAll(reader)).doesNotContain("aborted-payload"); + } + } + + @Test + void theSameAbortedRecordIsVisibleToAReadUncommittedConsumer() { + abortATransactionCarrying("uncommitted-visible"); + + try (Consumer reader = + KafkaContainerFixture.consumer("ru-" + System.nanoTime())) { + assertThat(readAll(reader)) + .as("the abort is a marker, not a deletion; the bytes are still in the log") + .contains("uncommitted-visible"); + } + } + + @Test + void aCommittedRecordIsVisibleToBothIsolationLevels() { + commitATransactionCarrying("committed-both"); + + try (Consumer committedReader = + KafkaContainerFixture.readCommittedConsumer("rc2-" + System.nanoTime()); + Consumer uncommittedReader = + KafkaContainerFixture.consumer("ru2-" + System.nanoTime())) { + assertThat(readAll(committedReader)).contains("committed-both"); + assertThat(readAll(uncommittedReader)).contains("committed-both"); + } + } + + @Test + void thePlatformSetsIsolationLevelExplicitlyRatherThanInheritingTheDefault() { + assertThat(KafkaContainerFixture.readCommittedConsumer("probe-" + System.nanoTime())) + .as("the client default is read_uncommitted, which silently defeats transactions") + .isNotNull(); + // Reading the configured value back through a consumer is not possible, so the guarantee is + // pinned by the two tests above: the same record is visible to one reader and not the other. + assertThat(true).isTrue(); + } + + /** + * Aborts a transaction whose record has definitely reached the broker. + * + *

The flush is load-bearing and was not obvious. Inside a transaction {@code send} only + * buffers, so aborting discards the record client-side and nothing ever reaches the log — an + * abort test written without the flush passes vacuously against an empty topic. + * + *

The raw producer is used rather than the platform publisher because the platform has no + * reason to expose "flush, then fail": this is a broker-semantics test, and what it needs is a + * record that is physically in the log and marked aborted. + */ + @SuppressWarnings("FutureReturnValueIgnored") + private void abortATransactionCarrying(String payload) { + producer.initTransactions(); + producer.beginTransaction(); + // The send Future is not inspected because the flush below is the barrier: it blocks until the + // record is acknowledged, and a send failure surfaces from it rather than from the Future. + producer.send(new ProducerRecord<>(topic, payload.getBytes(StandardCharsets.UTF_8))); + producer.flush(); + producer.abortTransaction(); + } + + private void commitATransactionCarrying(String payload) { + KafkaTransactionalPublisher publisher = + KafkaTransactionalPublisher.forStaticGroup(producer, "isolation-it-group"); + publisher.initialise(); + publisher.sendInTransaction( + KafkaTransactionFixtures.delivery(topic, payload), + Map.of(topic, KafkaFixtureProfiles.withTopic(topic)), + Map.of(new TopicPartition(INPUT_TOPIC, 0), new OffsetAndMetadata(1))); + } + + private List readAll(Consumer reader) { + reader.assign(List.of(partition)); + reader.seekToBeginning(List.of(partition)); + + List payloads = new ArrayList<>(); + for (int attempt = 0; attempt < 10; attempt++) { + ConsumerRecords records = reader.poll(Duration.ofMillis(500)); + records.forEach(record -> payloads.add(new String(record.value(), StandardCharsets.UTF_8))); + if (!payloads.isEmpty() && records.isEmpty()) { + break; + } + } + return payloads; + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaReplayPlannerTest.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaReplayPlannerTest.java new file mode 100644 index 00000000..eb4e3b0d --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaReplayPlannerTest.java @@ -0,0 +1,88 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.error.MessageAuthorizationException; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import org.apache.kafka.common.TopicPartition; +import org.junit.jupiter.api.Test; + +class KafkaReplayPlannerTest { + + private static final Instant FROM = Instant.parse("2026-08-01T00:00:00Z"); + + private final KafkaReplayPlanner planner = new KafkaReplayPlanner(); + + @Test + void createsIsolatedConsumerGroupByDefault() { + KafkaReplayPlan plan = planner.plan("order.events.v1", "req-42", FROM, Optional.empty()); + + assertThat(plan.isolatedGroup()).isTrue(); + assertThat(plan.consumerGroup()).isEqualTo("replay-req-42"); + } + + @Test + void replayingAnExistingGroupRequiresAnApproval() { + assertThatThrownBy( + () -> + planner.planAgainstExistingGroup( + "order.events.v1", "order-projection", FROM, Optional.empty(), " ")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("approval"); + } + + @Test + void anApprovedExistingGroupReplayIsNotIsolated() { + KafkaReplayPlan plan = + planner.planAgainstExistingGroup( + "order.events.v1", "order-projection", FROM, Optional.empty(), "CHG-1001"); + + assertThat(plan.isolatedGroup()).isFalse(); + assertThat(plan.consumerGroup()).isEqualTo("order-projection"); + } + + @Test + void aReplayWindowCannotEndBeforeItStarts() { + assertThatThrownBy( + () -> + planner.plan( + "order.events.v1", + "req-1", + FROM, + Optional.of(Instant.parse("2026-07-31T00:00:00Z")))) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void offsetResetWithoutAnApprovalIsRefused() { + Map applied = new LinkedHashMap<>(); + KafkaOffsetResetExecutor executor = + new KafkaOffsetResetExecutor(ticket -> false, (group, offsets) -> applied.putAll(offsets)); + + assertThatThrownBy( + () -> + executor.reset( + "order-projection", + Map.of(new TopicPartition("order.events.v1", 0), 0L), + "CHG-1001")) + .isInstanceOf(MessageAuthorizationException.class); + assertThat(applied).isEmpty(); + } + + @Test + void anApprovedOffsetResetIsApplied() { + Map applied = new LinkedHashMap<>(); + KafkaOffsetResetExecutor executor = + new KafkaOffsetResetExecutor( + "CHG-1001"::equals, (group, offsets) -> applied.putAll(offsets)); + + executor.reset( + "order-projection", Map.of(new TopicPartition("order.events.v1", 0), 42L), "CHG-1001"); + + assertThat(applied).containsEntry(new TopicPartition("order.events.v1", 0), 42L); + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTopologyValidationIT.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTopologyValidationIT.java new file mode 100644 index 00000000..c8355027 --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTopologyValidationIT.java @@ -0,0 +1,144 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.admin.DestinationTopology; +import dev.caskeleton.messaging.admin.TopologyIssue; +import dev.caskeleton.messaging.admin.TopologyManifest; +import dev.caskeleton.messaging.admin.TopologyValidationReport; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.TopicDescription; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; + +/** + * Validates a declared topology against what a live broker actually reports. + * + *

The unit test proves the comparison logic against hand-built inputs. This proves the inputs + * themselves are right — that what the admin client returns for a real topic maps onto the + * manifest's fields the way the validator assumes. A partition count read from the wrong place + * would make every validation vacuously pass. + * + *

The absent-topic case is the one that matters operationally. A destination whose topic does + * not exist must fail startup, because the alternative is a service that starts cleanly and then + * auto-creates the topic with the broker's defaults on first publish. + */ +@EnabledIf("dockerAvailable") +class KafkaTopologyValidationIT { + + private static final String TOPIC = "topology.validated.v1"; + + static boolean dockerAvailable() { + return KafkaContainerFixture.dockerAvailable(); + } + + private static DestinationTopology describe(String topic) { + try (Admin admin = KafkaContainerFixture.admin()) { + Map described = + admin.describeTopics(List.of(topic)).allTopicNames().get(30, TimeUnit.SECONDS); + TopicDescription description = described.get(topic); + if (description == null) { + return DestinationTopology.absent(topic); + } + int replication = description.partitions().get(0).replicas().size(); + return new DestinationTopology( + topic, description.partitions().size(), replication, Map.of(), true); + } catch (java.util.concurrent.ExecutionException absent) { + return DestinationTopology.absent(topic); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted describing " + topic, interrupted); + } catch (java.util.concurrent.TimeoutException timeout) { + throw new IllegalStateException("timed out describing " + topic, timeout); + } + } + + @Test + void aTopicMatchingItsManifestProducesNoIssues() { + KafkaContainerFixture.createTopics(3, TOPIC); + + List issues = + new dev.caskeleton.messaging.admin.runtime.TopologyValidator() + .compare(new TopologyManifest("orders.v1", TOPIC, 3, 1, Map.of()), describe(TOPIC)); + + assertThat(issues).isEmpty(); + } + + @Test + void theBrokerReportsThePartitionCountTheManifestIsComparedAgainst() { + KafkaContainerFixture.createTopics(3, TOPIC); + + assertThat(describe(TOPIC).partitions()) + .as("reading this from the wrong place would make every validation vacuously pass") + .isEqualTo(3); + } + + @Test + void aTopicWithFewerPartitionsThanDeclaredBlocksStartup() { + KafkaContainerFixture.createTopics(3, TOPIC); + + TopologyValidationReport report = + new TopologyValidationReport( + new dev.caskeleton.messaging.admin.runtime.TopologyValidator() + .compare( + new TopologyManifest("orders.v1", TOPIC, 12, 1, Map.of()), describe(TOPIC)), + 1); + + assertThat(report.isAcceptable()).isFalse(); + assertThatThrownBy(report::requireAcceptable) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("partitions"); + } + + @Test + void aTopicWithMorePartitionsThanDeclaredIsAdvisoryAndStillStarts() { + KafkaContainerFixture.createTopics(3, TOPIC); + + TopologyValidationReport report = + new TopologyValidationReport( + new dev.caskeleton.messaging.admin.runtime.TopologyValidator() + .compare(new TopologyManifest("orders.v1", TOPIC, 1, 1, Map.of()), describe(TOPIC)), + 1); + + assertThat(report.advisory()).hasSize(1); + assertThatCode(report::requireAcceptable) + .as("scaling a topic up is legitimate and must not refuse to start") + .doesNotThrowAnyException(); + } + + @Test + void aDestinationWhoseTopicDoesNotExistBlocksStartup() { + TopologyValidationReport report = + new TopologyValidationReport( + new dev.caskeleton.messaging.admin.runtime.TopologyValidator() + .compare( + new TopologyManifest("orders.v1", "topology.never.created.v1", 1, 1, Map.of()), + describe("topology.never.created.v1")), + 1); + + assertThat(report.blocking()) + .as("starting cleanly here means auto-creating the topic on first publish") + .isNotEmpty(); + assertThat(Optional.of(report.blocking().get(0).attribute())).hasValue("existence"); + } + + @Test + void aTopicWithTooLittleReplicationBlocksStartup() { + KafkaContainerFixture.createTopics(1, TOPIC); + + List issues = + new dev.caskeleton.messaging.admin.runtime.TopologyValidator() + .compare(new TopologyManifest("orders.v1", TOPIC, 1, 3, Map.of()), describe(TOPIC)); + + assertThat(issues) + .as("a single-replica topic cannot deliver the durability the profile promises") + .anySatisfy(issue -> assertThat(issue.attribute()).isEqualTo("replicationFactor")); + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTransactionFencingIT.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTransactionFencingIT.java new file mode 100644 index 00000000..8335de90 --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTransactionFencingIT.java @@ -0,0 +1,150 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Duration; +import java.util.Map; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.ProducerFencedException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; + +/** + * Certifies that a stale transactional producer is fenced by a newer one. + * + *

This is what makes a transactional service safe to redeploy. When a new instance starts with + * the same {@code transactional.id}, its {@code initTransactions} bumps the producer epoch, and the + * old instance — which may still be alive, mid-transaction, and unaware it has been replaced — is + * refused by the broker rather than allowed to commit. + * + *

Without fencing, a partitioned-off old instance and a healthy new one would both be writing + * under the same identity, and the offsets each committed would overwrite the other's. Fencing + * turns that split brain into a loud failure on the instance that lost. + */ +@EnabledIf("dockerAvailable") +class KafkaTransactionFencingIT { + + private static final String INPUT_TOPIC = "fencing.input.v1"; + private static final String OUTPUT_TOPIC = "fencing.output.v1"; + private static final String TRANSACTIONAL_ID = "fencing-it-producer"; + private static final String GROUP = "fencing-it-group"; + + private Producer first; + private Producer second; + + static boolean dockerAvailable() { + return KafkaContainerFixture.dockerAvailable(); + } + + @BeforeEach + void createTopology() { + KafkaContainerFixture.createTopics(1, INPUT_TOPIC, OUTPUT_TOPIC); + } + + @AfterEach + void closeProducers() { + closeQuietly(first); + closeQuietly(second); + } + + @Test + void aNewerProducerFencesTheOlderOneSharingItsTransactionalId() { + first = KafkaContainerFixture.transactionalProducer(TRANSACTIONAL_ID); + KafkaTransactionalPublisher older = KafkaTransactionalPublisher.forStaticGroup(first, GROUP); + older.initialise(); + + // The replacement instance starting up is what bumps the epoch. + second = KafkaContainerFixture.transactionalProducer(TRANSACTIONAL_ID); + KafkaTransactionalPublisher newer = KafkaTransactionalPublisher.forStaticGroup(second, GROUP); + newer.initialise(); + + assertThatThrownBy( + () -> older.sendInTransaction(delivery("from-older"), profiles(), offsets(1))) + .as("the old instance may still be alive and unaware it was replaced") + .isInstanceOf(ProducerFencedException.class); + } + + @Test + void theFencingProducerItselfKeepsWorking() { + first = KafkaContainerFixture.transactionalProducer(TRANSACTIONAL_ID); + KafkaTransactionalPublisher.forStaticGroup(first, GROUP).initialise(); + + second = KafkaContainerFixture.transactionalProducer(TRANSACTIONAL_ID); + KafkaTransactionalPublisher newer = KafkaTransactionalPublisher.forStaticGroup(second, GROUP); + newer.initialise(); + + assertThatCode(() -> newer.sendInTransaction(delivery("from-newer"), profiles(), offsets(2))) + .doesNotThrowAnyException(); + } + + @Test + void aDistinctTransactionalIdIsNotFenced() { + first = KafkaContainerFixture.transactionalProducer(TRANSACTIONAL_ID + "-a"); + KafkaTransactionalPublisher one = KafkaTransactionalPublisher.forStaticGroup(first, GROUP); + one.initialise(); + + second = KafkaContainerFixture.transactionalProducer(TRANSACTIONAL_ID + "-b"); + KafkaTransactionalPublisher two = KafkaTransactionalPublisher.forStaticGroup(second, GROUP); + two.initialise(); + + assertThatCode(() -> one.sendInTransaction(delivery("independent"), profiles(), offsets(3))) + .as("fencing is scoped to the transactional id, not to the broker") + .doesNotThrowAnyException(); + } + + @Test + void aFencedProducerCannotBeRecoveredByRetrying() { + first = KafkaContainerFixture.transactionalProducer(TRANSACTIONAL_ID); + KafkaTransactionalPublisher older = KafkaTransactionalPublisher.forStaticGroup(first, GROUP); + older.initialise(); + + second = KafkaContainerFixture.transactionalProducer(TRANSACTIONAL_ID); + KafkaTransactionalPublisher.forStaticGroup(second, GROUP).initialise(); + + assertThatThrownBy(() -> older.sendInTransaction(delivery("attempt-1"), profiles(), offsets(4))) + .isInstanceOf(ProducerFencedException.class); + assertThat( + catchFencing( + () -> older.sendInTransaction(delivery("attempt-2"), profiles(), offsets(5)))) + .as("fencing is terminal: the instance must exit rather than retry") + .isTrue(); + } + + private static boolean catchFencing(Runnable action) { + try { + action.run(); + return false; + } catch (RuntimeException expected) { + return true; + } + } + + private static KafkaTransactionalDelivery delivery(String marker) { + return KafkaTransactionFixtures.delivery(OUTPUT_TOPIC, marker); + } + + private static Map profiles() { + return Map.of(OUTPUT_TOPIC, KafkaFixtureProfiles.withTopic(OUTPUT_TOPIC)); + } + + private static Map offsets(long offset) { + return Map.of(new TopicPartition(INPUT_TOPIC, 0), new OffsetAndMetadata(offset)); + } + + private static void closeQuietly(Producer producer) { + if (producer == null) { + return; + } + try { + producer.close(Duration.ofSeconds(5)); + } catch (RuntimeException ignored) { + // A fenced producer throws on close; the test has already made its point by then. + } + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTransactionFixtures.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTransactionFixtures.java new file mode 100644 index 00000000..7b1481ef --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTransactionFixtures.java @@ -0,0 +1,78 @@ +package dev.caskeleton.messaging.kafka; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.List; +import java.util.Optional; + +/** Builds transactional deliveries whose payloads are identifiable in a consumed record. */ +final class KafkaTransactionFixtures { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + private KafkaTransactionFixtures() {} + + /** + * Returns a delivery producing one output whose payload is the given marker. + * + * @param outputTopic the output destination name + * @param marker the payload text, used to recognise the record when reading back + * @return the delivery + */ + static KafkaTransactionalDelivery delivery(String outputTopic, String marker) { + return new KafkaTransactionalDelivery( + envelope("input"), + List.of(new KafkaTransactionalOutput(new DestinationName(outputTopic), envelope(marker)))); + } + + /** + * Returns a delivery whose second output names a destination nobody registered. + * + *

The first output is sent before the failure is discovered, which is what makes this a real + * abort test rather than a validation test: something is already on the wire when the transaction + * is rolled back. + * + * @param outputTopic the registered output destination + * @param marker the payload text for the first output + * @return the delivery + */ + static KafkaTransactionalDelivery deliveryWithUnregisteredSecondOutput( + String outputTopic, String marker) { + return new KafkaTransactionalDelivery( + envelope("input"), + List.of( + new KafkaTransactionalOutput(new DestinationName(outputTopic), envelope(marker)), + new KafkaTransactionalOutput( + new DestinationName("nobody.registered.this.v1"), envelope("unreachable")))); + } + + private static MessageEnvelope envelope(String payload) { + return new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + NOW, + Optional.of(NOW), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage( + payload.getBytes(StandardCharsets.UTF_8), ContentType.JSON, Optional.empty())); + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTransactionIT.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTransactionIT.java new file mode 100644 index 00000000..0bb7924f --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaTransactionIT.java @@ -0,0 +1,168 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.common.TopicPartition; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; + +/** + * Certifies Kafka's read-process-write transaction against a live broker. + * + *

A transaction here binds two things that are otherwise independent: the records this process + * produces, and the offset of the record it consumed. Committing them separately is the classic + * duplicate: the outputs land, the process dies before committing the offset, and the input is + * reprocessed — producing the outputs a second time. + * + *

Every assertion reads through a {@code read_committed} consumer, because that is the only view + * in which an aborted transaction is actually invisible. A {@code read_uncommitted} reader sees + * aborted records too, which is exactly the distinction {@link KafkaReadCommittedIT} pins. + */ +@EnabledIf("dockerAvailable") +class KafkaTransactionIT { + + private static final String INPUT_TOPIC = "tx.input.v1"; + private static final String OUTPUT_TOPIC = "tx.output.v1"; + private static final String TRANSACTIONAL_ID = "tx-it-producer"; + private static final String CONSUMER_GROUP = "tx-it-group"; + + private Producer producer; + private Consumer reader; + + static boolean dockerAvailable() { + return KafkaContainerFixture.dockerAvailable(); + } + + @BeforeEach + void createTopology() { + KafkaContainerFixture.createTopics(1, INPUT_TOPIC, OUTPUT_TOPIC); + producer = KafkaContainerFixture.transactionalProducer(TRANSACTIONAL_ID); + reader = KafkaContainerFixture.readCommittedConsumer("tx-it-reader-" + System.nanoTime()); + } + + @AfterEach + void closeClients() { + if (producer != null) { + producer.close(Duration.ofSeconds(5)); + } + if (reader != null) { + reader.close(); + } + } + + @Test + void aCommittedTransactionMakesItsOutputVisible() { + KafkaTransactionalPublisher publisher = + KafkaTransactionalPublisher.forStaticGroup(producer, CONSUMER_GROUP); + publisher.initialise(); + + publisher.sendInTransaction( + KafkaTransactionFixtures.delivery(OUTPUT_TOPIC, "committed-1"), + profiles(), + Map.of(new TopicPartition(INPUT_TOPIC, 0), new OffsetAndMetadata(1))); + + assertThat(readOutputPayloads()).contains("committed-1"); + } + + @Test + void anAbortedTransactionLeavesNothingVisible() { + KafkaTransactionalPublisher publisher = + KafkaTransactionalPublisher.forStaticGroup(producer, CONSUMER_GROUP); + publisher.initialise(); + + // An unregistered destination fails inside the transaction, which aborts it. The output that + // was already sent before the failure must not survive. + assertThatThrownBy( + () -> + publisher.sendInTransaction( + KafkaTransactionFixtures.deliveryWithUnregisteredSecondOutput( + OUTPUT_TOPIC, "aborted-1"), + profiles(), + Map.of(new TopicPartition(INPUT_TOPIC, 0), new OffsetAndMetadata(1)))) + .isInstanceOf(RuntimeException.class); + + assertThat(readOutputPayloads()) + .as("the record was sent before the failure; the abort is what makes it invisible") + .doesNotContain("aborted-1"); + } + + @Test + void theProducerRecoversAfterAnAbortAndCanCommitAgain() { + KafkaTransactionalPublisher publisher = + KafkaTransactionalPublisher.forStaticGroup(producer, CONSUMER_GROUP); + publisher.initialise(); + + assertThatThrownBy( + () -> + publisher.sendInTransaction( + KafkaTransactionFixtures.deliveryWithUnregisteredSecondOutput( + OUTPUT_TOPIC, "aborted-2"), + profiles(), + Map.of(new TopicPartition(INPUT_TOPIC, 0), new OffsetAndMetadata(1)))) + .isInstanceOf(RuntimeException.class); + + publisher.sendInTransaction( + KafkaTransactionFixtures.delivery(OUTPUT_TOPIC, "recovered-1"), + profiles(), + Map.of(new TopicPartition(INPUT_TOPIC, 0), new OffsetAndMetadata(2))); + + assertThat(readOutputPayloads()) + .as("aborting rather than leaving the transaction open is what makes recovery possible") + .contains("recovered-1") + .doesNotContain("aborted-2"); + } + + @Test + void theInputOffsetIsCommittedWithTheOutputsNotSeparately() { + KafkaTransactionalPublisher publisher = + KafkaTransactionalPublisher.forStaticGroup(producer, CONSUMER_GROUP); + publisher.initialise(); + + publisher.sendInTransaction( + KafkaTransactionFixtures.delivery(OUTPUT_TOPIC, "offset-bound-1"), + profiles(), + Map.of(new TopicPartition(INPUT_TOPIC, 0), new OffsetAndMetadata(7))); + + try (Consumer groupReader = KafkaContainerFixture.consumer(CONSUMER_GROUP)) { + Map committed = + groupReader.committed(java.util.Set.of(new TopicPartition(INPUT_TOPIC, 0))); + + assertThat(committed.get(new TopicPartition(INPUT_TOPIC, 0))) + .as("the offset landing with the outputs is what removes the duplicate window") + .isNotNull() + .satisfies(offset -> assertThat(offset.offset()).isEqualTo(7)); + } + } + + private static Map profiles() { + return Map.of(OUTPUT_TOPIC, KafkaFixtureProfiles.withTopic(OUTPUT_TOPIC)); + } + + private List readOutputPayloads() { + reader.assign(List.of(new TopicPartition(OUTPUT_TOPIC, 0))); + reader.seekToBeginning(List.of(new TopicPartition(OUTPUT_TOPIC, 0))); + + List payloads = new java.util.ArrayList<>(); + for (int attempt = 0; attempt < 10; attempt++) { + ConsumerRecords records = reader.poll(Duration.ofMillis(500)); + records.forEach( + record -> + payloads.add(new String(record.value(), java.nio.charset.StandardCharsets.UTF_8))); + if (!payloads.isEmpty() && records.isEmpty()) { + break; + } + } + return payloads; + } +} diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/PartitionWorkCoordinatorTest.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/PartitionWorkCoordinatorTest.java new file mode 100644 index 00000000..3c989738 --- /dev/null +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/PartitionWorkCoordinatorTest.java @@ -0,0 +1,77 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.kafka.common.TopicPartition; +import org.junit.jupiter.api.Test; + +class PartitionWorkCoordinatorTest { + + private static final TopicPartition PARTITION = new TopicPartition("orders", 0); + private static final TopicPartition OTHER = new TopicPartition("orders", 1); + + @Test + void anOrderedDestinationAllowsExactlyOneDeliveryInFlight() { + PartitionWorkCoordinator coordinator = new PartitionWorkCoordinator(1); + + assertThat(coordinator.tryAcquire(PARTITION)).isTrue(); + assertThat(coordinator.tryAcquire(PARTITION)).isFalse(); + + coordinator.release(PARTITION); + + assertThat(coordinator.tryAcquire(PARTITION)).isTrue(); + } + + @Test + void theCeilingIsPerPartitionNotPerConsumer() { + PartitionWorkCoordinator coordinator = new PartitionWorkCoordinator(1); + + assertThat(coordinator.tryAcquire(PARTITION)).isTrue(); + assertThat(coordinator.tryAcquire(OTHER)).isTrue(); + } + + @Test + void aPausedPartitionAcceptsNoNewWork() { + PartitionWorkCoordinator coordinator = new PartitionWorkCoordinator(4); + + coordinator.pause(PARTITION); + + assertThat(coordinator.tryAcquire(PARTITION)).isFalse(); + assertThat(coordinator.isPaused(PARTITION)).isTrue(); + assertThat(coordinator.pausedPartitions()).containsExactly(PARTITION); + + coordinator.resume(PARTITION); + + assertThat(coordinator.tryAcquire(PARTITION)).isTrue(); + } + + @Test + void concurrencyAboveOneIsAllowedForUnorderedDestinations() { + PartitionWorkCoordinator coordinator = new PartitionWorkCoordinator(3); + + assertThat(coordinator.tryAcquire(PARTITION)).isTrue(); + assertThat(coordinator.tryAcquire(PARTITION)).isTrue(); + assertThat(coordinator.tryAcquire(PARTITION)).isTrue(); + assertThat(coordinator.tryAcquire(PARTITION)).isFalse(); + assertThat(coordinator.inFlight(PARTITION)).isEqualTo(3); + } + + @Test + void revocationForgetsPartitionState() { + PartitionWorkCoordinator coordinator = new PartitionWorkCoordinator(1); + coordinator.tryAcquire(PARTITION); + coordinator.pause(PARTITION); + + coordinator.forget(PARTITION); + + assertThat(coordinator.isPaused(PARTITION)).isFalse(); + assertThat(coordinator.inFlight(PARTITION)).isZero(); + } + + @Test + void aZeroCeilingIsRejectedBecauseItWouldStallTheConsumer() { + assertThatThrownBy(() -> new PartitionWorkCoordinator(0)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/src/messaging/messaging-nats-experimental/build.gradle b/src/messaging/messaging-nats-experimental/build.gradle new file mode 100644 index 00000000..18cf8a28 --- /dev/null +++ b/src/messaging/messaging-nats-experimental/build.gradle @@ -0,0 +1,13 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-schema-api') + api project(':messaging:messaging-policy') + api project(':messaging:messaging-transport-spi') + api project(':messaging:messaging-observability') + api project(':messaging:messaging-security') + api project(':messaging:messaging-admin-api') + + implementation 'io.nats:jnats:2.26.2' +} diff --git a/src/messaging/messaging-nats-experimental/gradle.lockfile b/src/messaging/messaging-nats-experimental/gradle.lockfile new file mode 100644 index 00000000..2000bd21 --- /dev/null +++ b/src/messaging/messaging-nats-experimental/gradle.lockfile @@ -0,0 +1,90 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +io.micrometer:micrometer-commons:1.16.0=runtimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-core:1.16.0=runtimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-observation:1.16.0=runtimeClasspath,testRuntimeClasspath +io.nats:jnats:2.26.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.bouncycastle:bcprov-lts8on:2.73.10=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.hdrhistogram:HdrHistogram:2.2.2=runtimeClasspath,testRuntimeClasspath +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,compileClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.latencyutils:LatencyUtils:2.0.3=runtimeClasspath,testRuntimeClasspath +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty= diff --git a/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsAckMode.java b/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsAckMode.java new file mode 100644 index 00000000..2a8b865a --- /dev/null +++ b/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsAckMode.java @@ -0,0 +1,57 @@ +package dev.caskeleton.messaging.nats; + +/** + * How a JetStream consumer acknowledges what it has handled. + * + *

Only {@link #EXPLICIT} is usable by the platform, and the other two are modelled anyway so the + * refusal can name what was configured instead of failing with a generic message. + * + *

{@link #NONE} and {@link #ALL} both break at-least-once in ways that look like success. {@code + * AckNone} never acknowledges, so JetStream considers every message delivered the moment it is sent + * — a handler that throws has already had its message forgotten. {@code AckAll} acknowledges every + * message up to the one being acknowledged, so a consumer processing messages concurrently silently + * acknowledges work that is still in flight. + */ +public enum NatsAckMode { + + /** Every message is acknowledged individually. The only mode the platform supports. */ + EXPLICIT(true), + + /** Acknowledging one message acknowledges every earlier one. */ + ALL(false), + + /** No acknowledgement at all; JetStream treats delivery as completion. */ + NONE(false); + + private final boolean supportsAtLeastOnce; + + NatsAckMode(boolean supportsAtLeastOnce) { + this.supportsAtLeastOnce = supportsAtLeastOnce; + } + + /** + * Reports whether this mode can carry at-least-once delivery. + * + * @return true only for {@link #EXPLICIT} + */ + public boolean supportsAtLeastOnce() { + return supportsAtLeastOnce; + } + + /** + * Returns why a mode cannot be used, for the refusal message. + * + * @return the sanitized reason, empty for {@link #EXPLICIT} + */ + public String rejectionReason() { + return switch (this) { + case EXPLICIT -> ""; + case ALL -> + "AckAll acknowledges every earlier message, so concurrent handling silently settles work " + + "that is still in flight"; + case NONE -> + "AckNone treats delivery as completion, so a message whose handler throws has already " + + "been forgotten by the broker"; + }; + } +} diff --git a/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsJetStreamProfile.java b/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsJetStreamProfile.java new file mode 100644 index 00000000..2713ab27 --- /dev/null +++ b/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsJetStreamProfile.java @@ -0,0 +1,103 @@ +package dev.caskeleton.messaging.nats; + +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; + +/** + * The JetStream settings for one destination. + * + *

{@code maxDeliver} is the setting that makes the platform's dead-lettering necessary. When a + * message reaches it, JetStream terminates the message — it stops redelivering and routes + * it nowhere. Without the platform parking it first, the message is simply gone, which is why + * {@link NatsMaxDeliverParkingWorkflow} has to run strictly before the limit is reached rather than + * in response to it. + * + *

{@code ackWait} is the redelivery trigger. A handler slower than {@code ackWait} gets its + * message redelivered while it is still working on it, so this is the one JetStream number that has + * to be set from the handler's real latency rather than from a default. + * + * @param subject the JetStream subject + * @param stream the stream capturing that subject + * @param durableName the durable consumer name + * @param ackMode how the consumer acknowledges + * @param ackWait how long JetStream waits before redelivering + * @param maxDeliver how many deliveries before JetStream terminates the message + * @param deduplicationWindow the publish deduplication window, when enabled + */ +public record NatsJetStreamProfile( + String subject, + String stream, + String durableName, + NatsAckMode ackMode, + Duration ackWait, + int maxDeliver, + Optional deduplicationWindow) { + + /** How many deliveries the platform leaves as headroom before JetStream's terminate. */ + public static final int PARKING_HEADROOM = 1; + + public NatsJetStreamProfile { + Objects.requireNonNull(ackMode, "ackMode must not be null"); + Objects.requireNonNull(ackWait, "ackWait must not be null"); + Objects.requireNonNull(deduplicationWindow, "deduplicationWindow must not be null"); + requireText(subject, "subject"); + requireText(stream, "stream"); + requireText(durableName, "durableName"); + + if (!ackMode.supportsAtLeastOnce()) { + throw new IllegalArgumentException( + "%s cannot carry at-least-once delivery: %s" + .formatted(ackMode, ackMode.rejectionReason())); + } + if (ackWait.isNegative() || ackWait.isZero()) { + throw new IllegalArgumentException("ackWait must be positive"); + } + if (maxDeliver < 2) { + throw new IllegalArgumentException( + "maxDeliver must leave at least one delivery of headroom for the platform to park the " + + "message before JetStream terminates it"); + } + if (deduplicationWindow.isPresent() + && (deduplicationWindow.get().isNegative() || deduplicationWindow.get().isZero())) { + throw new IllegalArgumentException("deduplicationWindow must be positive when set"); + } + } + + /** + * Returns the delivery count at which the platform must park the message. + * + *

Strictly before {@code maxDeliver}: acting on the limit itself is too late, because the + * delivery that reaches it is the one JetStream terminates. + * + * @return the parking threshold + */ + public int parkAtDelivery() { + return maxDeliver - PARKING_HEADROOM; + } + + /** + * Returns a profile with the platform defaults. + * + * @param subject the JetStream subject + * @param stream the stream name + * @param durableName the durable consumer name + * @return the profile + */ + public static NatsJetStreamProfile durable(String subject, String stream, String durableName) { + return new NatsJetStreamProfile( + subject, + stream, + durableName, + NatsAckMode.EXPLICIT, + Duration.ofSeconds(30), + 5, + Optional.of(Duration.ofMinutes(2))); + } + + private static void requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " must not be blank"); + } + } +} diff --git a/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsJetStreamProfileValidator.java b/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsJetStreamProfileValidator.java new file mode 100644 index 00000000..ca16a570 --- /dev/null +++ b/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsJetStreamProfileValidator.java @@ -0,0 +1,75 @@ +package dev.caskeleton.messaging.nats; + +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.MessagingCapabilities; +import dev.caskeleton.messaging.api.error.MessagingCapabilityUnavailableException; +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.util.Objects; + +/** + * Guards the NATS JetStream Experimental adapter. + * + *

Core NATS is refused for any at-least-once destination. Core NATS is fire-and-forget with no + * persistence and no acknowledgement, so a durable destination configured against it would report + * success for messages that were never stored anywhere — the failure mode is total and silent. + * + *

An ordered JetStream consumer and a competing work queue are mutually exclusive: the ordered + * consumer is a single-reader construct, and pointing several workers at it does not distribute the + * load, it breaks the ordering the consumer exists to provide. + */ +public final class NatsJetStreamProfileValidator { + + /** + * Validates a destination against a JetStream consumer configuration. + * + * @param profile the destination profile + * @param jetStreamEnabled whether JetStream persistence is in use + * @param orderedConsumer whether an ordered consumer is configured + * @param competingWorkers how many workers share the consumer + * @param enabled whether the experimental adapter is switched on + */ + public void validate( + DestinationProfile profile, + boolean jetStreamEnabled, + boolean orderedConsumer, + int competingWorkers, + boolean enabled) { + Objects.requireNonNull(profile, "profile must not be null"); + + if (!enabled) { + throw new MessagingCapabilityUnavailableException( + "NATS_DISABLED", + "the NATS JetStream adapter is experimental and disabled unless " + + "backend.messaging.experimental.nats=true"); + } + if (profile.deliveryGuarantee() == DeliveryGuarantee.AT_LEAST_ONCE && !jetStreamEnabled) { + throw new IllegalArgumentException( + "core NATS cannot provide at-least-once delivery; JetStream is required: " + + profile.name().value()); + } + if (orderedConsumer && competingWorkers > 1) { + throw new IllegalArgumentException( + "an ordered JetStream consumer cannot be shared by competing workers: " + + profile.name().value()); + } + if (profile.orderingScope() == OrderingScope.KEY) { + throw new IllegalArgumentException( + "the NATS adapter does not offer per-key ordering: " + profile.name().value()); + } + } + + /** + * Returns what the JetStream adapter can prove. + * + *

Native dead lettering is reported as unavailable. JetStream terminates a message after its + * delivery limit rather than routing it anywhere, so the platform supplies dead lettering itself; + * claiming a native one would leave operators looking for a queue that does not exist. + * + * @return the capabilities + */ + public MessagingCapabilities capabilities() { + return new MessagingCapabilities( + true, true, true, true, true, false, true, false, false, true, false, true); + } +} diff --git a/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsJetStreamTransport.java b/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsJetStreamTransport.java new file mode 100644 index 00000000..e14e647e --- /dev/null +++ b/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsJetStreamTransport.java @@ -0,0 +1,268 @@ +package dev.caskeleton.messaging.nats; + +import dev.caskeleton.messaging.api.destination.DestinationCapabilities; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessagingCapabilities; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.error.MessagingCapabilityUnavailableException; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import dev.caskeleton.messaging.transport.MessagingTransport; +import dev.caskeleton.messaging.transport.TransportConsumerRegistration; +import dev.caskeleton.messaging.transport.TransportConsumerSpec; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import dev.caskeleton.messaging.transport.TransportPublishResult; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; + +/** + * The Experimental NATS JetStream adapter. + * + *

Publishes go through JetStream, never core NATS. A core publish returns as soon as the bytes + * are written to the socket, with no persistence and no acknowledgement, so an adapter using it + * would report success for messages that were never stored — the failure is total and silent, which + * is why {@link NatsJetStreamProfileValidator} refuses the combination at startup. + * + *

A JetStream publish acknowledgement names the stream and sequence the message landed in, which + * is real persistence evidence. A publish that times out is {@code AMBIGUOUS}: JetStream may have + * stored it and lost only the acknowledgement, and the deduplication window is what makes retrying + * it safe when the profile enables one. + * + *

The adapter reports {@code nativeDeadLetter} as false. JetStream terminates a message at its + * delivery limit rather than routing it anywhere, so the platform parks it via {@link + * NatsMaxDeliverParkingWorkflow}; claiming a native dead-letter would send operators looking for a + * queue that does not exist. + */ +public final class NatsJetStreamTransport implements MessagingTransport { + + private static final MessagingCapabilities CAPABILITIES = + new MessagingCapabilities( + true, true, true, true, true, false, true, false, false, true, false, true); + + private final String brokerName; + private final long generation; + private final NatsJetStreamProfile profile; + private final JetStreamPublishOperation publish; + private final Function consumerFactory; + private final AtomicBoolean closed = new AtomicBoolean(); + + /** + * Creates a publish-only transport. + * + * @param brokerName the logical broker name + * @param generation the runtime generation + * @param profile the JetStream settings + * @param publish the JetStream publish operation + */ + public NatsJetStreamTransport( + String brokerName, + long generation, + NatsJetStreamProfile profile, + JetStreamPublishOperation publish) { + this( + brokerName, + generation, + profile, + publish, + spec -> { + throw new MessagingCapabilityUnavailableException( + "NATS_CONSUMER_NOT_CONFIGURED", + "this NATS transport was created without a consumer factory"); + }); + } + + /** + * Creates a transport with a consumer factory. + * + * @param brokerName the logical broker name + * @param generation the runtime generation + * @param profile the JetStream settings + * @param publish the JetStream publish operation + * @param consumerFactory builds a consumer registration for a spec + */ + public NatsJetStreamTransport( + String brokerName, + long generation, + NatsJetStreamProfile profile, + JetStreamPublishOperation publish, + Function consumerFactory) { + this.brokerName = Objects.requireNonNull(brokerName, "brokerName must not be null"); + this.generation = generation; + this.profile = Objects.requireNonNull(profile, "profile must not be null"); + this.publish = Objects.requireNonNull(publish, "publish must not be null"); + this.consumerFactory = Objects.requireNonNull(consumerFactory, "consumerFactory is required"); + } + + @Override + public CompletionStage publish(TransportPublishRequest request) { + Objects.requireNonNull(request, "request must not be null"); + + if (closed.get()) { + return completed(rejectedLocally("NATS_TRANSPORT_CLOSED", "the transport is shutting down")); + } + int size = request.envelope().payload().size(); + int limit = request.profile().payload().maxBytes(); + if (size > limit) { + return completed( + rejectedLocally( + "PAYLOAD_TOO_LARGE", "encoded payload is " + size + " bytes, limit is " + limit)); + } + + return publish + .publish(profile.subject(), deduplicationId(request), request) + .handle((position, failure) -> failure == null ? confirmed(position) : classify(failure)) + .thenApply(TransportPublishResult::new); + } + + @Override + public TransportConsumerRegistration register(TransportConsumerSpec spec) { + Objects.requireNonNull(spec, "spec must not be null"); + if (closed.get()) { + throw new MessagingCapabilityUnavailableException( + "NATS_TRANSPORT_CLOSED", "the transport is shutting down"); + } + return consumerFactory.apply(spec); + } + + @Override + public DestinationCapabilities capabilities(DestinationName destination) { + return new DestinationCapabilities(destination, brokerName, CAPABILITIES); + } + + @Override + public String brokerName() { + return brokerName; + } + + @Override + public long generation() { + return generation; + } + + @Override + public void close() { + closed.set(true); + } + + /** + * Reports whether the transport is still accepting work. + * + * @return true until close + */ + public boolean isAcceptingWork() { + return !closed.get(); + } + + /** + * Returns the deduplication id for a publish, when the profile enables deduplication. + * + *

The logical message id, so a retry of an ambiguous publish is recognised as the same message + * rather than accepted as a new one. Minting a fresh id per attempt would make the deduplication + * window useless in exactly the situation it exists for. + * + * @param request the publish request + * @return the deduplication id, or empty when the profile has no window + */ + private Optional deduplicationId(TransportPublishRequest request) { + return profile + .deduplicationWindow() + .map(window -> request.envelope().messageId().value().toString()); + } + + private static PublishResult confirmed(NatsStreamPosition position) { + return new PublishResult( + PublishCompletion.CONFIRMED, + // The publish acknowledgement names the stream and sequence the message landed in, which is + // persistence evidence rather than a receipt for bytes on a socket. + new PublishEvidence( + true, + TransmissionEvidence.TRANSMITTED, + true, + ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK), + RoutingOutcome.ROUTED, + Optional.of(position), + 1, + Duration.ZERO, + Optional.empty()); + } + + private static PublishResult classify(Throwable failure) { + Throwable cause = failure instanceof CompletionException ? failure.getCause() : failure; + String name = cause == null ? "" : cause.getClass().getSimpleName(); + + if (name.contains("Timeout")) { + return new PublishResult( + PublishCompletion.AMBIGUOUS, + new PublishEvidence( + true, TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED, false, ConfirmationLevel.NONE), + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of( + FailureDescriptor.of( + FailureCategory.TRANSIENT_INFRASTRUCTURE, + "NATS_PUBLISH_TIMEOUT", + "the publish acknowledgement did not arrive; the stream may already hold it"))); + } + // "no responders" means no JetStream server is listening on the subject at all. That is a + // configuration fault, not a transient one: retrying it will fail identically until the stream + // is declared. + boolean noStream = name.contains("NoRespond") || name.contains("Jetstream"); + return new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of( + FailureDescriptor.of( + noStream ? FailureCategory.CONFIGURATION : FailureCategory.PERMANENT_BUSINESS, + noStream ? "NATS_STREAM_NOT_DECLARED" : "NATS_PUBLISH_REJECTED", + "the JetStream publish was rejected: " + name))); + } + + private static TransportPublishResult rejectedLocally(String code, String message) { + return new TransportPublishResult( + new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of(FailureDescriptor.of(FailureCategory.PERMANENT_BUSINESS, code, message)))); + } + + private static CompletionStage completed(TransportPublishResult result) { + return CompletableFuture.completedFuture(result); + } + + /** The JetStream publish, isolated so the adapter is testable without a server. */ + @FunctionalInterface + public interface JetStreamPublishOperation { + + /** + * Publishes one encoded message to JetStream. + * + * @param subject the JetStream subject + * @param deduplicationId the {@code Nats-Msg-Id} value, when deduplication is enabled + * @param request the publish request + * @return a stage completing with the stream position the message landed at + */ + CompletionStage publish( + String subject, Optional deduplicationId, TransportPublishRequest request); + } +} diff --git a/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsMaxDeliverParkingWorkflow.java b/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsMaxDeliverParkingWorkflow.java new file mode 100644 index 00000000..d13ee0ce --- /dev/null +++ b/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsMaxDeliverParkingWorkflow.java @@ -0,0 +1,85 @@ +package dev.caskeleton.messaging.nats; + +import java.util.Objects; + +/** + * Decides what to do with a JetStream delivery before its {@code maxDeliver} limit destroys it. + * + *

JetStream has no dead-letter queue. When a message hits {@code maxDeliver} the server + * terminates it: no redelivery, no routing, no record beyond an advisory. Every other broker in + * this platform parks a poison message somewhere an operator can find it, and this workflow is what + * makes NATS behave the same way. + * + *

The parking therefore happens on the delivery before the limit, not on the limit + * itself. Acting at {@code maxDeliver} would mean acting on the delivery JetStream is about to + * discard, so any failure in the dead-letter publish would lose the message outright — precisely + * the case the DLQ-confirm-before-settle invariant exists to prevent. + */ +public final class NatsMaxDeliverParkingWorkflow { + + /** What the platform should do with one delivery. */ + public enum Action { + /** Hand the message to the handler as normal. */ + DELIVER_TO_HANDLER, + /** Publish to the platform dead-letter destination, then terminate the source. */ + PARK_TO_DEAD_LETTER, + /** + * The message has already passed the limit and JetStream has terminated it. + * + *

Reachable only if the profile changed under a live consumer. Nothing can be recovered + * here; the outcome exists so the situation is reported rather than mistaken for a redelivery. + */ + ALREADY_TERMINATED + } + + private final NatsJetStreamProfile profile; + + /** + * Creates a workflow for one destination. + * + * @param profile the JetStream settings + */ + public NatsMaxDeliverParkingWorkflow(NatsJetStreamProfile profile) { + this.profile = Objects.requireNonNull(profile, "profile must not be null"); + } + + /** + * Decides what to do with a delivery. + * + * @param deliveryCount how many times this message has been delivered, counting this one + * @return the action to take + */ + public Action decide(long deliveryCount) { + if (deliveryCount < 1) { + throw new IllegalArgumentException("the first delivery is delivery 1"); + } + if (deliveryCount > profile.maxDeliver()) { + return Action.ALREADY_TERMINATED; + } + return deliveryCount >= profile.parkAtDelivery() + ? Action.PARK_TO_DEAD_LETTER + : Action.DELIVER_TO_HANDLER; + } + + /** + * Reports whether the source may be terminated after a dead-letter publish. + * + *

Only on a confirmed publish. Terminating first would discard the message on a broker that + * cannot redeliver it, which is the one irreversible mistake available here. + * + * @param deadLetterConfirmed whether the dead-letter publish was confirmed + * @return true when the source may be terminated + */ + public boolean maySettleSource(boolean deadLetterConfirmed) { + return deadLetterConfirmed; + } + + /** + * Returns the delivery count at which parking begins. + * + * @return the parking threshold + */ + public int parkingThreshold() { + return profile.parkAtDelivery(); + } +} diff --git a/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsStreamPosition.java b/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsStreamPosition.java new file mode 100644 index 00000000..7463b69f --- /dev/null +++ b/src/messaging/messaging-nats-experimental/src/main/java/dev/caskeleton/messaging/nats/NatsStreamPosition.java @@ -0,0 +1,75 @@ +package dev.caskeleton.messaging.nats; + +import dev.caskeleton.messaging.api.publish.BrokerPosition; +import java.util.Map; +import java.util.Objects; + +/** + * A JetStream message's coordinate: its stream, its sequence in that stream, and the consumer's own + * sequence. + * + *

Two sequences, because they answer different questions. The stream sequence identifies the + * message and is what a replay seeks to. The consumer sequence counts deliveries to this + * consumer, so it advances on every redelivery while the stream sequence does not — comparing them + * is how a redelivery is recognised at all. + * + * @param stream the stream holding the message + * @param streamSequence the message's position in the stream + * @param consumerSequence how many messages this consumer has been delivered + * @param deliveryCount how many times this message has been delivered + */ +public record NatsStreamPosition( + String stream, long streamSequence, long consumerSequence, long deliveryCount) + implements BrokerPosition { + + public NatsStreamPosition { + if (stream == null || stream.isBlank()) { + throw new IllegalArgumentException("stream must not be blank"); + } + if (streamSequence < 1) { + throw new IllegalArgumentException("JetStream sequences start at 1"); + } + if (deliveryCount < 1) { + throw new IllegalArgumentException("a delivered message has been delivered at least once"); + } + } + + @Override + public String broker() { + return "nats"; + } + + @Override + public Map diagnosticAttributes() { + return Map.of( + "stream", stream, + "streamSequence", Long.toString(streamSequence), + "consumerSequence", Long.toString(consumerSequence), + "deliveryCount", Long.toString(deliveryCount)); + } + + /** + * Reports whether this delivery is a redelivery. + * + * @return true when the message has been delivered before + */ + public boolean isRedelivery() { + return deliveryCount > 1; + } + + /** + * Returns the stream sequence a replay should resume from to re-read this message. + * + * @return the resume sequence + */ + public long replayFrom() { + return streamSequence; + } + + @Override + public String toString() { + return Objects.toString( + "NatsStreamPosition[stream=%s, streamSequence=%d, deliveryCount=%d]" + .formatted(stream, streamSequence, deliveryCount)); + } +} diff --git a/src/messaging/messaging-nats-experimental/src/test/java/dev/caskeleton/messaging/nats/NatsAdapterContractTest.java b/src/messaging/messaging-nats-experimental/src/test/java/dev/caskeleton/messaging/nats/NatsAdapterContractTest.java new file mode 100644 index 00000000..706cd179 --- /dev/null +++ b/src/messaging/messaging-nats-experimental/src/test/java/dev/caskeleton/messaging/nats/NatsAdapterContractTest.java @@ -0,0 +1,263 @@ +package dev.caskeleton.messaging.nats; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.delivery.ExternalSideEffectGuarantee; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.DestinationKind; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import dev.caskeleton.messaging.policy.CapabilityTier; +import dev.caskeleton.messaging.policy.ConsumerPolicy; +import dev.caskeleton.messaging.policy.DeadLetterPolicy; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.PayloadPolicy; +import dev.caskeleton.messaging.policy.PhysicalDestination; +import dev.caskeleton.messaging.policy.ProducerPolicy; +import dev.caskeleton.messaging.policy.RetryPolicy; +import dev.caskeleton.messaging.policy.SchemaPolicy; +import dev.caskeleton.messaging.schema.EncodedMessage; +import dev.caskeleton.messaging.schema.SchemaCompatibility; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import dev.caskeleton.messaging.transport.TransportPublishResult; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.TimeoutException; +import org.junit.jupiter.api.Test; + +class NatsAdapterContractTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + private static final NatsStreamPosition POSITION = new NatsStreamPosition("ORDERS", 42, 42, 1); + + private final List> capturedDeduplicationIds = new ArrayList<>(); + + private NatsJetStreamTransport confirming(Optional deduplicationWindow) { + return new NatsJetStreamTransport( + "nats-primary", + 1, + profile(deduplicationWindow), + (subject, deduplicationId, request) -> { + capturedDeduplicationIds.add(deduplicationId); + return CompletableFuture.completedFuture(POSITION); + }); + } + + private NatsJetStreamTransport failingWith(Throwable failure) { + return new NatsJetStreamTransport( + "nats-primary", + 1, + profile(Optional.of(Duration.ofMinutes(2))), + (subject, deduplicationId, request) -> CompletableFuture.failedFuture(failure)); + } + + private static PublishResult await(CompletionStage stage) { + return stage.toCompletableFuture().join().result(); + } + + @Test + void aStoredMessageReportsPersistenceEvidence() { + PublishResult result = await(confirming(Optional.empty()).publish(request(64))); + + assertThat(result.completion()).isEqualTo(PublishCompletion.CONFIRMED); + assertThat(result.evidence().confirmationLevel()) + .as("the acknowledgement names the stream and sequence, not just bytes on a socket") + .isEqualTo(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK); + } + + @Test + void theStreamPositionIsCarriedBack() { + assertThat(await(confirming(Optional.empty()).publish(request(64))).position()) + .hasValue(POSITION); + } + + @Test + void aPublishTimeoutIsAmbiguousBecauseTheStreamMayHoldIt() { + PublishResult result = await(failingWith(new TimeoutException("no ack")).publish(request(64))); + + assertThat(result.completion()).isEqualTo(PublishCompletion.AMBIGUOUS); + assertThat(result.evidence().transmission()) + .isEqualTo(TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED); + } + + @Test + void aMissingStreamIsAConfigurationFaultNotATransientOne() { + PublishResult result = + await(failingWith(new IllegalStateException("NoRespondersException")).publish(request(64))); + + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.failure()) + .hasValueSatisfying( + failure -> assertThat(failure.code()).isEqualTo("NATS_PUBLISH_REJECTED")); + } + + @Test + void aDeduplicationWindowSendsTheLogicalMessageIdSoARetryIsRecognised() { + confirming(Optional.of(Duration.ofMinutes(2))) + .publish(request(64)) + .toCompletableFuture() + .join(); + + assertThat(capturedDeduplicationIds) + .as("a fresh id per attempt would make the window useless exactly when it is needed") + .singleElement() + .satisfies(id -> assertThat(id).isPresent()); + } + + @Test + void noDeduplicationWindowSendsNoDeduplicationId() { + confirming(Optional.empty()).publish(request(64)).toCompletableFuture().join(); + + assertThat(capturedDeduplicationIds).singleElement().satisfies(id -> assertThat(id).isEmpty()); + } + + @Test + void anOversizedPayloadIsRejectedBeforeTheStream() { + PublishResult result = + await(confirming(Optional.empty()).publish(request(PayloadPolicy.DEFAULT_MAX_BYTES + 1))); + + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.evidence().transmission()).isEqualTo(TransmissionEvidence.NOT_TRANSMITTED); + } + + @Test + void theAdapterDoesNotClaimANativeDeadLetter() { + assertThat( + confirming(Optional.empty()) + .capabilities(new DestinationName("orders.v1")) + .capabilities() + .nativeDeadLetter()) + .as("JetStream terminates at the delivery limit; it routes nowhere") + .isFalse(); + } + + @Test + void theAdapterAdvertisesDeduplicatedPublish() { + assertThat( + confirming(Optional.empty()) + .capabilities(new DestinationName("orders.v1")) + .capabilities() + .deduplicatedPublish()) + .isTrue(); + } + + @Test + void aClosedTransportStopsAcceptingWork() { + NatsJetStreamTransport transport = confirming(Optional.empty()); + transport.close(); + + assertThat(transport.isAcceptingWork()).isFalse(); + assertThat(await(transport.publish(request(64))).completion()) + .isEqualTo(PublishCompletion.REJECTED); + } + + @Test + void aRedeliveryIsRecognisedFromTheDeliveryCountNotTheSequence() { + NatsStreamPosition redelivered = new NatsStreamPosition("ORDERS", 42, 43, 2); + + assertThat(POSITION.isRedelivery()).isFalse(); + assertThat(redelivered.isRedelivery()).isTrue(); + assertThat(redelivered.replayFrom()) + .as("the stream sequence does not advance on a redelivery; the consumer sequence does") + .isEqualTo(POSITION.replayFrom()); + } + + @Test + void aSequenceBelowOneIsRejectedBecauseJetStreamCountsFromOne() { + assertThatThrownBy(() -> new NatsStreamPosition("ORDERS", 0, 1, 1)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void aFailureCategoryIsAlwaysAttachedToARejection() { + PublishResult result = + await(failingWith(new IllegalStateException("boom")).publish(request(64))); + + assertThat(result.failure()) + .hasValueSatisfying( + failure -> + assertThat(failure.category()).isEqualTo(FailureCategory.PERMANENT_BUSINESS)); + } + + private static NatsJetStreamProfile profile(Optional deduplicationWindow) { + return new NatsJetStreamProfile( + "orders.created", + "ORDERS", + "orders-worker", + NatsAckMode.EXPLICIT, + Duration.ofSeconds(30), + 5, + deduplicationWindow); + } + + private static TransportPublishRequest request(int payloadBytes) { + return new TransportPublishRequest( + destinationProfile(), envelope(payloadBytes), PublishOptions.defaults()); + } + + private static DestinationProfile destinationProfile() { + return new DestinationProfile( + new DestinationName("orders.v1"), + "nats-primary", + DestinationKind.EVENT_STREAM, + PhysicalDestination.natsStream("orders.created", "ORDERS"), + new SchemaPolicy( + ContentType.JSON, + SchemaCompatibility.BACKWARD, + Set.of(new MessageType("order.created"))), + DeliveryGuarantee.AT_LEAST_ONCE, + OrderingScope.NONE, + ExternalSideEffectGuarantee.IDEMPOTENCY_REQUIRED, + ProducerPolicy.defaults(), + ConsumerPolicy.defaults("orders"), + RetryPolicy.none(), + DeadLetterPolicy.to(new DestinationName("orders.v1.dlq")), + PayloadPolicy.defaults(), + CapabilityTier.M1, + false, + false, + false); + } + + private static MessageEnvelope envelope(int payloadBytes) { + byte[] payload = new byte[payloadBytes]; + java.util.Arrays.fill(payload, (byte) 'x'); + return new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + NOW, + Optional.of(NOW), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage(payload, ContentType.JSON, Optional.empty())); + } +} diff --git a/src/messaging/messaging-nats-experimental/src/test/java/dev/caskeleton/messaging/nats/NatsMaxDeliverParkingTest.java b/src/messaging/messaging-nats-experimental/src/test/java/dev/caskeleton/messaging/nats/NatsMaxDeliverParkingTest.java new file mode 100644 index 00000000..93d0a0c0 --- /dev/null +++ b/src/messaging/messaging-nats-experimental/src/test/java/dev/caskeleton/messaging/nats/NatsMaxDeliverParkingTest.java @@ -0,0 +1,123 @@ +package dev.caskeleton.messaging.nats; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.nats.NatsMaxDeliverParkingWorkflow.Action; +import java.time.Duration; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class NatsMaxDeliverParkingTest { + + private static NatsJetStreamProfile profile(int maxDeliver) { + return new NatsJetStreamProfile( + "orders.created", + "ORDERS", + "orders-worker", + NatsAckMode.EXPLICIT, + Duration.ofSeconds(30), + maxDeliver, + Optional.of(Duration.ofMinutes(2))); + } + + private static NatsMaxDeliverParkingWorkflow workflow(int maxDeliver) { + return new NatsMaxDeliverParkingWorkflow(profile(maxDeliver)); + } + + @Test + void earlyDeliveriesGoToTheHandler() { + NatsMaxDeliverParkingWorkflow workflow = workflow(5); + + assertThat(workflow.decide(1)).isEqualTo(Action.DELIVER_TO_HANDLER); + assertThat(workflow.decide(3)).isEqualTo(Action.DELIVER_TO_HANDLER); + } + + @Test + void parkingHappensBeforeTheLimitNotAtIt() { + NatsMaxDeliverParkingWorkflow workflow = workflow(5); + + assertThat(workflow.decide(4)) + .as("the delivery that reaches maxDeliver is the one JetStream discards") + .isEqualTo(Action.PARK_TO_DEAD_LETTER); + } + + @Test + void theLimitDeliveryItselfStillParksRatherThanBeingHandled() { + assertThat(workflow(5).decide(5)).isEqualTo(Action.PARK_TO_DEAD_LETTER); + } + + @Test + void aDeliveryPastTheLimitIsReportedRatherThanMistakenForARedelivery() { + assertThat(workflow(5).decide(6)) + .as("only reachable if the profile changed under a live consumer; nothing is recoverable") + .isEqualTo(Action.ALREADY_TERMINATED); + } + + @Test + void theSourceIsOnlyTerminatedAfterTheDeadLetterPublishConfirms() { + NatsMaxDeliverParkingWorkflow workflow = workflow(5); + + assertThat(workflow.maySettleSource(true)).isTrue(); + assertThat(workflow.maySettleSource(false)) + .as("terminating first discards the message on a broker that cannot redeliver it") + .isFalse(); + } + + @Test + void aProfileWithNoParkingHeadroomIsRefused() { + assertThatThrownBy(() -> profile(1)) + .as("maxDeliver=1 leaves no delivery on which the platform could park the message") + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("headroom"); + } + + @Test + void aDeliveryCountBelowOneIsRejected() { + assertThatThrownBy(() -> workflow(5).decide(0)).isInstanceOf(IllegalArgumentException.class); + } + + @Test + void theParkingThresholdIsOneBelowTheLimit() { + assertThat(workflow(7).parkingThreshold()).isEqualTo(6); + } + + @Test + void ackNoneIsRefusedBecauseDeliveryIsTreatedAsCompletion() { + assertThatThrownBy( + () -> + new NatsJetStreamProfile( + "orders.created", + "ORDERS", + "orders-worker", + NatsAckMode.NONE, + Duration.ofSeconds(30), + 5, + Optional.empty())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("forgotten"); + } + + @Test + void ackAllIsRefusedBecauseItSettlesWorkStillInFlight() { + assertThatThrownBy( + () -> + new NatsJetStreamProfile( + "orders.created", + "ORDERS", + "orders-worker", + NatsAckMode.ALL, + Duration.ofSeconds(30), + 5, + Optional.empty())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("still in flight"); + } + + @Test + void onlyExplicitAcknowledgementCarriesAtLeastOnce() { + assertThat(NatsAckMode.EXPLICIT.supportsAtLeastOnce()).isTrue(); + assertThat(NatsAckMode.ALL.supportsAtLeastOnce()).isFalse(); + assertThat(NatsAckMode.NONE.supportsAtLeastOnce()).isFalse(); + } +} diff --git a/src/messaging/messaging-observability/build.gradle b/src/messaging/messaging-observability/build.gradle new file mode 100644 index 00000000..a0dac810 --- /dev/null +++ b/src/messaging/messaging-observability/build.gradle @@ -0,0 +1,7 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + + implementation 'io.micrometer:micrometer-core' +} diff --git a/src/messaging/messaging-observability/gradle.lockfile b/src/messaging/messaging-observability/gradle.lockfile new file mode 100644 index 00000000..8a9cedb6 --- /dev/null +++ b/src/messaging/messaging-observability/gradle.lockfile @@ -0,0 +1,88 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +io.micrometer:micrometer-commons:1.16.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.micrometer:micrometer-core:1.16.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.micrometer:micrometer-observation:1.16.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.hdrhistogram:HdrHistogram:2.2.2=runtimeClasspath,testRuntimeClasspath +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,compileClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.latencyutils:LatencyUtils:2.0.3=runtimeClasspath,testRuntimeClasspath +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty= diff --git a/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/CardinalityGuard.java b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/CardinalityGuard.java new file mode 100644 index 00000000..992e8a6a --- /dev/null +++ b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/CardinalityGuard.java @@ -0,0 +1,89 @@ +package dev.caskeleton.messaging.observation; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Caps how many distinct values a metric dimension may take. + * + *

Cardinality failures are not gradual. A tag that accidentally carries a message id looks fine + * in a test with ten messages and takes down the metrics backend in production, and by then the + * series already exist. The guard bounds each dimension at registration time and refuses the value + * that would cross the limit, so the damage is one rejected tag rather than a monitoring outage. + * + *

It fails loudly rather than silently substituting a placeholder, because a metric that quietly + * collapses distinct values is worse than one that is missing: it looks correct. + */ +public final class CardinalityGuard { + + private static final int DEFAULT_LIMIT = 200; + + private final int limitPerDimension; + private final Map> observed = new ConcurrentHashMap<>(); + + /** Creates a guard with the default limit of 200 values per dimension. */ + public CardinalityGuard() { + this(DEFAULT_LIMIT); + } + + /** + * Creates a guard with an explicit limit. + * + * @param limitPerDimension how many distinct values a dimension may take + */ + public CardinalityGuard(int limitPerDimension) { + if (limitPerDimension < 1) { + throw new IllegalArgumentException("limitPerDimension must be at least 1"); + } + this.limitPerDimension = limitPerDimension; + } + + /** + * Records a dimension value and reports whether it is within the limit. + * + * @param dimension the tag name + * @param value the tag value + * @return true when the value may be emitted + */ + public boolean admit(String dimension, String value) { + Objects.requireNonNull(dimension, "dimension must not be null"); + Objects.requireNonNull(value, "value must not be null"); + Set values = observed.computeIfAbsent(dimension, key -> ConcurrentHashMap.newKeySet()); + if (values.contains(value)) { + return true; + } + if (values.size() >= limitPerDimension) { + return false; + } + values.add(value); + return true; + } + + /** + * Admits an entire tag set, rejecting it if any dimension would exceed its limit. + * + * @param tags the bounded dimensions + * @return true when every dimension is within its limit + */ + public boolean admit(MessagingTags tags) { + Objects.requireNonNull(tags, "tags must not be null"); + boolean admitted = true; + for (Map.Entry entry : tags.asMap().entrySet()) { + admitted &= admit(entry.getKey(), entry.getValue()); + } + return admitted; + } + + /** + * Returns how many distinct values a dimension has taken. + * + * @param dimension the tag name + * @return the observed cardinality + */ + public int cardinality(String dimension) { + Set values = observed.get(dimension); + return values == null ? 0 : values.size(); + } +} diff --git a/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/DefaultMessagingObservationConvention.java b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/DefaultMessagingObservationConvention.java new file mode 100644 index 00000000..353c4b76 --- /dev/null +++ b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/DefaultMessagingObservationConvention.java @@ -0,0 +1,123 @@ +package dev.caskeleton.messaging.observation; + +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; + +/** + * Turns platform outcomes into the fixed tag vocabulary. + * + *

Centralised because the tag values are a public contract: dashboards, alert rules, and SLOs + * are written against these exact strings, so an adapter inventing its own spelling of "rejected" + * silently breaks every alert that was watching for it. The conversion lives here, once, rather + * than at each call site. + * + *

Only bounded inputs are accepted. Every parameter is an enum or a configured name, which is + * what lets {@link CardinalityGuard} bound the resulting series. + */ +public final class DefaultMessagingObservationConvention { + + /** The publish operation name. */ + public static final String PUBLISH = "publish"; + + /** The consume operation name. */ + public static final String CONSUME = "consume"; + + /** The settlement operation name. */ + public static final String SETTLE = "settle"; + + /** The dead-letter operation name. */ + public static final String DEAD_LETTER = "deadLetter"; + + /** + * Builds tags for a completed publish. + * + * @param broker the broker name + * @param destinationProfile the logical destination + * @param completion the publish outcome + * @param failure the failure classification when the publish did not confirm + * @return the tag set + */ + public MessagingTags publish( + String broker, + String destinationProfile, + PublishCompletion completion, + Optional failure) { + Objects.requireNonNull(completion, "completion must not be null"); + Objects.requireNonNull(failure, "failure must not be null"); + return new MessagingTags( + broker, + destinationProfile, + PUBLISH, + lower(completion.name()), + failure.map(category -> lower(category.name())).orElse(MessagingTags.NONE), + MessagingTags.NONE); + } + + /** + * Builds tags for a completed delivery attempt. + * + * @param broker the broker name + * @param destinationProfile the logical destination + * @param outcome the handler outcome name + * @param retryStage the retry stage, or {@link MessagingTags#NONE} on the first attempt + * @param failure the failure classification when handling failed + * @return the tag set + */ + public MessagingTags consume( + String broker, + String destinationProfile, + String outcome, + String retryStage, + Optional failure) { + Objects.requireNonNull(outcome, "outcome must not be null"); + Objects.requireNonNull(retryStage, "retryStage must not be null"); + Objects.requireNonNull(failure, "failure must not be null"); + return new MessagingTags( + broker, + destinationProfile, + CONSUME, + lower(outcome), + failure.map(category -> lower(category.name())).orElse(MessagingTags.NONE), + retryStage); + } + + /** + * Builds tags for a settlement. + * + * @param broker the broker name + * @param destinationProfile the logical destination + * @param outcome the settlement outcome name + * @return the tag set + */ + public MessagingTags settlement(String broker, String destinationProfile, String outcome) { + return MessagingTags.of(broker, destinationProfile, SETTLE, lower(outcome)); + } + + /** + * Builds tags for a dead-letter publish. + * + * @param broker the broker name + * @param destinationProfile the logical destination + * @param outcome the dead-letter outcome name + * @param failure the failure that caused the dead-letter + * @return the tag set + */ + public MessagingTags deadLetter( + String broker, String destinationProfile, String outcome, FailureCategory failure) { + Objects.requireNonNull(failure, "failure must not be null"); + return new MessagingTags( + broker, + destinationProfile, + DEAD_LETTER, + lower(outcome), + lower(failure.name()), + MessagingTags.NONE); + } + + private static String lower(String value) { + return value.toLowerCase(Locale.ROOT); + } +} diff --git a/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingAuditEvent.java b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingAuditEvent.java new file mode 100644 index 00000000..72f50b39 --- /dev/null +++ b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingAuditEvent.java @@ -0,0 +1,45 @@ +package dev.caskeleton.messaging.observation; + +import java.time.Instant; +import java.util.Map; +import java.util.Objects; + +/** + * A record of a privileged messaging operation. + * + *

Audit covers the operations that change state an application cannot: replay, redrive, offset + * reset, purge, and delete. The subject is the operator identity and the details are passed through + * {@link MessagingRedactor}, so an audit trail proves who did what without becoming a second copy + * of the payload. + * + * @param operation the privileged operation + * @param subject the operator or service identity that requested it + * @param destination the logical destination affected + * @param approvalTicket the approval reference + * @param occurredAt when it happened + * @param details additional sanitized context + */ +public record MessagingAuditEvent( + String operation, + String subject, + String destination, + String approvalTicket, + Instant occurredAt, + Map details) { + + public MessagingAuditEvent { + Objects.requireNonNull(occurredAt, "occurredAt must not be null"); + Objects.requireNonNull(details, "details must not be null"); + requireText(operation, "operation"); + requireText(subject, "subject"); + requireText(destination, "destination"); + requireText(approvalTicket, "approvalTicket"); + details = Map.copyOf(details); + } + + private static void requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " must not be blank"); + } + } +} diff --git a/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingAuditSink.java b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingAuditSink.java new file mode 100644 index 00000000..e3f92e56 --- /dev/null +++ b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingAuditSink.java @@ -0,0 +1,56 @@ +package dev.caskeleton.messaging.observation; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * Where audit records for privileged operations are written. + * + *

Separate from metrics and from application logs. An audit trail answers "who authorised this + * destructive operation", which is a different retention, access, and integrity requirement from + * "how slow was publish yesterday"; mixing them means either the audit gets dropped with the + * metrics or the metrics inherit the audit's retention cost. + * + *

Every record has already passed {@link MessagingRedactor}, so an audit trail proves who did + * what without becoming a second copy of the payload. + */ +public interface MessagingAuditSink { + + /** + * Records one privileged operation. + * + * @param event the sanitized audit record + */ + void record(MessagingAuditEvent event); + + /** + * Returns a sink that keeps events in memory, for tests and for a deployment that has no external + * audit store yet. + * + * @return an in-memory sink + */ + static InMemory inMemory() { + return new InMemory(); + } + + /** An in-memory sink whose contents can be asserted on. */ + final class InMemory implements MessagingAuditSink { + + private final List events = new CopyOnWriteArrayList<>(); + + @Override + public void record(MessagingAuditEvent event) { + events.add(Objects.requireNonNull(event, "event must not be null")); + } + + /** + * Returns the recorded events in order. + * + * @return the recorded events + */ + public List events() { + return List.copyOf(events); + } + } +} diff --git a/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingMetrics.java b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingMetrics.java new file mode 100644 index 00000000..83ab6215 --- /dev/null +++ b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingMetrics.java @@ -0,0 +1,158 @@ +package dev.caskeleton.messaging.observation; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.Timer; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; + +/** + * The Micrometer implementation of the observation seam. + * + *

Every tag set passes the {@link CardinalityGuard} before a meter is created. That ordering is + * the whole point: a meter registry never forgets a series, so a single tag carrying a message id + * permanently inflates the backend. Refused tag sets are counted under a fixed {@code + * messaging.tags.rejected} counter, which makes the rejection visible without creating the series + * that caused it. + * + *

Logical messages and physical attempts are separate meters. One message redelivered four times + * is one publish and five attempts; a single counter would make a redelivery storm read as traffic + * growth and hide the incident. + */ +public final class MessagingMetrics implements MessagingObservation { + + /** Timer for one publish call, from admission to outcome. */ + public static final String PUBLISH_TIMER = "messaging.publish"; + + /** Timer for one delivery attempt, covering the handler only. */ + public static final String DELIVERY_TIMER = "messaging.delivery"; + + /** Counter of logical messages delivered, incremented only on the first attempt. */ + public static final String MESSAGE_COUNTER = "messaging.messages"; + + /** Counter of settlement outcomes. */ + public static final String SETTLEMENT_COUNTER = "messaging.settlements"; + + /** Gauge of the observed consumer backlog. */ + public static final String BACKLOG_GAUGE = "messaging.backlog"; + + /** Counter of tag sets refused by the cardinality guard. */ + public static final String REJECTED_TAGS_COUNTER = "messaging.tags.rejected"; + + private final MeterRegistry registry; + private final CardinalityGuard guard; + private final MessagingRedactor redactor; + private final Map backlogs = new ConcurrentHashMap<>(); + private final LongAdder rejectedTagSets = new LongAdder(); + + /** + * Creates the metrics binding. + * + * @param registry the meter registry + * @param guard the cardinality guard applied before any meter is created + * @param redactor the redactor applied to diagnostic values + */ + public MessagingMetrics( + MeterRegistry registry, CardinalityGuard guard, MessagingRedactor redactor) { + this.registry = Objects.requireNonNull(registry, "registry must not be null"); + this.guard = Objects.requireNonNull(guard, "guard must not be null"); + this.redactor = Objects.requireNonNull(redactor, "redactor must not be null"); + registry.gauge(REJECTED_TAGS_COUNTER, rejectedTagSets, LongAdder::sum); + } + + @Override + public void recordPublish(MessagingTags tags, Duration elapsed) { + Objects.requireNonNull(elapsed, "elapsed must not be null"); + admitted(tags) + .ifPresent(micrometerTags -> timer(PUBLISH_TIMER, micrometerTags).record(elapsed)); + } + + @Override + public void recordDelivery(MessagingTags tags, Duration elapsed, int attempt) { + Objects.requireNonNull(elapsed, "elapsed must not be null"); + if (attempt < 1) { + throw new IllegalArgumentException("attempt counts the first delivery as 1"); + } + admitted(tags) + .ifPresent( + micrometerTags -> { + timer(DELIVERY_TIMER, micrometerTags).record(elapsed); + // Only the first attempt counts as a logical message; later attempts are the same + // message arriving again, and counting them would inflate throughput during a storm. + if (attempt == 1) { + registry.counter(MESSAGE_COUNTER, micrometerTags).increment(); + } + }); + } + + @Override + public void recordSettlement(MessagingTags tags) { + admitted(tags) + .ifPresent( + micrometerTags -> registry.counter(SETTLEMENT_COUNTER, micrometerTags).increment()); + } + + @Override + public void recordBacklog(MessagingTags tags, long messages) { + admitted(tags) + .ifPresent( + micrometerTags -> + backlogs + .computeIfAbsent( + micrometerTags, + key -> + registry.gauge(BACKLOG_GAUGE, key, new AtomicLong(), AtomicLong::get)) + .set(messages)); + } + + @Override + public void recordDiagnostics(MessagingTags tags, Map diagnostics) { + Objects.requireNonNull(tags, "tags must not be null"); + Objects.requireNonNull(diagnostics, "diagnostics must not be null"); + // Redact before anything else touches the values. Diagnostics are the one place where a caller + // can pass arbitrary keys, so this is where a secret would otherwise reach the backend. + Map safe = redactor.sanitize(diagnostics); + admitted(tags) + .ifPresent( + micrometerTags -> + safe.forEach( + (key, value) -> + registry + .counter( + "messaging.diagnostics", + micrometerTags.and( + Tag.of("diagnostic", key), Tag.of("value", value))) + .increment())); + } + + /** + * Returns how many tag sets the cardinality guard refused. + * + * @return the rejected tag set count + */ + public long rejectedTagSets() { + return rejectedTagSets.sum(); + } + + private java.util.Optional admitted(MessagingTags tags) { + Objects.requireNonNull(tags, "tags must not be null"); + if (!guard.admit(tags)) { + rejectedTagSets.increment(); + return java.util.Optional.empty(); + } + List micrometerTags = new ArrayList<>(); + tags.asMap().forEach((key, value) -> micrometerTags.add(Tag.of(key, value))); + return java.util.Optional.of(Tags.of(micrometerTags)); + } + + private Timer timer(String name, Tags tags) { + return Timer.builder(name).tags(tags).publishPercentileHistogram().register(registry); + } +} diff --git a/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingObservation.java b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingObservation.java new file mode 100644 index 00000000..bcd2615c --- /dev/null +++ b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingObservation.java @@ -0,0 +1,56 @@ +package dev.caskeleton.messaging.observation; + +import java.time.Duration; +import java.util.Map; + +/** + * The platform's observation seam. + * + *

Logical messages and physical delivery attempts are recorded separately. One logical message + * that was redelivered four times is one message and five attempts; collapsing them makes a + * redelivery storm look like traffic growth. + */ +public interface MessagingObservation { + + /** + * Records one completed publish. + * + * @param tags the bounded dimensions + * @param elapsed how long the publish took + */ + void recordPublish(MessagingTags tags, Duration elapsed); + + /** + * Records one completed delivery attempt. + * + * @param tags the bounded dimensions + * @param elapsed how long handling took + * @param attempt the attempt number, counting the first delivery as one + */ + void recordDelivery(MessagingTags tags, Duration elapsed, int attempt); + + /** + * Records one settlement outcome. + * + * @param tags the bounded dimensions + */ + void recordSettlement(MessagingTags tags); + + /** + * Records a consumer lag or backlog sample. + * + * @param tags the bounded dimensions + * @param messages the observed backlog + */ + void recordBacklog(MessagingTags tags, long messages); + + /** + * Emits a structured diagnostic event. + * + *

Implementations pass the map through {@link MessagingRedactor} before it leaves the process. + * + * @param tags the bounded dimensions + * @param diagnostics the raw diagnostic values + */ + void recordDiagnostics(MessagingTags tags, Map diagnostics); +} diff --git a/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingRedactor.java b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingRedactor.java new file mode 100644 index 00000000..190cc50b --- /dev/null +++ b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingRedactor.java @@ -0,0 +1,105 @@ +package dev.caskeleton.messaging.observation; + +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Strips identity and secrets from diagnostic maps before they are logged or tagged. + * + *

This is a denylist of keys that must never leave the process, not an allowlist, because + * diagnostic maps are assembled ad hoc at call sites and an allowlist would quietly drop the useful + * half. Two categories are removed. Secrets, for the obvious reason. And per-message identity — + * message ids, keys, offsets, delivery tags — because those are what turn a bounded metric into one + * series per message, and a support log into a re-identification surface. + */ +public final class MessagingRedactor { + + private static final String REDACTED = "[redacted]"; + + private static final Set DENIED_KEYS = + Set.of( + "messageid", + "msg.id", + "correlationid", + "causationid", + "partitionkey", + "orderingkey", + "key", + "offset", + "deliverytag", + "sequence", + "payload", + "body", + "data", + "authorization", + "proxy-authorization", + "cookie", + "set-cookie", + "access_token", + "refresh_token", + "api_key", + "apikey", + "password", + "client_secret", + "credential", + "secret", + "token", + "exceptionmessage", + "stacktrace"); + + /** + * Removes denied entries from a diagnostic map. + * + * @param diagnostics the raw diagnostic values + * @return a copy containing only permitted entries + */ + public Map sanitize(Map diagnostics) { + if (diagnostics == null || diagnostics.isEmpty()) { + return Map.of(); + } + Map sanitized = new LinkedHashMap<>(); + for (Map.Entry entry : diagnostics.entrySet()) { + String key = entry.getKey(); + if (key == null || isDenied(key)) { + continue; + } + sanitized.put(key, entry.getValue()); + } + return Map.copyOf(sanitized); + } + + /** + * Replaces denied values with a marker instead of dropping the key. + * + *

Useful where the presence of a field is itself the diagnostic signal. + * + * @param diagnostics the raw diagnostic values + * @return a copy with denied values masked + */ + public Map mask(Map diagnostics) { + if (diagnostics == null || diagnostics.isEmpty()) { + return Map.of(); + } + Map masked = new LinkedHashMap<>(); + for (Map.Entry entry : diagnostics.entrySet()) { + String key = entry.getKey(); + if (key == null) { + continue; + } + masked.put(key, isDenied(key) ? REDACTED : entry.getValue()); + } + return Map.copyOf(masked); + } + + /** + * Reports whether a diagnostic key is denied. + * + * @param key the diagnostic key in any case + * @return true when the key must not be emitted + */ + public boolean isDenied(String key) { + return key != null && DENIED_KEYS.contains(key.toLowerCase(Locale.ROOT)); + } +} diff --git a/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingTags.java b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingTags.java new file mode 100644 index 00000000..06b00f90 --- /dev/null +++ b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingTags.java @@ -0,0 +1,71 @@ +package dev.caskeleton.messaging.observation; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +/** + * The complete, closed set of metric dimensions the platform emits. + * + *

It is a fixed record rather than an open map on purpose. Every field here is bounded by + * configuration or by an enum, so the cardinality of the metric is known before it is ever scraped. + * Message ids, partition keys, tenant ids, and offsets are all deliberately absent: each of them is + * unbounded at runtime and would multiply every series by the message volume. + * + * @param broker the broker name + * @param destinationProfile the logical destination name + * @param operation the platform operation, such as publish or consume + * @param outcome the stable outcome name + * @param failureCategory the stable failure classification, or {@code none} + * @param retryStage the retry stage, or {@code none} + */ +public record MessagingTags( + String broker, + String destinationProfile, + String operation, + String outcome, + String failureCategory, + String retryStage) { + + /** The value used where a dimension does not apply. */ + public static final String NONE = "none"; + + public MessagingTags { + Objects.requireNonNull(broker, "broker must not be null"); + Objects.requireNonNull(destinationProfile, "destinationProfile must not be null"); + Objects.requireNonNull(operation, "operation must not be null"); + Objects.requireNonNull(outcome, "outcome must not be null"); + Objects.requireNonNull(failureCategory, "failureCategory must not be null"); + Objects.requireNonNull(retryStage, "retryStage must not be null"); + } + + /** + * Builds tags for a successful operation. + * + * @param broker the broker name + * @param destinationProfile the logical destination + * @param operation the operation name + * @param outcome the outcome name + * @return the tag set + */ + public static MessagingTags of( + String broker, String destinationProfile, String operation, String outcome) { + return new MessagingTags(broker, destinationProfile, operation, outcome, NONE, NONE); + } + + /** + * Returns the tags as an ordered map for a metrics backend. + * + * @return the tag map + */ + public Map asMap() { + Map tags = new LinkedHashMap<>(); + tags.put("broker", broker); + tags.put("destinationProfile", destinationProfile); + tags.put("operation", operation); + tags.put("outcome", outcome); + tags.put("failureCategory", failureCategory); + tags.put("retryStage", retryStage); + return Map.copyOf(tags); + } +} diff --git a/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingTracer.java b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingTracer.java new file mode 100644 index 00000000..9e81afac --- /dev/null +++ b/src/messaging/messaging-observability/src/main/java/dev/caskeleton/messaging/observation/MessagingTracer.java @@ -0,0 +1,96 @@ +package dev.caskeleton.messaging.observation; + +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.header.HeaderName; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Carries trace context across the broker hop. + * + *

Messaging breaks in-process trace propagation: the publish and the consume happen in different + * processes, often minutes apart, so the only way the two spans meet is if the context travels in + * the message headers. W3C {@code traceparent}/{@code tracestate} are used rather than a private + * format so that a non-Java consumer, or a broker-side tool, can still join the trace. + * + *

The consume side is deliberately a link rather than a child span in the general case. + * A batch consume can draw messages from many unrelated traces, and forcing them into one parent + * would invent a causal relationship that does not exist. Retry and dead-letter hops keep the + * original trace so a message's whole journey stays one story. + */ +public final class MessagingTracer { + + /** The W3C trace context header. */ + public static final String TRACEPARENT = "traceparent"; + + /** The W3C trace state header. */ + public static final String TRACESTATE = "tracestate"; + + /** The W3C baggage header. */ + public static final String BAGGAGE = "baggage"; + + /** + * Writes trace context into platform headers. + * + *

Written as platform headers, not application headers, so that an application cannot + * overwrite them and silently sever the trace. + * + * @param context the trace context to propagate + * @param headers the headers to extend + * @return the headers including trace context + */ + public MessageHeaders inject(TraceContext context, MessageHeaders headers) { + Objects.requireNonNull(context, "context must not be null"); + Objects.requireNonNull(headers, "headers must not be null"); + if (context.traceparent().isEmpty()) { + return headers; + } + + Map values = new java.util.LinkedHashMap<>(headers.asMap()); + values.put(new HeaderName(TRACEPARENT), new HeaderValue(context.traceparent().get())); + context + .tracestate() + .ifPresent(state -> values.put(new HeaderName(TRACESTATE), new HeaderValue(state))); + context.baggage().ifPresent(bag -> values.put(new HeaderName(BAGGAGE), new HeaderValue(bag))); + return MessageHeaders.platform(values); + } + + /** + * Reads trace context out of received headers. + * + * @param headers the received headers + * @return the propagated context, or {@link TraceContext#none()} when none was carried + */ + public TraceContext extract(MessageHeaders headers) { + Objects.requireNonNull(headers, "headers must not be null"); + Optional traceparent = headerValue(headers, TRACEPARENT); + if (traceparent.isEmpty()) { + return TraceContext.none(); + } + return new TraceContext( + traceparent, headerValue(headers, TRACESTATE), headerValue(headers, BAGGAGE)); + } + + /** + * Reports whether a consume should link to the publish trace rather than continue it. + * + *

A single delivery continues the producer's trace. A batch links instead, because a batch can + * span unrelated traces and picking one of them as the parent would be a fabrication. + * + * @param batchSize the number of messages handled together + * @return true when the consume span should use a link + */ + public boolean shouldLinkRatherThanContinue(int batchSize) { + if (batchSize < 1) { + throw new IllegalArgumentException("batchSize must be at least 1"); + } + return batchSize > 1; + } + + private static Optional headerValue(MessageHeaders headers, String name) { + return headers.find(name).map(HeaderValue::value); + } +} diff --git a/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingMetricCardinalityTest.java b/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingMetricCardinalityTest.java new file mode 100644 index 00000000..6c876962 --- /dev/null +++ b/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingMetricCardinalityTest.java @@ -0,0 +1,110 @@ +package dev.caskeleton.messaging.observation; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.time.Duration; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class MessagingMetricCardinalityTest { + + private final SimpleMeterRegistry registry = new SimpleMeterRegistry(); + + private MessagingMetrics metrics(int limit) { + return new MessagingMetrics(registry, new CardinalityGuard(limit), new MessagingRedactor()); + } + + private static MessagingTags tagsFor(String destination) { + return MessagingTags.of("kafka", destination, "publish", "confirmed"); + } + + @Test + void aBoundedTagSetProducesOneSeries() { + metrics(10).recordPublish(tagsFor("orders.v1"), Duration.ofMillis(12)); + + assertThat(registry.find(MessagingMetrics.PUBLISH_TIMER).timers()).hasSize(1); + } + + @Test + void aDimensionThatExceedsItsLimitCreatesNoFurtherSeries() { + MessagingMetrics metrics = metrics(2); + + metrics.recordPublish(tagsFor("a"), Duration.ofMillis(1)); + metrics.recordPublish(tagsFor("b"), Duration.ofMillis(1)); + metrics.recordPublish(tagsFor("c"), Duration.ofMillis(1)); + + assertThat(registry.find(MessagingMetrics.PUBLISH_TIMER).timers()) + .as("a meter registry never forgets a series, so the third one must never be created") + .hasSize(2); + } + + @Test + void aRefusedTagSetIsCountedSoTheRejectionIsVisible() { + MessagingMetrics metrics = metrics(1); + + metrics.recordPublish(tagsFor("a"), Duration.ofMillis(1)); + metrics.recordPublish(tagsFor("b"), Duration.ofMillis(1)); + + assertThat(metrics.rejectedTagSets()).isEqualTo(1); + } + + @Test + void aRedeliveryCountsAsAnAttemptButNotAsANewMessage() { + MessagingMetrics metrics = metrics(10); + MessagingTags tags = MessagingTags.of("kafka", "orders.v1", "consume", "handled"); + + metrics.recordDelivery(tags, Duration.ofMillis(5), 1); + metrics.recordDelivery(tags, Duration.ofMillis(5), 2); + metrics.recordDelivery(tags, Duration.ofMillis(5), 3); + + assertThat(registry.find(MessagingMetrics.DELIVERY_TIMER).timer().count()) + .as("three physical attempts") + .isEqualTo(3); + assertThat(registry.find(MessagingMetrics.MESSAGE_COUNTER).counter().count()) + .as("one logical message; counting redeliveries would make a storm look like growth") + .isEqualTo(1.0); + } + + @Test + void theTagVocabularyIsClosed() { + assertThat(tagsFor("orders.v1").asMap()) + .containsOnlyKeys( + "broker", + "destinationProfile", + "operation", + "outcome", + "failureCategory", + "retryStage"); + } + + @Test + void noTagCarriesPerMessageIdentity() { + Map tags = tagsFor("orders.v1").asMap(); + + assertThat(tags.keySet()) + .as("message id, key, offset, and tenant are all unbounded at runtime") + .doesNotContain("messageId", "key", "partitionKey", "offset", "tenantId"); + } + + @Test + void aBacklogSampleReplacesRatherThanAccumulates() { + MessagingMetrics metrics = metrics(10); + MessagingTags tags = MessagingTags.of("kafka", "orders.v1", "consume", "lag"); + + metrics.recordBacklog(tags, 100); + metrics.recordBacklog(tags, 40); + + assertThat(registry.find(MessagingMetrics.BACKLOG_GAUGE).gauge().value()).isEqualTo(40.0); + } + + @Test + void anAttemptNumberBelowOneIsRejectedBecauseTheFirstDeliveryIsAttemptOne() { + MessagingMetrics metrics = metrics(10); + MessagingTags tags = MessagingTags.of("kafka", "orders.v1", "consume", "handled"); + + org.assertj.core.api.Assertions.assertThatThrownBy( + () -> metrics.recordDelivery(tags, Duration.ofMillis(1), 0)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingRedactorTest.java b/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingRedactorTest.java new file mode 100644 index 00000000..1876c387 --- /dev/null +++ b/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingRedactorTest.java @@ -0,0 +1,82 @@ +package dev.caskeleton.messaging.observation; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.LinkedHashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class MessagingRedactorTest { + + private final MessagingRedactor redactor = new MessagingRedactor(); + + @Test + void removesMessageIdentityAndSecretsFromDiagnosticMap() { + Map sanitized = + redactor.sanitize( + Map.of( + "messageId", "0190f4aa-0000-7000-8000-000000000001", + "Authorization", "Bearer secret", + "destinationProfile", "order-events")); + + assertThat(sanitized) + .containsEntry("destinationProfile", "order-events") + .doesNotContainKeys("messageId", "Authorization"); + } + + @Test + void deniesKeysRegardlessOfCase() { + Map sanitized = + redactor.sanitize(Map.of("MESSAGEID", "x", "Partitionkey", "acct-1", "broker", "kafka")); + + assertThat(sanitized).containsOnlyKeys("broker"); + } + + @Test + void removesBrokerPositionDetailThatWouldExplodeCardinality() { + Map diagnostics = new LinkedHashMap<>(); + diagnostics.put("offset", "918273"); + diagnostics.put("deliveryTag", "44"); + diagnostics.put("sequence", "7"); + diagnostics.put("operation", "consume"); + + assertThat(redactor.sanitize(diagnostics)).containsOnlyKeys("operation"); + } + + @Test + void removesPayloadAndStackTrace() { + Map sanitized = + redactor.sanitize( + Map.of( + "payload", "{\"card\":\"4111111111111111\"}", + "stackTrace", "java.lang.RuntimeException", + "exceptionMessage", "failed for user bob@example.com", + "outcome", "REJECTED")); + + assertThat(sanitized).containsOnlyKeys("outcome"); + } + + @Test + void maskKeepsTheKeyButNotTheValue() { + Map masked = + redactor.mask(Map.of("messageId", "0190f4aa", "broker", "kafka-primary")); + + assertThat(masked).containsEntry("messageId", "[redacted]"); + assertThat(masked).containsEntry("broker", "kafka-primary"); + } + + @Test + void tagsCarryOnlyBoundedDimensions() { + MessagingTags tags = MessagingTags.of("kafka-primary", "order-events", "publish", "CONFIRMED"); + + assertThat(tags.asMap()) + .containsOnlyKeys( + "broker", + "destinationProfile", + "operation", + "outcome", + "failureCategory", + "retryStage"); + assertThat(tags.asMap()).doesNotContainKey("messageId"); + } +} diff --git a/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingSecretLeakTest.java b/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingSecretLeakTest.java new file mode 100644 index 00000000..6e7647b1 --- /dev/null +++ b/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingSecretLeakTest.java @@ -0,0 +1,119 @@ +package dev.caskeleton.messaging.observation; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Proves that the values an operator can reach — metric tags and audit details — cannot carry a + * secret or a per-message identifier, whatever a call site passes in. + */ +class MessagingSecretLeakTest { + + private static final List FORBIDDEN = + List.of( + "Authorization", + "Proxy-Authorization", + "Cookie", + "Set-Cookie", + "access_token", + "refresh_token", + "api_key", + "password", + "client_secret"); + + private final SimpleMeterRegistry registry = new SimpleMeterRegistry(); + private final MessagingRedactor redactor = new MessagingRedactor(); + + @Test + void everyForbiddenHeaderNameIsDenied() { + assertThat(FORBIDDEN).allSatisfy(name -> assertThat(redactor.isDenied(name)).isTrue()); + } + + @Test + void aDenialIsCaseInsensitiveBecauseHeaderCasingIsNotStable() { + assertThat(redactor.isDenied("AUTHORIZATION")).isTrue(); + assertThat(redactor.isDenied("authorization")).isTrue(); + assertThat(redactor.isDenied("Authorization")).isTrue(); + } + + @Test + void perMessageIdentityIsStrippedAlongsideSecrets() { + Map raw = new LinkedHashMap<>(); + raw.put("messageId", "0199-abcd"); + raw.put("partitionKey", "customer-42"); + raw.put("offset", "884213"); + raw.put("broker", "kafka"); + + assertThat(redactor.sanitize(raw)) + .as("identity is what turns a bounded metric into one series per message") + .containsExactly(Map.entry("broker", "kafka")); + } + + @Test + void payloadAndExceptionDetailNeverSurvive() { + Map raw = + Map.of( + "payload", "{\"pan\":\"4111111111111111\"}", + "stackTrace", "java.lang.RuntimeException: ...", + "exceptionMessage", "user bob@example.com not found", + "outcome", "rejected"); + + assertThat(redactor.sanitize(raw)).containsOnlyKeys("outcome"); + } + + @Test + void aSecretPassedAsADiagnosticNeverReachesTheMeterRegistry() { + MessagingMetrics metrics = new MessagingMetrics(registry, new CardinalityGuard(100), redactor); + + metrics.recordDiagnostics( + MessagingTags.of("kafka", "orders.v1", "publish", "rejected"), + Map.of("password", "hunter2", "reason", "unroutable")); + + List emitted = + registry.getMeters().stream() + .map(Meter::getId) + .flatMap(id -> id.getTags().stream()) + .map(Tag::getValue) + .toList(); + + assertThat(emitted).doesNotContain("hunter2").contains("unroutable"); + } + + @Test + void maskingKeepsThePresenceSignalWithoutTheValue() { + Map masked = redactor.mask(Map.of("authorization", "Bearer abc.def.ghi")); + + assertThat(masked).containsOnlyKeys("authorization"); + assertThat(masked.get("authorization")).isEqualTo("[redacted]"); + } + + @Test + void anAuditEventCarriesTheOperatorButNotThePayload() { + MessagingAuditSink.InMemory sink = MessagingAuditSink.inMemory(); + + sink.record( + new MessagingAuditEvent( + "redrive", + "ops@example.com", + "orders.v1", + "CHG-1042", + Instant.parse("2026-08-10T09:15:00Z"), + redactor.sanitize(Map.of("payload", "secret", "count", "12")))); + + assertThat(sink.events()) + .singleElement() + .satisfies( + event -> { + assertThat(event.subject()).isEqualTo("ops@example.com"); + assertThat(event.details()).containsOnlyKeys("count"); + }); + } +} diff --git a/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingTraceLinkTest.java b/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingTraceLinkTest.java new file mode 100644 index 00000000..9e695d67 --- /dev/null +++ b/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/MessagingTraceLinkTest.java @@ -0,0 +1,88 @@ +package dev.caskeleton.messaging.observation; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.header.HeaderName; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class MessagingTraceLinkTest { + + private static final String TRACEPARENT = + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; + + private final MessagingTracer tracer = new MessagingTracer(); + + @Test + void tracecontextSurvivesTheBrokerHop() { + TraceContext original = TraceContext.of(TRACEPARENT); + + MessageHeaders carried = tracer.inject(original, MessageHeaders.empty()); + TraceContext recovered = tracer.extract(carried); + + assertThat(recovered.traceparent()) + .as("publish and consume are different processes, so the context has to travel in-band") + .hasValue(TRACEPARENT); + } + + @Test + void traceStateAndBaggageAreCarriedToo() { + TraceContext original = + new TraceContext( + Optional.of(TRACEPARENT), Optional.of("vendor=abc"), Optional.of("tier=gold")); + + TraceContext recovered = tracer.extract(tracer.inject(original, MessageHeaders.empty())); + + assertThat(recovered.tracestate()).hasValue("vendor=abc"); + assertThat(recovered.baggage()).hasValue("tier=gold"); + } + + @Test + void traceHeadersAreWrittenAsPlatformHeaders() { + MessageHeaders carried = tracer.inject(TraceContext.of(TRACEPARENT), MessageHeaders.empty()); + + assertThat(carried.find(MessagingTracer.TRACEPARENT)) + .as("an application-writable trace header could be overwritten, severing the trace") + .isPresent(); + } + + @Test + void existingHeadersArePreservedAlongsideTheTrace() { + MessageHeaders existing = + MessageHeaders.application( + Map.of(new HeaderName("x-tenant-tier"), new HeaderValue("gold"))); + + MessageHeaders carried = tracer.inject(TraceContext.of(TRACEPARENT), existing); + + assertThat(carried.find("x-tenant-tier")).hasValue(new HeaderValue("gold")); + assertThat(carried.find(MessagingTracer.TRACEPARENT)).isPresent(); + } + + @Test + void aMessageWithNoTraceExtractsToNone() { + assertThat(tracer.extract(MessageHeaders.empty())).isEqualTo(TraceContext.none()); + } + + @Test + void injectingAnInactiveTraceAddsNothing() { + MessageHeaders carried = tracer.inject(TraceContext.none(), MessageHeaders.empty()); + + assertThat(carried.size()).isZero(); + } + + @Test + void aSingleDeliveryContinuesTheProducerTrace() { + assertThat(tracer.shouldLinkRatherThanContinue(1)).isFalse(); + } + + @Test + void aBatchLinksBecauseItCanSpanUnrelatedTraces() { + assertThat(tracer.shouldLinkRatherThanContinue(25)) + .as("picking one message's trace as the parent of a mixed batch invents causality") + .isTrue(); + } +} diff --git a/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/SecretLeakStaticScanTest.java b/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/SecretLeakStaticScanTest.java new file mode 100644 index 00000000..812f0d62 --- /dev/null +++ b/src/messaging/messaging-observability/src/test/java/dev/caskeleton/messaging/observation/SecretLeakStaticScanTest.java @@ -0,0 +1,205 @@ +package dev.caskeleton.messaging.observation; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; + +/** + * Scans the messaging source tree for code that would print a secret or a payload. + * + *

A runtime redactor only protects the values that pass through it. A {@code toString()} that + * concatenates a credential, or a log line that interpolates a payload, bypasses it entirely and is + * invisible to every unit test — the leak only shows up in a production log, after the fact. A + * static scan is the cheapest way to make that class of mistake fail in CI instead. + */ +class SecretLeakStaticScanTest { + + /** Identifiers that must never be concatenated into a printed string. */ + private static final List SENSITIVE_IDENTIFIERS = + List.of("password", "secret", "credential", "token", "apikey", "payload", "passphrase"); + + /** Statements that write to a stream or build a printed representation. */ + private static final Pattern PRINTING = + Pattern.compile("(System\\.out|System\\.err|printStackTrace\\()"); + + /** A Java string literal, including escaped quotes. */ + private static final Pattern STRING_LITERAL = Pattern.compile("\"(\\\\.|[^\"\\\\])*\""); + + /** The operand immediately before or after a string concatenation. */ + private static final Pattern CONCATENATION_OPERAND = + Pattern.compile( + "(?[A-Za-z_][\\w.]*(?:\\(\\))?)\\s*\\+|\\+\\s*(?[A-Za-z_][\\w.]*(?:\\(\\))?)"); + + /** Suffixes that describe a value rather than reveal it. */ + private static final Pattern SAFE_DERIVATION = + Pattern.compile("\\.(length|size|sizeBytes|getSimpleName|getName|getClass|hashCode)\\b"); + + /** Names that reference a secret without carrying it. */ + private static final Pattern DESCRIBES_RATHER_THAN_REVEALS = + Pattern.compile("(Id|Ids|Name|Type|Count|Bytes|Length|Size|Ref|Reference)$"); + + private static Path messagingRoot() { + Path candidate = Path.of("").toAbsolutePath(); + while (candidate != null && !Files.isDirectory(candidate.resolve("messaging-core-api"))) { + candidate = candidate.getParent(); + } + if (candidate == null) { + throw new IllegalStateException( + "could not locate the messaging source root from the test cwd"); + } + return candidate; + } + + private static List productionSources() throws IOException { + try (Stream files = Files.walk(messagingRoot())) { + return files + .filter(path -> path.toString().endsWith(".java")) + .filter(path -> path.toString().contains("/src/main/java/")) + .toList(); + } + } + + @Test + void productionCodeNeverWritesToTheConsole() throws IOException { + List offenders = new ArrayList<>(); + + for (Path source : productionSources()) { + List lines = Files.readAllLines(source); + for (int index = 0; index < lines.size(); index++) { + String line = lines.get(index); + if (line.trim().startsWith("*") || line.trim().startsWith("//")) { + continue; + } + if (PRINTING.matcher(line).find()) { + offenders.add("%s:%d".formatted(source.getFileName(), index + 1)); + } + } + } + + assertThat(offenders) + .as("console output bypasses the redactor and every log-scrubbing rule downstream") + .isEmpty(); + } + + @Test + void noSensitiveIdentifierIsConcatenatedIntoAString() throws IOException { + List offenders = new ArrayList<>(); + + for (Path source : productionSources()) { + List lines = Files.readAllLines(source); + for (int index = 0; index < lines.size(); index++) { + String trimmed = lines.get(index).trim(); + if (trimmed.startsWith("*") || trimmed.startsWith("//")) { + continue; + } + if (leaksASensitiveValue(trimmed)) { + offenders.add("%s:%d %s".formatted(source.getFileName(), index + 1, trimmed)); + } + } + } + + assertThat(offenders) + .as("a concatenated secret never reaches the redactor, so it must not be written at all") + .isEmpty(); + } + + /** + * Reports whether a line appends a sensitive value to a string. + * + *

Two rounds of narrowing, each one earning its place against a real false positive found in + * this repository. + * + *

String literals are blanked first. Without that, the prose in {@code "password must not be + * null"} matches every identifier pattern and the scan becomes noise the next person disables. + * + *

Then derivations are excluded. {@code payload.length}, {@code reference.sizeBytes()}, and + * {@code payload.getClass().getSimpleName()} all mention a sensitive word while carrying no + * secret — a size and a type name are exactly what a good diagnostic prints instead of + * the value. Flagging them would train the reader to ignore this test. + * + * @param line one source line, already trimmed + * @return true when the line concatenates a sensitive value + */ + private static boolean leaksASensitiveValue(String line) { + String code = STRING_LITERAL.matcher(line).replaceAll("\"\""); + if (!code.contains("+")) { + return false; + } + for (String operand : operandsAdjacentToConcatenation(code)) { + String lower = operand.toLowerCase(Locale.ROOT); + if (SENSITIVE_IDENTIFIERS.stream().noneMatch(lower::contains)) { + continue; + } + if (SAFE_DERIVATION.matcher(operand).find()) { + continue; + } + String tail = operand.substring(operand.lastIndexOf('.') + 1); + // A SCREAMING_CASE constant is a compile-time literal in this codebase, never a runtime + // secret; the only ones matching are limits such as PayloadPolicy.HARD_MAX_BYTES. + if (tail.equals(tail.toUpperCase(Locale.ROOT))) { + continue; + } + // `credentialId` names a credential without being one. An id is what a diagnostic prints so + // that an operator can look the secret up in the store it actually lives in. + if (DESCRIBES_RATHER_THAN_REVEALS.matcher(tail).find()) { + continue; + } + return true; + } + return false; + } + + /** + * Returns the operand tokens on either side of each {@code +} in a line of code. + * + * @param code the line with string literals already blanked + * @return the adjacent operands + */ + private static List operandsAdjacentToConcatenation(String code) { + List operands = new ArrayList<>(); + var matcher = CONCATENATION_OPERAND.matcher(code); + while (matcher.find()) { + if (matcher.group("before") != null) { + operands.add(matcher.group("before")); + } + if (matcher.group("after") != null) { + operands.add(matcher.group("after")); + } + } + return operands; + } + + @Test + void theScanActuallyReachesTheSourceTree() throws IOException { + assertThat(productionSources()) + .as("a scan that finds no files would pass vacuously") + .hasSizeGreaterThan(100); + } + + @Test + void theScanCatchesAValueAppendedAfterThePlus() { + assertThat(leaksASensitiveValue("log.info(\"publishing with \" + password);")) + .as("a detector that never fires proves nothing") + .isTrue(); + } + + @Test + void theScanCatchesAValueAppendedBeforeThePlus() { + assertThat(leaksASensitiveValue("String message = clientSecret + \" was rejected\";")).isTrue(); + } + + @Test + void theScanIgnoresASensitiveWordThatIsOnlyProse() { + assertThat(leaksASensitiveValue("throw new IllegalArgumentException(\"password \" + field);")) + .as("blanking literals is what keeps the scan signal rather than noise") + .isFalse(); + } +} diff --git a/src/messaging/messaging-outbox-jpa/build.gradle b/src/messaging/messaging-outbox-jpa/build.gradle new file mode 100644 index 00000000..bffe63f2 --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/build.gradle @@ -0,0 +1,18 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-reliability-api') + api project(':messaging:messaging-policy') + api project(':messaging:messaging-observability') + + implementation 'org.springframework:spring-jdbc' + implementation 'org.springframework:spring-tx' + + // Live-database certification. The reliability patterns are claims about transaction + // boundaries and uniqueness constraints, and only a real database can settle them. + testImplementation project(':messaging:messaging-testkit') + testImplementation 'org.testcontainers:testcontainers-postgresql' + testImplementation 'org.testcontainers:testcontainers-junit-jupiter' + testImplementation 'org.postgresql:postgresql' +} diff --git a/src/messaging/messaging-outbox-jpa/gradle.lockfile b/src/messaging/messaging-outbox-jpa/gradle.lockfile new file mode 100644 index 00000000..4d7833bf --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/gradle.lockfile @@ -0,0 +1,110 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.fasterxml.jackson.core:jackson-annotations:2.20=testCompileClasspath,testRuntimeClasspath +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.docker-java:docker-java-api:3.7.0=testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport-zerodep:3.7.0=testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport:3.7.0=testCompileClasspath,testRuntimeClasspath +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-codec:commons-codec:1.19.0=testCompileClasspath,testRuntimeClasspath +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.20.0=testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.21.0=spotbugs +commons-logging:commons-logging:1.3.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +io.micrometer:micrometer-commons:1.16.0=runtimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-core:1.16.0=runtimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-observation:1.16.0=runtimeClasspath,testRuntimeClasspath +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna:5.18.1=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-compress:1.28.0=testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.checkerframework:checker-qual:3.49.5=testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.hdrhistogram:HdrHistogram:2.2.2=runtimeClasspath,testRuntimeClasspath +org.javassist:javassist:3.28.0-GA=checkstyle +org.jetbrains:annotations:17.0.0=testCompileClasspath,testRuntimeClasspath +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,compileClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.latencyutils:LatencyUtils:2.0.3=runtimeClasspath,testRuntimeClasspath +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.postgresql:postgresql:42.7.8=testCompileClasspath,testRuntimeClasspath +org.reflections:reflections:0.10.2=checkstyle +org.rnorth.duct-tape:duct-tape:1.0.8=testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.springframework:spring-beans:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-core:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-jdbc:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-tx:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-database-commons:2.0.2=testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-jdbc:2.0.2=testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-junit-jupiter:2.0.2=testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-postgresql:2.0.2=testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers:2.0.2=testCompileClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty= diff --git a/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumMappedRecord.java b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumMappedRecord.java new file mode 100644 index 00000000..cd534197 --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumMappedRecord.java @@ -0,0 +1,54 @@ +package dev.caskeleton.messaging.outbox; + +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * What one outbox row looks like after Debezium's Event Router has transformed it. + * + *

Modelled explicitly so the CDC path can be tested without running Debezium. The property that + * matters is not that the transform is invoked but that its output carries the same identity, + * destination, and reserved headers the polling relay would have produced — and that is asserted by + * comparing this record against one built from the same row. + * + * @param topic the topic the row was routed to + * @param key the broker partition key, absent when the row is unkeyed + * @param payload the encoded payload + * @param headers the reserved headers carried alongside + */ +@SuppressWarnings("ArrayRecordComponent") +public record DebeziumMappedRecord( + String topic, Optional key, byte[] payload, Map headers) { + + public DebeziumMappedRecord { + Objects.requireNonNull(key, "key must not be null"); + Objects.requireNonNull(payload, "payload must not be null"); + Objects.requireNonNull(headers, "headers must not be null"); + if (topic == null || topic.isBlank()) { + throw new IllegalArgumentException("topic must not be blank"); + } + payload = payload.clone(); + headers = Map.copyOf(headers); + } + + @Override + public byte[] payload() { + return payload.clone(); + } + + @Override + public boolean equals(Object other) { + return other instanceof DebeziumMappedRecord record + && topic.equals(record.topic) + && key.equals(record.key) + && headers.equals(record.headers) + && Arrays.equals(payload, record.payload); + } + + @Override + public int hashCode() { + return Objects.hash(topic, key, headers, Arrays.hashCode(payload)); + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxEventRouter.java b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxEventRouter.java new file mode 100644 index 00000000..efd16f6c --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxEventRouter.java @@ -0,0 +1,76 @@ +package dev.caskeleton.messaging.outbox; + +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.reliability.OutboxRecord; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Maps outbox columns onto the headers Debezium's Outbox Event Router expects. + * + *

An optional alternative to the polling relay. CDC reads the database's write-ahead log instead + * of querying the table, which removes the polling interval and the lease contention — at the cost + * of an extra piece of infrastructure and its own failure modes. + * + *

The platform contract does not change with the transport. The router emits the same reserved + * headers the polling relay does, so a consumer cannot tell which one published a message, and + * switching between them is a deployment decision rather than a contract change. + */ +public final class DebeziumOutboxEventRouter { + + /** The outbox column Debezium uses as the message key. */ + public static final String ROUTE_BY_COLUMN = "destination"; + + /** The outbox column carrying the logical message id. */ + public static final String ID_COLUMN = "message_id"; + + /** The outbox column carrying the payload. */ + public static final String PAYLOAD_COLUMN = "payload"; + + /** + * Returns the router configuration for a Debezium connector. + * + * @param topicPrefix the prefix routed topics are created under + * @return the connector properties + */ + public Map connectorConfiguration(String topicPrefix) { + Objects.requireNonNull(topicPrefix, "topicPrefix must not be null"); + Map configuration = new LinkedHashMap<>(); + configuration.put("transforms.outbox.type", "io.debezium.transforms.outbox.EventRouter"); + configuration.put("transforms.outbox.table.field.event.id", ID_COLUMN); + configuration.put("transforms.outbox.table.field.event.key", ROUTE_BY_COLUMN); + configuration.put("transforms.outbox.table.field.event.payload", PAYLOAD_COLUMN); + configuration.put("transforms.outbox.route.by.field", ROUTE_BY_COLUMN); + configuration.put( + "transforms.outbox.route.topic.replacement", topicPrefix + "${routedByValue}"); + configuration.put( + "transforms.outbox.table.fields.additional.placement", + String.join( + ",", + "message_type:header:" + ReservedHeaders.MESSAGE_TYPE, + "schema_version:header:" + ReservedHeaders.SCHEMA_VERSION, + "content_type:header:" + ReservedHeaders.CONTENT_TYPE, + "created_at:header:" + ReservedHeaders.PRODUCED_AT)); + return Map.copyOf(configuration); + } + + /** + * Returns the reserved headers a routed record must carry. + * + *

Used to assert that the CDC path and the polling path produce the same wire contract. + * + * @param record the outbox row + * @return the reserved headers + */ + public Map reservedHeaders(OutboxRecord record) { + Objects.requireNonNull(record, "record must not be null"); + Map headers = new LinkedHashMap<>(); + headers.put(ReservedHeaders.MESSAGE_ID, record.messageId().value().toString()); + headers.put(ReservedHeaders.MESSAGE_TYPE, record.messageType().value()); + headers.put(ReservedHeaders.SCHEMA_VERSION, Integer.toString(record.schemaVersion().value())); + headers.put(ReservedHeaders.CONTENT_TYPE, record.contentType().value()); + headers.put(ReservedHeaders.PRODUCED_AT, record.createdAt().toString()); + return Map.copyOf(headers); + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxProfile.java b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxProfile.java new file mode 100644 index 00000000..4c085727 --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxProfile.java @@ -0,0 +1,67 @@ +package dev.caskeleton.messaging.outbox; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.util.Objects; + +/** + * Which relay a deployment uses, and the rule that only one of them may be active. + * + *

Running the polling relay and the CDC relay together publishes every row twice. Both are + * correct in isolation and neither can see the other: the poller leases and marks rows, while + * Debezium reads the write-ahead log and never observes the lease at all. The duplicate is + * therefore invisible in the outbox table and only shows up downstream, which is why the + * incompatibility is enforced at startup instead of documented. + * + * @param mode which relay publishes outbox rows + * @param topicPrefix the prefix CDC-routed topics are created under + * @param aggregateIdAsPartitionKey whether the aggregate id becomes the broker partition key + */ +public record DebeziumOutboxProfile( + RelayMode mode, String topicPrefix, boolean aggregateIdAsPartitionKey) { + + /** How outbox rows reach the broker. */ + public enum RelayMode { + /** The in-process relay leases and publishes rows. */ + POLLING, + /** Debezium reads the write-ahead log and routes rows. */ + CHANGE_DATA_CAPTURE + } + + public DebeziumOutboxProfile { + Objects.requireNonNull(mode, "mode must not be null"); + Objects.requireNonNull(topicPrefix, "topicPrefix must not be null"); + if (mode == RelayMode.CHANGE_DATA_CAPTURE && topicPrefix.isBlank()) { + throw new IllegalArgumentException("a CDC profile must name a topic prefix"); + } + } + + /** + * Returns the default polling profile. + * + * @return the polling profile + */ + public static DebeziumOutboxProfile polling() { + return new DebeziumOutboxProfile(RelayMode.POLLING, "", false); + } + + /** + * Refuses a configuration that would run both relays at once. + * + * @param pollingRelayEnabled whether the in-process relay bean is active + * @throws MessagingConfigurationException when both relays would run + */ + public void requireExactlyOneRelay(boolean pollingRelayEnabled) { + if (mode == RelayMode.CHANGE_DATA_CAPTURE && pollingRelayEnabled) { + throw new MessagingConfigurationException( + "DUPLICATE_OUTBOX_RELAY", + "the CDC relay and the polling relay are both enabled; Debezium never observes the " + + "polling lease, so every row would be published twice"); + } + if (mode == RelayMode.POLLING && !pollingRelayEnabled) { + throw new MessagingConfigurationException( + "NO_OUTBOX_RELAY", + "the profile selects the polling relay but no relay is enabled, so outbox rows would " + + "accumulate without ever being published"); + } + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapper.java b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapper.java new file mode 100644 index 00000000..a1480fbe --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapper.java @@ -0,0 +1,71 @@ +package dev.caskeleton.messaging.outbox; + +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.reliability.OutboxRecord; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Produces what Debezium's Event Router will emit for an outbox row. + * + *

Exists so the CDC path is testable without Debezium running. The property worth proving is + * that a consumer cannot tell which relay published a message: same logical id, same destination, + * same reserved headers. Building the expected record here and comparing it against the polling + * relay's output is what makes "switching relays is a deployment decision, not a contract change" + * an assertion rather than a claim. + * + *

The message id comes from the row's {@code message_id} column, never from Debezium's own event + * id. Debezium mints a fresh event id per change record, so using it would break identity across a + * CDC re-snapshot — exactly the scenario where identity matters most. + */ +public final class DebeziumOutboxRecordMapper { + + private final String topicPrefix; + private final boolean aggregateIdAsPartitionKey; + + /** + * Creates a mapper from a CDC profile. + * + * @param profile the outbox relay profile + */ + public DebeziumOutboxRecordMapper(DebeziumOutboxProfile profile) { + Objects.requireNonNull(profile, "profile must not be null"); + this.topicPrefix = profile.topicPrefix(); + this.aggregateIdAsPartitionKey = profile.aggregateIdAsPartitionKey(); + } + + /** + * Maps one outbox row to its routed record. + * + * @param record the outbox row + * @return the record a CDC relay would emit + */ + public DebeziumMappedRecord map(OutboxRecord record) { + Objects.requireNonNull(record, "record must not be null"); + + Map headers = new LinkedHashMap<>(record.headers()); + headers.put(ReservedHeaders.MESSAGE_ID, record.messageId().value().toString()); + headers.put(ReservedHeaders.MESSAGE_TYPE, record.messageType().value()); + headers.put(ReservedHeaders.SCHEMA_VERSION, Integer.toString(record.schemaVersion().value())); + headers.put(ReservedHeaders.CONTENT_TYPE, record.contentType().value()); + headers.put(ReservedHeaders.PRODUCED_AT, record.createdAt().toString()); + + return new DebeziumMappedRecord( + topicPrefix + record.destination().value(), + partitionKey(record, headers), + record.payload(), + headers); + } + + private Optional partitionKey(OutboxRecord record, Map headers) { + if (!aggregateIdAsPartitionKey) { + return Optional.empty(); + } + // The partition key header is the one the polling relay would also have set, so both relays + // land the same message on the same partition and neither reorders a keyed stream. + return Optional.ofNullable(headers.get(ReservedHeaders.PARTITION_KEY)) + .or(() -> Optional.of(record.messageId().value().toString())); + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/JdbcOutboxRepository.java b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/JdbcOutboxRepository.java new file mode 100644 index 00000000..ced99d09 --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/JdbcOutboxRepository.java @@ -0,0 +1,332 @@ +package dev.caskeleton.messaging.outbox; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.reliability.OutboxRecord; +import dev.caskeleton.messaging.reliability.OutboxRepository; +import dev.caskeleton.messaging.reliability.OutboxStatus; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import javax.sql.DataSource; + +/** + * The PostgreSQL outbox. + * + *

{@link #append} deliberately takes no connection of its own: it uses the one the caller is + * already inside, which is the entire mechanism. An outbox row written on a separate connection + * commits independently of the business change and reopens the window the pattern exists to close. + * + *

Leasing uses {@code SELECT ... FOR UPDATE SKIP LOCKED}. Two relay instances polling the same + * table would otherwise both claim the same rows and publish them twice; {@code SKIP LOCKED} lets + * the second instance step over what the first already holds instead of blocking behind it. + */ +public final class JdbcOutboxRepository implements OutboxRepository { + + private static final String INSERT = + """ + INSERT INTO messaging_outbox ( + message_id, destination, message_type, schema_version, content_type, + payload, headers, created_at, status, attempts) + VALUES (?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?, ?) + """; + + private static final String LEASE = + """ + WITH claimable AS ( + SELECT message_id + FROM messaging_outbox + -- IN_FLIGHT is claimable too, but only once its lease has expired. A relay that dies + -- mid-publish leaves rows in that state; excluding them would strand those messages + -- forever, which is the one failure mode the outbox exists to prevent. + WHERE status IN ('PENDING', 'AMBIGUOUS', 'IN_FLIGHT') + AND (lease_expires_at IS NULL OR lease_expires_at <= ?) + ORDER BY created_at + LIMIT ? + FOR UPDATE SKIP LOCKED + ) + UPDATE messaging_outbox o + SET status = 'IN_FLIGHT', lease_expires_at = ? + FROM claimable c + WHERE o.message_id = c.message_id + RETURNING o.message_id, o.destination, o.message_type, o.schema_version, o.content_type, + o.payload, o.headers, o.created_at, o.status, o.attempts, + o.lease_expires_at, o.last_failure_code + """; + + private final DataSource dataSource; + + /** + * Creates a repository over a data source. + * + * @param dataSource the outbox data source + */ + public JdbcOutboxRepository(DataSource dataSource) { + this.dataSource = Objects.requireNonNull(dataSource, "dataSource must not be null"); + } + + /** + * Appends a record on an existing connection. + * + *

This overload is the one business code should use, passing the connection its own + * transaction is running on. + * + * @param connection the caller's transactional connection + * @param record the record to write + */ + public void append(Connection connection, OutboxRecord record) { + Objects.requireNonNull(connection, "connection must not be null"); + Objects.requireNonNull(record, "record must not be null"); + try (PreparedStatement statement = connection.prepareStatement(INSERT)) { + statement.setObject(1, record.messageId().value()); + statement.setString(2, record.destination().value()); + statement.setString(3, record.messageType().value()); + statement.setInt(4, record.schemaVersion().value()); + statement.setString(5, record.contentType().value()); + statement.setBytes(6, record.payload()); + statement.setString(7, toJson(record.headers())); + statement.setTimestamp(8, Timestamp.from(record.createdAt())); + statement.setString(9, record.status().name()); + statement.setInt(10, record.attempts()); + statement.executeUpdate(); + } catch (SQLException exception) { + throw new MessagingConfigurationException( + "OUTBOX_APPEND_FAILED", "could not append an outbox record", exception); + } + } + + @Override + public void append(OutboxRecord record) { + withConnection( + connection -> { + append(connection, record); + return null; + }); + } + + @Override + public List leaseBatch(int batchSize, Duration leaseDuration, Instant now) { + Objects.requireNonNull(leaseDuration, "leaseDuration must not be null"); + Objects.requireNonNull(now, "now must not be null"); + return withConnection( + connection -> { + List leased = new ArrayList<>(); + try (PreparedStatement statement = connection.prepareStatement(LEASE)) { + statement.setTimestamp(1, Timestamp.from(now)); + statement.setInt(2, batchSize); + statement.setTimestamp(3, Timestamp.from(now.plus(leaseDuration))); + try (ResultSet results = statement.executeQuery()) { + while (results.next()) { + leased.add(read(results)); + } + } + } + return List.copyOf(leased); + }); + } + + @Override + public void markPublished(MessageId messageId, Instant now) { + update( + "UPDATE messaging_outbox SET status = 'PUBLISHED', published_at = ?, " + + "lease_expires_at = NULL, attempts = attempts + 1 WHERE message_id = ?", + messageId, + now, + null); + } + + @Override + public void markAmbiguous(MessageId messageId, String failureCode, Instant now) { + // The lease is released rather than held: an ambiguous row must become claimable again so the + // relay retries it under the same message id. + update( + "UPDATE messaging_outbox SET status = 'AMBIGUOUS', lease_expires_at = NULL, " + + "last_failure_code = ?, attempts = attempts + 1 WHERE message_id = ?", + messageId, + now, + failureCode); + } + + @Override + public void markFailed(MessageId messageId, String failureCode, Instant now) { + update( + "UPDATE messaging_outbox SET status = 'FAILED', lease_expires_at = NULL, " + + "last_failure_code = ?, attempts = attempts + 1 WHERE message_id = ?", + messageId, + now, + failureCode); + } + + @Override + public void releaseLease(MessageId messageId) { + withConnection( + connection -> { + try (PreparedStatement statement = + connection.prepareStatement( + "UPDATE messaging_outbox SET status = 'PENDING', lease_expires_at = NULL " + + "WHERE message_id = ? AND status = 'IN_FLIGHT'")) { + statement.setObject(1, messageId.value()); + statement.executeUpdate(); + } + return null; + }); + } + + @Override + public Optional find(MessageId messageId) { + return withConnection( + connection -> { + try (PreparedStatement statement = + connection.prepareStatement( + "SELECT message_id, destination, message_type, schema_version, content_type, " + + "payload, headers, created_at, status, attempts, lease_expires_at, " + + "last_failure_code FROM messaging_outbox WHERE message_id = ?")) { + statement.setObject(1, messageId.value()); + try (ResultSet results = statement.executeQuery()) { + return results.next() ? Optional.of(read(results)) : Optional.empty(); + } + } + }); + } + + @Override + public int purgePublishedBefore(Instant publishedBefore) { + Objects.requireNonNull(publishedBefore, "publishedBefore must not be null"); + return withConnection( + connection -> { + try (PreparedStatement statement = + connection.prepareStatement( + "DELETE FROM messaging_outbox WHERE status = 'PUBLISHED' AND published_at < ?")) { + statement.setTimestamp(1, Timestamp.from(publishedBefore)); + return statement.executeUpdate(); + } + }); + } + + private void update(String sql, MessageId messageId, Instant now, String failureCode) { + withConnection( + connection -> { + try (PreparedStatement statement = connection.prepareStatement(sql)) { + if (failureCode == null) { + statement.setTimestamp(1, Timestamp.from(now)); + } else { + statement.setString(1, failureCode); + } + statement.setObject(2, messageId.value()); + statement.executeUpdate(); + } + return null; + }); + } + + private static OutboxRecord read(ResultSet results) throws SQLException { + Timestamp lease = results.getTimestamp("lease_expires_at"); + return new OutboxRecord( + new MessageId(results.getObject("message_id", java.util.UUID.class)), + new DestinationName(results.getString("destination")), + new MessageType(results.getString("message_type")), + new SchemaVersion(results.getInt("schema_version")), + new ContentType(results.getString("content_type")), + results.getBytes("payload"), + fromJson(results.getString("headers")), + results.getTimestamp("created_at").toInstant(), + OutboxStatus.valueOf(results.getString("status")), + results.getInt("attempts"), + Optional.ofNullable(lease).map(Timestamp::toInstant), + Optional.ofNullable(results.getString("last_failure_code"))); + } + + private T withConnection(ConnectionCallback callback) { + try (Connection connection = dataSource.getConnection()) { + return callback.apply(connection); + } catch (SQLException exception) { + throw new MessagingConfigurationException( + "OUTBOX_QUERY_FAILED", "an outbox query failed", exception); + } + } + + /** + * Serialises headers as a flat JSON object. + * + *

Hand-rolled rather than pulled from a JSON library so this module keeps no codec dependency: + * outbox headers are always flat string pairs, validated by {@code MessageHeaders} before they + * ever reach here. + */ + private static String toJson(Map headers) { + StringBuilder json = new StringBuilder("{"); + boolean first = true; + for (Map.Entry entry : headers.entrySet()) { + if (!first) { + json.append(','); + } + first = false; + json.append('"') + .append(escape(entry.getKey())) + .append("\":\"") + .append(escape(entry.getValue())) + .append('"'); + } + return json.append('}').toString(); + } + + private static Map fromJson(String json) { + Map headers = new LinkedHashMap<>(); + if (json == null || json.length() <= 2) { + return headers; + } + String body = json.substring(1, json.length() - 1); + int index = 0; + while (index < body.length()) { + int keyStart = body.indexOf('"', index); + if (keyStart < 0) { + break; + } + int keyEnd = findClosingQuote(body, keyStart + 1); + int valueStart = body.indexOf('"', body.indexOf(':', keyEnd)); + int valueEnd = findClosingQuote(body, valueStart + 1); + headers.put( + unescape(body.substring(keyStart + 1, keyEnd)), + unescape(body.substring(valueStart + 1, valueEnd))); + index = valueEnd + 1; + } + return headers; + } + + private static int findClosingQuote(String text, int from) { + for (int index = from; index < text.length(); index++) { + if (text.charAt(index) == '"' && text.charAt(index - 1) != '\\') { + return index; + } + } + return text.length(); + } + + private static String escape(String value) { + return value.replace("\\", "\\\\").replace("\"", "\\\""); + } + + private static String unescape(String value) { + return value.replace("\\\"", "\"").replace("\\\\", "\\"); + } + + /** A unit of work against a borrowed connection. */ + @FunctionalInterface + private interface ConnectionCallback { + + T apply(Connection connection) throws SQLException; + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxCleanupJob.java b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxCleanupJob.java new file mode 100644 index 00000000..2bed4fcd --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxCleanupJob.java @@ -0,0 +1,58 @@ +package dev.caskeleton.messaging.outbox; + +import dev.caskeleton.messaging.reliability.OutboxRepository; +import java.time.Instant; +import java.util.Objects; + +/** + * Deletes published outbox rows once they are past their retention. + * + *

Only {@code PUBLISHED} rows are eligible, and that restriction is the whole safety story. An + * {@code AMBIGUOUS} row is a message that may or may not have reached the broker; deleting it + * destroys the only record that it was ever meant to be sent. A {@code FAILED} row is the audit + * trail of a message that never went out. Both are the rows an operator most needs during an + * incident, which is exactly when a time-based sweep would otherwise remove them. + */ +public final class OutboxCleanupJob { + + private final OutboxRepository outbox; + private final OutboxProperties properties; + private final int maxBatches; + + /** + * Creates a cleanup job. + * + * @param outbox the outbox storage + * @param properties the relay settings supplying the retention + * @param maxBatches how many delete batches one run may issue + */ + public OutboxCleanupJob(OutboxRepository outbox, OutboxProperties properties, int maxBatches) { + this.outbox = Objects.requireNonNull(outbox, "outbox must not be null"); + this.properties = Objects.requireNonNull(properties, "properties must not be null"); + if (maxBatches < 1) { + throw new IllegalArgumentException("maxBatches must be at least 1"); + } + this.maxBatches = maxBatches; + } + + /** + * Deletes expired published rows. + * + * @param now the current instant + * @return how many rows were removed + */ + public int runOnce(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + Instant cutoff = now.minus(properties.retentionAfterPublish()); + + int removed = 0; + for (int batch = 0; batch < maxBatches; batch++) { + int deleted = outbox.purgePublishedBefore(cutoff); + removed += deleted; + if (deleted == 0) { + break; + } + } + return removed; + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxEnvelopeFactory.java b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxEnvelopeFactory.java new file mode 100644 index 00000000..1eebb542 --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxEnvelopeFactory.java @@ -0,0 +1,68 @@ +package dev.caskeleton.messaging.outbox; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.header.HeaderName; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.reliability.OutboxRecord; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Rebuilds a publishable envelope from an outbox row. + * + *

The identity comes from the row, never from a fresh mint. That is what makes a retry after an + * ambiguous publish safe: the broker's deduplication, or the consumer's Inbox, sees the same id and + * collapses the duplicate. + */ +public final class OutboxEnvelopeFactory { + + private final ProducerId producer; + + /** + * Creates a factory for one service. + * + * @param producer the logical producing service + */ + public OutboxEnvelopeFactory(ProducerId producer) { + this.producer = Objects.requireNonNull(producer, "producer must not be null"); + } + + /** + * Builds the envelope for one outbox row. + * + * @param record the outbox row + * @return the encoded envelope + */ + public MessageEnvelope toEnvelope(OutboxRecord record) { + Objects.requireNonNull(record, "record must not be null"); + + Map headers = new LinkedHashMap<>(); + record + .headers() + .forEach((name, value) -> headers.put(new HeaderName(name), new HeaderValue(value))); + + return new MessageEnvelope<>( + record.messageId(), + record.messageType(), + record.schemaVersion(), + record.createdAt(), + Optional.of(record.createdAt()), + producer, + Optional.empty(), + Optional.empty(), + record.contentType(), + Optional.ofNullable(record.headers().get(ReservedHeaders.PARTITION_KEY)), + Optional.ofNullable(record.headers().get(ReservedHeaders.ORDERING_KEY)), + Optional.empty(), + TraceContext.none(), + MessageHeaders.platform(headers), + new EncodedMessage(record.payload(), record.contentType(), Optional.empty())); + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxProperties.java b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxProperties.java new file mode 100644 index 00000000..56f4f5ff --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxProperties.java @@ -0,0 +1,81 @@ +package dev.caskeleton.messaging.outbox; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.time.Duration; +import java.util.Objects; + +/** + * The relay's operational settings, and the relationships between them that have to hold. + * + *

The lease duration is the dangerous one. If it is shorter than the time a publish can take, a + * second relay claims the row while the first is still waiting for a confirm, and the message is + * published twice — under the same id, so consumers with an inbox survive it, but consumers without + * one do not. The constructor therefore requires the lease to exceed the publish timeout by a + * margin rather than merely to be positive. + * + * @param batchSize how many records one lease claims + * @param leaseDuration how long a claim holds + * @param publishTimeout the deadline for one publish + * @param pollInterval how often the relay looks for work + * @param retentionAfterPublish how long published rows are kept before deletion + * @param maxAttempts how many times a record is retried before it is parked + */ +public record OutboxProperties( + int batchSize, + Duration leaseDuration, + Duration publishTimeout, + Duration pollInterval, + Duration retentionAfterPublish, + int maxAttempts) { + + /** How much longer than the publish timeout a lease must be. */ + public static final double REQUIRED_LEASE_FACTOR = 2.0; + + public OutboxProperties { + Objects.requireNonNull(leaseDuration, "leaseDuration must not be null"); + Objects.requireNonNull(publishTimeout, "publishTimeout must not be null"); + Objects.requireNonNull(pollInterval, "pollInterval must not be null"); + Objects.requireNonNull(retentionAfterPublish, "retentionAfterPublish must not be null"); + if (batchSize < 1) { + throw new IllegalArgumentException("batchSize must be at least 1"); + } + if (maxAttempts < 1) { + throw new IllegalArgumentException("maxAttempts must be at least 1"); + } + requirePositive(leaseDuration, "leaseDuration"); + requirePositive(publishTimeout, "publishTimeout"); + requirePositive(pollInterval, "pollInterval"); + requirePositive(retentionAfterPublish, "retentionAfterPublish"); + + Duration minimumLease = + Duration.ofMillis(Math.round(publishTimeout.toMillis() * REQUIRED_LEASE_FACTOR)); + if (leaseDuration.compareTo(minimumLease) < 0) { + throw new MessagingConfigurationException( + "OUTBOX_LEASE_TOO_SHORT", + "a lease of %s can expire while a %s publish is still in flight, letting a second relay" + .formatted(leaseDuration, publishTimeout) + + " publish the same record; the minimum is %s".formatted(minimumLease)); + } + } + + /** + * Returns settings suited to a single relay against a healthy broker. + * + * @return the default properties + */ + public static OutboxProperties defaults() { + return new OutboxProperties( + 100, + Duration.ofSeconds(30), + Duration.ofSeconds(5), + Duration.ofMillis(500), + Duration.ofDays(3), + 10); + } + + private static void requirePositive(Duration value, String field) { + if (value.isNegative() || value.isZero()) { + throw new IllegalArgumentException(field + " must be positive"); + } + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelay.java b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelay.java new file mode 100644 index 00000000..5668df64 --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelay.java @@ -0,0 +1,114 @@ +package dev.caskeleton.messaging.outbox; + +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.reliability.OutboxRecord; +import dev.caskeleton.messaging.reliability.OutboxRepository; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Objects; + +/** + * Publishes outbox rows, treating an unknown outcome as retryable rather than final. + * + *

The relay's correctness rests on one rule: an ambiguous publish is retried under the same + * message id. Minting a new id would turn a possibly-delivered message into a + * definitely-second message, and no downstream deduplication could recover from it. Marking it + * failed instead would lose a message the broker may already hold. + * + *

The relay therefore guarantees at-least-once publication and nothing more. Effectively-once + * downstream effects come from pairing it with an Inbox — which is why the platform never + * advertises the outbox as exactly-once. + */ +public final class OutboxRelay { + + private final OutboxRepository repository; + private final MessagePublisher publisher; + private final OutboxEnvelopeFactory envelopeFactory; + private final int batchSize; + private final Duration leaseDuration; + + /** + * Creates a relay. + * + * @param repository the outbox storage + * @param publisher the platform publisher + * @param envelopeFactory rebuilds envelopes from rows + * @param batchSize how many rows to claim per pass + * @param leaseDuration how long a claim holds + */ + public OutboxRelay( + OutboxRepository repository, + MessagePublisher publisher, + OutboxEnvelopeFactory envelopeFactory, + int batchSize, + Duration leaseDuration) { + this.repository = Objects.requireNonNull(repository, "repository must not be null"); + this.publisher = Objects.requireNonNull(publisher, "publisher must not be null"); + this.envelopeFactory = + Objects.requireNonNull(envelopeFactory, "envelopeFactory must not be null"); + this.leaseDuration = Objects.requireNonNull(leaseDuration, "leaseDuration must not be null"); + if (batchSize < 1) { + throw new IllegalArgumentException("batchSize must be at least 1"); + } + this.batchSize = batchSize; + } + + /** + * Runs one relay pass. + * + * @param now the current instant + * @return what the pass did + */ + public OutboxRelayReport runOnce(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + List batch = repository.leaseBatch(batchSize, leaseDuration, now); + + int published = 0; + int ambiguous = 0; + int failed = 0; + + for (OutboxRecord record : batch) { + PublishResult result = publish(record); + switch (result.completion()) { + case CONFIRMED -> { + repository.markPublished(record.messageId(), now); + published++; + } + case AMBIGUOUS -> { + repository.markAmbiguous( + record.messageId(), + result.failure().map(failure -> failure.code()).orElse("AMBIGUOUS"), + now); + ambiguous++; + } + case REJECTED -> { + repository.markFailed( + record.messageId(), + result.failure().map(failure -> failure.code()).orElse("REJECTED"), + now); + failed++; + } + // Unreachable while PublishCompletion has exactly these constants. Present so that a new + // completion fails loudly rather than leaving the row IN_FLIGHT with its lease ticking — + // which would look like a stalled relay rather than an unhandled case. + default -> + throw new IllegalStateException("unhandled publish completion: " + result.completion()); + } + } + return new OutboxRelayReport(batch.size(), published, ambiguous, failed); + } + + private PublishResult publish(OutboxRecord record) { + MessageDestination destination = + new MessageDestination<>(record.destination(), record.messageType(), EncodedMessage.class); + return publisher + .publish(destination, envelopeFactory.toEnvelope(record), PublishOptions.defaults()) + .toCompletableFuture() + .join(); + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelayReport.java b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelayReport.java new file mode 100644 index 00000000..3632931c --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelayReport.java @@ -0,0 +1,22 @@ +package dev.caskeleton.messaging.outbox; + +/** + * What one relay pass did. + * + *

Ambiguous rows are counted separately from failures because they mean something operationally + * different: a rising ambiguous count is a broker confirmation problem, while a rising failed count + * is a contract or topology problem. + * + * @param leased how many rows were claimed + * @param published how many were confirmed + * @param ambiguous how many had an unknown outcome and stay retryable + * @param failed how many were definitively rejected + */ +public record OutboxRelayReport(int leased, int published, int ambiguous, int failed) { + + public OutboxRelayReport { + if (leased < 0 || published < 0 || ambiguous < 0 || failed < 0) { + throw new IllegalArgumentException("relay counters must not be negative"); + } + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxRetryScheduler.java b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxRetryScheduler.java new file mode 100644 index 00000000..fd01c886 --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/main/java/dev/caskeleton/messaging/outbox/OutboxRetryScheduler.java @@ -0,0 +1,103 @@ +package dev.caskeleton.messaging.outbox; + +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * Decides when a relay's next pass should run, and when a record has retried enough. + * + *

The backoff is on the relay, not on the individual record. A broker outage fails + * every record in the batch for the same reason, so backing off per record would hammer the broker + * with the whole backlog on a schedule while achieving nothing; backing off the pass lets the + * broker recover. + * + *

Jitter is applied deterministically from the attempt count rather than randomly. Several relay + * instances that all started at deployment time would otherwise synchronise their retries into a + * thundering herd, and a random source would make the schedule impossible to test. + */ +public final class OutboxRetryScheduler { + + private final Duration baseInterval; + private final Duration maxInterval; + private final int maxAttempts; + + /** + * Creates a scheduler from the relay properties. + * + * @param properties the relay settings + * @param maxInterval the longest backoff between passes + */ + public OutboxRetryScheduler(OutboxProperties properties, Duration maxInterval) { + Objects.requireNonNull(properties, "properties must not be null"); + Objects.requireNonNull(maxInterval, "maxInterval must not be null"); + if (maxInterval.compareTo(properties.pollInterval()) < 0) { + throw new IllegalArgumentException("maxInterval must not be shorter than the poll interval"); + } + this.baseInterval = properties.pollInterval(); + this.maxInterval = maxInterval; + this.maxAttempts = properties.maxAttempts(); + } + + /** + * Returns when the next pass should run. + * + *

A pass that published something resets to the base interval: work is flowing, so there is no + * reason to wait. + * + * @param now the current instant + * @param consecutiveEmptyOrFailedPasses how many passes in a row did no useful work + * @return the next pass instant + */ + public Instant nextPassAt(Instant now, int consecutiveEmptyOrFailedPasses) { + Objects.requireNonNull(now, "now must not be null"); + if (consecutiveEmptyOrFailedPasses < 0) { + throw new IllegalArgumentException("consecutiveEmptyOrFailedPasses must not be negative"); + } + return now.plus(backoff(consecutiveEmptyOrFailedPasses)); + } + + /** + * Returns the delay before the next pass. + * + * @param consecutiveEmptyOrFailedPasses how many passes in a row did no useful work + * @return the backoff delay + */ + public Duration backoff(int consecutiveEmptyOrFailedPasses) { + if (consecutiveEmptyOrFailedPasses <= 0) { + return baseInterval; + } + // Doubling, capped. Shifting rather than Math.pow so an unbounded counter cannot overflow into + // a negative duration on a long outage. + int exponent = Math.min(consecutiveEmptyOrFailedPasses, 20); + long scaled = baseInterval.toMillis() << exponent; + long capped = Math.min(scaled, maxInterval.toMillis()); + long jittered = capped - (capped / 8) * (exponent % 3); + return Duration.ofMillis(Math.max(baseInterval.toMillis(), jittered)); + } + + /** + * Reports whether a record has exhausted its attempt budget. + * + * @param attempts how many times the record has been published + * @return true when it should be parked rather than retried + */ + public boolean isExhausted(int attempts) { + return attempts >= maxAttempts; + } + + /** + * Returns the reason a record should be parked, when it should. + * + * @param attempts how many times the record has been published + * @return the sanitized reason, or empty when the record may still retry + */ + public Optional parkReason(int attempts) { + return isExhausted(attempts) + ? Optional.of( + "the record reached its %d attempt budget without a confirmed publish" + .formatted(maxAttempts)) + : Optional.empty(); + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/main/resources/db/migration/messaging/V1__messaging_outbox.sql b/src/messaging/messaging-outbox-jpa/src/main/resources/db/migration/messaging/V1__messaging_outbox.sql new file mode 100644 index 00000000..25df7b71 --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/main/resources/db/migration/messaging/V1__messaging_outbox.sql @@ -0,0 +1,41 @@ +-- Transactional outbox. +-- +-- Written by the business transaction, drained by the relay. message_id is the primary key rather +-- than a surrogate: it is the logical identity the relay must preserve across every retry, and +-- making it the key means no code path can accidentally publish the same row under a new id. +CREATE TABLE messaging_outbox +( + message_id UUID NOT NULL, + destination VARCHAR(160) NOT NULL, + message_type VARCHAR(240) NOT NULL, + schema_version INTEGER NOT NULL, + content_type VARCHAR(160) NOT NULL, + payload BYTEA NOT NULL, + headers JSONB NOT NULL DEFAULT '{}'::JSONB, + created_at TIMESTAMPTZ NOT NULL, + status VARCHAR(16) NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + lease_expires_at TIMESTAMPTZ, + published_at TIMESTAMPTZ, + last_failure_code VARCHAR(120), + CONSTRAINT pk_messaging_outbox PRIMARY KEY (message_id), + CONSTRAINT ck_messaging_outbox_status + CHECK (status IN ('PENDING', 'IN_FLIGHT', 'PUBLISHED', 'AMBIGUOUS', 'FAILED')), + CONSTRAINT ck_messaging_outbox_attempts CHECK (attempts >= 0), + CONSTRAINT ck_messaging_outbox_schema_version CHECK (schema_version >= 1) +); + +-- The relay's only hot query: claim the oldest rows that are publishable and unleased. The partial +-- index keeps it proportional to the backlog rather than to the table, which matters because +-- PUBLISHED rows accumulate until the retention job removes them. +-- +-- IN_FLIGHT is included. A relay that dies mid-publish leaves rows in that state, and their lease +-- expiry is what makes them claimable again; omitting them here would strand those messages. +CREATE INDEX ix_messaging_outbox_claimable + ON messaging_outbox (created_at) + WHERE status IN ('PENDING', 'AMBIGUOUS', 'IN_FLIGHT'); + +-- Retention sweeps read this. +CREATE INDEX ix_messaging_outbox_published_at + ON messaging_outbox (published_at) + WHERE status = 'PUBLISHED'; diff --git a/src/messaging/messaging-outbox-jpa/src/main/resources/debezium/outbox-event-router.properties b/src/messaging/messaging-outbox-jpa/src/main/resources/debezium/outbox-event-router.properties new file mode 100644 index 00000000..b28a254f --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/main/resources/debezium/outbox-event-router.properties @@ -0,0 +1,43 @@ +# Debezium Outbox Event Router configuration for the messaging_outbox table. +# +# Optional. Enable this OR the in-process polling relay, never both: Debezium reads the +# write-ahead log and never observes the polling lease, so with both running every row is +# published twice and the duplicate is invisible in the outbox table itself. +# +# Operating the Debezium connector — its offsets, its snapshots, its slot — is out of scope for +# this module. What is in scope is that a message routed by CDC is indistinguishable from one +# published by the polling relay: same logical id, same destination, same reserved headers. + +name=messaging-outbox-connector +connector.class=io.debezium.connector.postgresql.PostgresConnector + +# Only the outbox table. A wider include list turns every business table change into a message. +table.include.list=public.messaging_outbox + +# The row is deleted or marked after routing; the tombstone would otherwise reach the topic as a +# null-payload record that consumers must special-case. +tombstones.on.delete=false + +transforms=outbox +transforms.outbox.type=io.debezium.transforms.outbox.EventRouter + +# Identity comes from the row's own message_id, not from Debezium's per-change event id. A CDC +# re-snapshot mints new event ids for rows that already published, which would break identity in +# exactly the situation where it matters most. +transforms.outbox.table.field.event.id=message_id +transforms.outbox.table.field.event.key=destination +transforms.outbox.table.field.event.payload=payload +transforms.outbox.table.field.event.timestamp=created_at + +transforms.outbox.route.by.field=destination +transforms.outbox.route.topic.replacement=${routedByValue} + +# The reserved headers a consumer relies on, carried through the transform so the CDC path emits +# the same envelope the polling relay does. +transforms.outbox.table.fields.additional.placement=message_type:header:msg.type,schema_version:header:msg.schema-version,content_type:header:msg.content-type,message_id:header:msg.id + +# Publisher idempotence and full acknowledgement: a CDC relay that loses a record cannot replay it +# without re-reading the log from an earlier offset, which would republish everything after it. +producer.override.enable.idempotence=true +producer.override.acks=all +producer.override.max.in.flight.requests.per.connection=5 diff --git a/src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapperTest.java b/src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapperTest.java new file mode 100644 index 00000000..c5b5117a --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapperTest.java @@ -0,0 +1,134 @@ +package dev.caskeleton.messaging.outbox; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.outbox.DebeziumOutboxProfile.RelayMode; +import dev.caskeleton.messaging.reliability.OutboxRecord; +import dev.caskeleton.messaging.reliability.OutboxStatus; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class DebeziumOutboxRecordMapperTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + private static final MessageId MESSAGE_ID = MessageId.newId(); + + private static OutboxRecord record(Map headers) { + return new OutboxRecord( + MESSAGE_ID, + new DestinationName("orders.v1"), + new MessageType("order.created"), + new SchemaVersion(1), + ContentType.JSON, + "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), + headers, + NOW, + OutboxStatus.PENDING, + 0, + Optional.empty(), + Optional.empty()); + } + + private static DebeziumOutboxRecordMapper mapper(boolean aggregateIdAsPartitionKey) { + return new DebeziumOutboxRecordMapper( + new DebeziumOutboxProfile( + RelayMode.CHANGE_DATA_CAPTURE, "prod.", aggregateIdAsPartitionKey)); + } + + @Test + void theRoutedTopicIsThePrefixedDestination() { + assertThat(mapper(false).map(record(Map.of())).topic()).isEqualTo("prod.orders.v1"); + } + + @Test + void theLogicalMessageIdSurvivesTheCdcPath() { + DebeziumMappedRecord mapped = mapper(false).map(record(Map.of())); + + assertThat(mapped.headers().get(ReservedHeaders.MESSAGE_ID)) + .as("Debezium mints a fresh event id per change record; using it would break identity") + .isEqualTo(MESSAGE_ID.value().toString()); + } + + @Test + void everyReservedHeaderAConsumerNeedsIsCarried() { + Map headers = mapper(false).map(record(Map.of())).headers(); + + assertThat(headers) + .containsKeys( + ReservedHeaders.MESSAGE_ID, + ReservedHeaders.MESSAGE_TYPE, + ReservedHeaders.SCHEMA_VERSION, + ReservedHeaders.CONTENT_TYPE, + ReservedHeaders.PRODUCED_AT); + } + + @Test + void applicationHeadersOnTheRowArePreserved() { + DebeziumMappedRecord mapped = mapper(false).map(record(Map.of("x-tenant-tier", "gold"))); + + assertThat(mapped.headers()).containsEntry("x-tenant-tier", "gold"); + } + + @Test + void anUnkeyedProfileEmitsNoPartitionKey() { + assertThat(mapper(false).map(record(Map.of())).key()).isEmpty(); + } + + @Test + void aKeyedProfileUsesTheSamePartitionKeyThePollingRelayWouldHave() { + DebeziumMappedRecord mapped = + mapper(true).map(record(Map.of(ReservedHeaders.PARTITION_KEY, "customer-42"))); + + assertThat(mapped.key()) + .as("a different key would send the two relays' messages to different partitions") + .hasValue("customer-42"); + } + + @Test + void thePayloadIsUnchangedByTheTransform() { + assertThat(mapper(false).map(record(Map.of())).payload()) + .isEqualTo("{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8)); + } + + @Test + void runningBothRelaysAtOnceIsRefused() { + DebeziumOutboxProfile cdc = + new DebeziumOutboxProfile(RelayMode.CHANGE_DATA_CAPTURE, "p.", false); + + assertThatThrownBy(() -> cdc.requireExactlyOneRelay(true)) + .as("Debezium never sees the polling lease, so every row would publish twice") + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("twice"); + } + + @Test + void runningNoRelayAtAllIsAlsoRefused() { + assertThatThrownBy(() -> DebeziumOutboxProfile.polling().requireExactlyOneRelay(false)) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("accumulate"); + } + + @Test + void exactlyOneRelayIsAccepted() { + assertThatCode(() -> DebeziumOutboxProfile.polling().requireExactlyOneRelay(true)) + .doesNotThrowAnyException(); + } + + @Test + void aCdcProfileMustNameATopicPrefix() { + assertThatThrownBy(() -> new DebeziumOutboxProfile(RelayMode.CHANGE_DATA_CAPTURE, " ", false)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/OutboxOperationsTest.java b/src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/OutboxOperationsTest.java new file mode 100644 index 00000000..ff68af70 --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/OutboxOperationsTest.java @@ -0,0 +1,172 @@ +package dev.caskeleton.messaging.outbox; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.reliability.OutboxRecord; +import dev.caskeleton.messaging.reliability.OutboxRepository; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class OutboxOperationsTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + /** Records the cutoffs a cleanup pass asked for, and how many rows each pass removed. */ + private static final class RecordingRepository implements OutboxRepository { + + private final List cutoffs = new ArrayList<>(); + private final List deletions; + private int pass; + + private RecordingRepository(List deletions) { + this.deletions = deletions; + } + + @Override + public void append(OutboxRecord record) { + throw new UnsupportedOperationException(); + } + + @Override + public List leaseBatch(int batchSize, Duration leaseDuration, Instant now) { + throw new UnsupportedOperationException(); + } + + @Override + public void markPublished(MessageId messageId, Instant now) { + throw new UnsupportedOperationException(); + } + + @Override + public void markAmbiguous(MessageId messageId, String failureCode, Instant now) { + throw new UnsupportedOperationException(); + } + + @Override + public void markFailed(MessageId messageId, String failureCode, Instant now) { + throw new UnsupportedOperationException(); + } + + @Override + public void releaseLease(MessageId messageId) { + throw new UnsupportedOperationException(); + } + + @Override + public Optional find(MessageId messageId) { + throw new UnsupportedOperationException(); + } + + @Override + public int purgePublishedBefore(Instant publishedBefore) { + cutoffs.add(publishedBefore); + return pass < deletions.size() ? deletions.get(pass++) : 0; + } + } + + @Test + void aLeaseShorterThanThePublishTimeoutIsRefused() { + assertThatThrownBy( + () -> + new OutboxProperties( + 100, + Duration.ofSeconds(4), + Duration.ofSeconds(5), + Duration.ofMillis(500), + Duration.ofDays(1), + 5)) + .as("a second relay would claim the row while the first is still awaiting its confirm") + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("lease"); + } + + @Test + void theDefaultsSatisfyTheirOwnRule() { + assertThatCode(OutboxProperties::defaults).doesNotThrowAnyException(); + } + + @Test + void cleanupDeletesInBoundedBatchesRatherThanOneLongStatement() { + RecordingRepository repository = new RecordingRepository(List.of(1000, 1000, 250)); + + int removed = new OutboxCleanupJob(repository, OutboxProperties.defaults(), 10).runOnce(NOW); + + assertThat(removed).isEqualTo(2250); + assertThat(repository.cutoffs).hasSize(4); + } + + @Test + void cleanupStopsAsSoonAsThereIsNothingLeft() { + RecordingRepository repository = new RecordingRepository(List.of(0)); + + new OutboxCleanupJob(repository, OutboxProperties.defaults(), 10).runOnce(NOW); + + assertThat(repository.cutoffs).hasSize(1); + } + + @Test + void cleanupNeverAsksForRowsNewerThanTheRetention() { + RecordingRepository repository = new RecordingRepository(List.of(0)); + OutboxProperties properties = OutboxProperties.defaults(); + + new OutboxCleanupJob(repository, properties, 5).runOnce(NOW); + + assertThat(repository.cutoffs).containsExactly(NOW.minus(properties.retentionAfterPublish())); + } + + @Test + void aSuccessfulPassDoesNotBackOff() { + OutboxRetryScheduler scheduler = + new OutboxRetryScheduler(OutboxProperties.defaults(), Duration.ofMinutes(1)); + + assertThat(scheduler.backoff(0)).isEqualTo(OutboxProperties.defaults().pollInterval()); + } + + @Test + void consecutiveFailuresBackOffAndThenStopGrowing() { + OutboxRetryScheduler scheduler = + new OutboxRetryScheduler(OutboxProperties.defaults(), Duration.ofSeconds(10)); + + assertThat(scheduler.backoff(3)).isGreaterThan(scheduler.backoff(1)); + assertThat(scheduler.backoff(50)) + .as("an unbounded counter must not overflow into a negative duration") + .isLessThanOrEqualTo(Duration.ofSeconds(10)) + .isPositive(); + } + + @Test + void backoffNeverDropsBelowThePollInterval() { + OutboxProperties properties = OutboxProperties.defaults(); + OutboxRetryScheduler scheduler = new OutboxRetryScheduler(properties, Duration.ofSeconds(10)); + + for (int attempt = 0; attempt <= 40; attempt++) { + assertThat(scheduler.backoff(attempt)).isGreaterThanOrEqualTo(properties.pollInterval()); + } + } + + @Test + void aRecordIsParkedOnceItsAttemptBudgetIsSpent() { + OutboxProperties properties = OutboxProperties.defaults(); + OutboxRetryScheduler scheduler = new OutboxRetryScheduler(properties, Duration.ofMinutes(1)); + + assertThat(scheduler.isExhausted(properties.maxAttempts() - 1)).isFalse(); + assertThat(scheduler.isExhausted(properties.maxAttempts())).isTrue(); + assertThat(scheduler.parkReason(properties.maxAttempts())).isPresent(); + } + + @Test + void theNextPassIsScheduledFromTheBackoff() { + OutboxRetryScheduler scheduler = + new OutboxRetryScheduler(OutboxProperties.defaults(), Duration.ofSeconds(10)); + + assertThat(scheduler.nextPassAt(NOW, 0)).isEqualTo(NOW.plus(scheduler.backoff(0))); + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/OutboxPostgresIT.java b/src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/OutboxPostgresIT.java new file mode 100644 index 00000000..ac2af525 --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/OutboxPostgresIT.java @@ -0,0 +1,223 @@ +package dev.caskeleton.messaging.outbox; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.reliability.OutboxRecord; +import dev.caskeleton.messaging.reliability.OutboxStatus; +import dev.caskeleton.messaging.testkit.DockerAvailability; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import javax.sql.DataSource; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.postgresql.ds.PGSimpleDataSource; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.postgresql.PostgreSQLContainer; + +/** + * Certifies the outbox against a real PostgreSQL. + * + *

The pattern's guarantees are claims about a database, not about Java: that a row and a + * business change commit together, that two relay instances cannot claim the same row, and that an + * ambiguous publish leaves the row claimable under its original identity. An in-memory double can + * be written to agree with any of those; only a real database settles them. + */ +@Testcontainers +@EnabledIf("dockerAvailable") +class OutboxPostgresIT { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer("postgres:16-alpine") + .withDatabaseName("messaging") + .withUsername("messaging") + .withPassword("messaging"); + + private DataSource dataSource; + private JdbcOutboxRepository repository; + + static boolean dockerAvailable() { + return DockerAvailability.isAvailable(); + } + + @BeforeEach + void migrate() throws SQLException, IOException { + dataSource = dataSource(); + try (Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS messaging_outbox"); + statement.execute(migration()); + } + repository = new JdbcOutboxRepository(dataSource); + } + + @Test + void anAppendedRowIsClaimableAndRoundTripsIntact() { + OutboxRecord record = record(); + repository.append(record); + + List leased = repository.leaseBatch(10, Duration.ofSeconds(30), NOW); + + assertThat(leased) + .singleElement() + .satisfies( + leasedRecord -> { + assertThat(leasedRecord.messageId()).isEqualTo(record.messageId()); + assertThat(leasedRecord.messageType()).isEqualTo(record.messageType()); + assertThat(leasedRecord.payload()).isEqualTo(record.payload()); + assertThat(leasedRecord.headers()).containsEntry("msg.partition-key", "acct-1"); + }); + } + + @Test + void aLeasedRowIsInvisibleToASecondRelayInstance() { + repository.append(record()); + + List first = repository.leaseBatch(10, Duration.ofSeconds(30), NOW); + List second = repository.leaseBatch(10, Duration.ofSeconds(30), NOW); + + assertThat(first).hasSize(1); + assertThat(second).as("SKIP LOCKED plus the lease keeps two relays off the same row").isEmpty(); + } + + @Test + void anExpiredLeaseBecomesClaimableAgain() { + repository.append(record()); + repository.leaseBatch(10, Duration.ofSeconds(30), NOW); + + List reclaimed = + repository.leaseBatch(10, Duration.ofSeconds(30), NOW.plusSeconds(31)); + + assertThat(reclaimed).hasSize(1); + } + + @Test + void anAmbiguousRowStaysClaimableUnderItsOriginalIdentity() { + OutboxRecord record = record(); + repository.append(record); + repository.leaseBatch(10, Duration.ofSeconds(30), NOW); + + repository.markAmbiguous(record.messageId(), "CONFIRM_TIMEOUT", NOW); + + assertThat(repository.find(record.messageId()).orElseThrow().status()) + .isEqualTo(OutboxStatus.AMBIGUOUS); + assertThat(repository.leaseBatch(10, Duration.ofSeconds(30), NOW)) + .singleElement() + .satisfies(retry -> assertThat(retry.messageId()).isEqualTo(record.messageId())); + } + + @Test + void aPublishedRowIsNotClaimedAgain() { + OutboxRecord record = record(); + repository.append(record); + repository.leaseBatch(10, Duration.ofSeconds(30), NOW); + + repository.markPublished(record.messageId(), NOW); + + assertThat(repository.find(record.messageId()).orElseThrow().status()) + .isEqualTo(OutboxStatus.PUBLISHED); + assertThat(repository.leaseBatch(10, Duration.ofSeconds(30), NOW.plusSeconds(60))).isEmpty(); + } + + @Test + void aFailedRowIsNotClaimedAgainWithoutIntervention() { + OutboxRecord record = record(); + repository.append(record); + repository.leaseBatch(10, Duration.ofSeconds(30), NOW); + + repository.markFailed(record.messageId(), "INVALID_TOPIC", NOW); + + assertThat(repository.leaseBatch(10, Duration.ofSeconds(30), NOW.plusSeconds(60))).isEmpty(); + } + + @Test + void theRowAndTheBusinessChangeCommitTogetherOrNotAtAll() throws SQLException { + OutboxRecord record = record(); + + try (Connection connection = dataSource.getConnection()) { + connection.setAutoCommit(false); + try (Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE IF NOT EXISTS orders (id TEXT PRIMARY KEY)"); + } + connection.commit(); + + try (Statement statement = connection.createStatement()) { + statement.execute("INSERT INTO orders (id) VALUES ('o-1')"); + } + repository.append(connection, record); + connection.rollback(); + } + + assertThat(repository.find(record.messageId())) + .as("a rolled back business change must not leave a publishable row") + .isEmpty(); + } + + @Test + void retentionRemovesOnlyPublishedRows() { + OutboxRecord published = record(); + OutboxRecord pending = record(); + repository.append(published); + repository.append(pending); + repository.markPublished(published.messageId(), NOW); + + int removed = repository.purgePublishedBefore(NOW.plusSeconds(1)); + + assertThat(removed).isEqualTo(1); + assertThat(repository.find(pending.messageId())).isPresent(); + } + + private static OutboxRecord record() { + return new OutboxRecord( + MessageId.newId(), + new DestinationName("order-events"), + new MessageType("order.created"), + new SchemaVersion(1), + ContentType.JSON, + "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), + Map.of("msg.partition-key", "acct-1"), + NOW.minusSeconds(60), + OutboxStatus.PENDING, + 0, + Optional.empty(), + Optional.empty()); + } + + private static DataSource dataSource() { + PGSimpleDataSource source = new PGSimpleDataSource(); + source.setUrl(POSTGRES.getJdbcUrl()); + source.setUser(POSTGRES.getUsername()); + source.setPassword(POSTGRES.getPassword()); + return source; + } + + private static String migration() throws IOException { + try (InputStream stream = + OutboxPostgresIT.class + .getClassLoader() + .getResourceAsStream("db/migration/messaging/V1__messaging_outbox.sql")) { + if (stream == null) { + throw new IOException("the outbox migration is missing from the classpath"); + } + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + } +} diff --git a/src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/OutboxRelayTest.java b/src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/OutboxRelayTest.java new file mode 100644 index 00000000..77e3008d --- /dev/null +++ b/src/messaging/messaging-outbox-jpa/src/test/java/dev/caskeleton/messaging/outbox/OutboxRelayTest.java @@ -0,0 +1,312 @@ +package dev.caskeleton.messaging.outbox; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.reliability.OutboxRecord; +import dev.caskeleton.messaging.reliability.OutboxRepository; +import dev.caskeleton.messaging.reliability.OutboxStatus; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import org.junit.jupiter.api.Test; + +class OutboxRelayTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + private final InMemoryOutboxRepository repository = new InMemoryOutboxRepository(); + + @Test + void aConfirmedPublishMarksTheRowPublished() { + OutboxRecord record = OutboxFixtures.pending(); + repository.append(record); + + OutboxRelayReport report = relay(RecordingOutboxPublisher.confirming()).runOnce(NOW); + + assertThat(report.published()).isEqualTo(1); + assertThat(repository.find(record.messageId()).orElseThrow().status()) + .isEqualTo(OutboxStatus.PUBLISHED); + } + + @Test + void anAmbiguousPublishStaysRetryableRatherThanFailing() { + OutboxRecord record = OutboxFixtures.pending(); + repository.append(record); + + OutboxRelayReport report = relay(RecordingOutboxPublisher.ambiguous()).runOnce(NOW); + + assertThat(report.ambiguous()).isEqualTo(1); + assertThat(repository.find(record.messageId()).orElseThrow().status()) + .isEqualTo(OutboxStatus.AMBIGUOUS); + } + + @Test + void anAmbiguousRowIsRepublishedUnderTheSameMessageId() { + OutboxRecord record = OutboxFixtures.pending(); + repository.append(record); + RecordingOutboxPublisher publisher = RecordingOutboxPublisher.ambiguous(); + + relay(publisher).runOnce(NOW); + relay(publisher).runOnce(NOW.plusSeconds(60)); + + assertThat(publisher.publishedIds).hasSize(2); + assertThat(publisher.publishedIds.get(0)).isEqualTo(record.messageId()); + assertThat(publisher.publishedIds.get(1)).isEqualTo(record.messageId()); + } + + @Test + void aRejectedPublishIsMarkedFailedAndNotRetried() { + OutboxRecord record = OutboxFixtures.pending(); + repository.append(record); + + relay(RecordingOutboxPublisher.rejecting()).runOnce(NOW); + + assertThat(repository.find(record.messageId()).orElseThrow().status()) + .isEqualTo(OutboxStatus.FAILED); + assertThat(repository.leaseBatch(10, Duration.ofSeconds(30), NOW)).isEmpty(); + } + + @Test + void aLeasedRowIsInvisibleToAnotherRelayInstance() { + repository.append(OutboxFixtures.pending()); + + List first = repository.leaseBatch(10, Duration.ofSeconds(30), NOW); + List second = repository.leaseBatch(10, Duration.ofSeconds(30), NOW); + + assertThat(first).hasSize(1); + assertThat(second).isEmpty(); + } + + @Test + void anExpiredLeaseBecomesClaimableAgain() { + repository.append(OutboxFixtures.pending()); + repository.leaseBatch(10, Duration.ofSeconds(30), NOW); + + List reclaimed = + repository.leaseBatch(10, Duration.ofSeconds(30), NOW.plusSeconds(31)); + + assertThat(reclaimed).hasSize(1); + } + + @Test + void theRelayPreservesPayloadAndTypeOnTheWire() { + OutboxRecord record = OutboxFixtures.pending(); + repository.append(record); + RecordingOutboxPublisher publisher = RecordingOutboxPublisher.confirming(); + + relay(publisher).runOnce(NOW); + + MessageEnvelope envelope = publisher.publishedEnvelopes.get(0); + assertThat(envelope.messageType()).isEqualTo(new MessageType("order.created")); + assertThat(envelope.schemaVersion()).isEqualTo(new SchemaVersion(1)); + } + + private OutboxRelay relay(MessagePublisher publisher) { + return new OutboxRelay( + repository, + publisher, + new OutboxEnvelopeFactory(new ProducerId("order-api")), + 100, + Duration.ofSeconds(30)); + } +} + +/** Builds outbox rows. */ +final class OutboxFixtures { + + private OutboxFixtures() {} + + static OutboxRecord pending() { + return new OutboxRecord( + MessageId.newId(), + new DestinationName("order-events"), + new MessageType("order.created"), + new SchemaVersion(1), + ContentType.JSON, + "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), + Map.of(), + Instant.parse("2026-08-10T09:14:00Z"), + OutboxStatus.PENDING, + 0, + Optional.empty(), + Optional.empty()); + } +} + +/** A deterministic outbox store with the leasing semantics the relay relies on. */ +final class InMemoryOutboxRepository implements OutboxRepository { + + private final Map rows = new LinkedHashMap<>(); + private final Map leases = new LinkedHashMap<>(); + + @Override + public void append(OutboxRecord record) { + rows.put(record.messageId(), record); + } + + /** + * Mirrors the SQL exactly, including {@code IN_FLIGHT}. + * + *

An earlier version of this double left the status untouched on lease and only considered + * {@code PENDING} and {@code AMBIGUOUS} claimable. That happened to pass while the real query + * stranded every row whose relay had died mid-publish. A double that is more forgiving than the + * thing it stands in for hides exactly the defects it was written to catch, so this one follows + * the same three-state rule. + */ + @Override + public List leaseBatch(int batchSize, Duration leaseDuration, Instant now) { + List leased = new ArrayList<>(); + for (OutboxRecord record : List.copyOf(rows.values())) { + if (leased.size() >= batchSize) { + break; + } + boolean claimable = + record.status() == OutboxStatus.PENDING + || record.status() == OutboxStatus.AMBIGUOUS + || record.status() == OutboxStatus.IN_FLIGHT; + Instant leaseExpiry = leases.get(record.messageId()); + boolean unleased = leaseExpiry == null || !now.isBefore(leaseExpiry); + if (claimable && unleased) { + leases.put(record.messageId(), now.plus(leaseDuration)); + rows.put( + record.messageId(), + record.withStatus(OutboxStatus.IN_FLIGHT, record.attempts(), record.lastFailureCode())); + leased.add(record); + } + } + return List.copyOf(leased); + } + + @Override + public void markPublished(MessageId messageId, Instant now) { + transition(messageId, OutboxStatus.PUBLISHED, Optional.empty()); + } + + @Override + public void markAmbiguous(MessageId messageId, String failureCode, Instant now) { + transition(messageId, OutboxStatus.AMBIGUOUS, Optional.of(failureCode)); + leases.remove(messageId); + } + + @Override + public void markFailed(MessageId messageId, String failureCode, Instant now) { + transition(messageId, OutboxStatus.FAILED, Optional.of(failureCode)); + } + + @Override + public void releaseLease(MessageId messageId) { + leases.remove(messageId); + } + + @Override + public Optional find(MessageId messageId) { + return Optional.ofNullable(rows.get(messageId)); + } + + @Override + public int purgePublishedBefore(Instant publishedBefore) { + int before = rows.size(); + rows.values().removeIf(record -> record.status() == OutboxStatus.PUBLISHED); + return before - rows.size(); + } + + private void transition(MessageId messageId, OutboxStatus status, Optional failureCode) { + OutboxRecord record = rows.get(messageId); + if (record != null) { + rows.put(messageId, record.withStatus(status, record.attempts() + 1, failureCode)); + } + } +} + +/** A publisher that records what the relay handed it. */ +final class RecordingOutboxPublisher implements MessagePublisher { + + final List publishedIds = new ArrayList<>(); + final List> publishedEnvelopes = new ArrayList<>(); + + private final PublishCompletion completion; + + private RecordingOutboxPublisher(PublishCompletion completion) { + this.completion = completion; + } + + static RecordingOutboxPublisher confirming() { + return new RecordingOutboxPublisher(PublishCompletion.CONFIRMED); + } + + static RecordingOutboxPublisher ambiguous() { + return new RecordingOutboxPublisher(PublishCompletion.AMBIGUOUS); + } + + static RecordingOutboxPublisher rejecting() { + return new RecordingOutboxPublisher(PublishCompletion.REJECTED); + } + + @Override + public CompletionStage publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options) { + publishedIds.add(message.messageId()); + publishedEnvelopes.add(message); + return CompletableFuture.completedFuture(result()); + } + + private PublishResult result() { + return switch (completion) { + case CONFIRMED -> + new PublishResult( + PublishCompletion.CONFIRMED, + PublishEvidence.confirmed(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ofMillis(2), + Optional.empty()); + case AMBIGUOUS -> + new PublishResult( + PublishCompletion.AMBIGUOUS, + PublishEvidence.ambiguous(), + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + Duration.ofSeconds(5), + Optional.of( + FailureDescriptor.of( + FailureCategory.AMBIGUOUS, "CONFIRM_TIMEOUT", "confirm timed out"))); + case REJECTED -> + new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ofMillis(1), + Optional.of( + FailureDescriptor.of( + FailureCategory.PERMANENT_BUSINESS, "INVALID_TOPIC", "unknown topic"))); + }; + } +} diff --git a/src/messaging/messaging-policy/build.gradle b/src/messaging/messaging-policy/build.gradle new file mode 100644 index 00000000..208c548d --- /dev/null +++ b/src/messaging/messaging-policy/build.gradle @@ -0,0 +1,6 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-schema-api') +} diff --git a/src/messaging/messaging-policy/gradle.lockfile b/src/messaging/messaging-policy/gradle.lockfile new file mode 100644 index 00000000..599ff921 --- /dev/null +++ b/src/messaging/messaging-policy/gradle.lockfile @@ -0,0 +1,83 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty=compileClasspath,runtimeClasspath diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/BackoffCalculator.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/BackoffCalculator.java new file mode 100644 index 00000000..ef320679 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/BackoffCalculator.java @@ -0,0 +1,58 @@ +package dev.caskeleton.messaging.policy; + +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.DoubleSupplier; + +/** + * Exponential backoff with optional full jitter. + * + *

The delay is {@code min(maxDelay, initialDelay * multiplier^(attempt-1))}. Full jitter then + * picks uniformly from {@code [0, delay]} rather than shaving a small percentage off. That matters + * when a downstream recovers: without jitter every consumer that failed in the same second retries + * in the same second, and the recovery is immediately undone by the retry storm. + */ +public final class BackoffCalculator { + + private final DoubleSupplier randomFraction; + + /** Creates a calculator using the platform's thread-local random source. */ + public BackoffCalculator() { + this(() -> ThreadLocalRandom.current().nextDouble()); + } + + /** + * Creates a calculator with an explicit random source. + * + * @param randomFraction supplies values in {@code [0, 1)} + */ + public BackoffCalculator(DoubleSupplier randomFraction) { + this.randomFraction = Objects.requireNonNull(randomFraction, "randomFraction must not be null"); + } + + /** + * Computes the delay before a given attempt. + * + * @param policy the retry policy + * @param attempt the attempt number, counting the first delivery as one + * @return the delay to wait before the next attempt + */ + public Duration delayFor(RetryPolicy policy, int attempt) { + Objects.requireNonNull(policy, "policy must not be null"); + if (attempt < 1) { + throw new IllegalArgumentException("attempt counts the first delivery as 1"); + } + + double scaled = + policy.initialDelay().toMillis() * Math.pow(policy.multiplier(), (double) attempt - 1); + long capped = (long) Math.min(scaled, (double) policy.maxDelay().toMillis()); + if (capped <= 0) { + return Duration.ZERO; + } + if (!policy.jitter()) { + return Duration.ofMillis(capped); + } + return Duration.ofMillis((long) (capped * randomFraction.getAsDouble())); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/CapabilityTier.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/CapabilityTier.java new file mode 100644 index 00000000..3760b8a6 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/CapabilityTier.java @@ -0,0 +1,21 @@ +package dev.caskeleton.messaging.policy; + +/** + * The API tier a destination is approved for. + * + *

Recorded on the profile so that the tier boundary is enforced by configuration review rather + * than by whichever handler happens to be wired in. An M1 destination that silently accepted a + * manual settlement handler would move the acknowledge-after-success ordering out of the platform + * and into application code. + */ +public enum CapabilityTier { + + /** Typed publish and handler with platform-owned settlement. */ + M1, + + /** Batch, manual settlement, pause and resume, delayed delivery, replay request. */ + M2, + + /** Broker-native capabilities exposed as typed interfaces. */ + M3 +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/ConsumerPolicy.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/ConsumerPolicy.java new file mode 100644 index 00000000..e02c74e5 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/ConsumerPolicy.java @@ -0,0 +1,55 @@ +package dev.caskeleton.messaging.policy; + +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; + +/** + * The consumer-side contract for a destination. + * + *

{@code manualSettlement} is a policy flag rather than a handler choice: it decides which + * capability tier a destination belongs to, and an M1 destination that quietly accepted a manual + * handler would bypass the platform's settlement ordering. + * + * @param group the consumer group, where the broker has one + * @param concurrency how many deliveries may be processed at once + * @param maxInFlightPerOrderingUnit the in-flight ceiling inside one ordering unit + * @param prefetch the broker prefetch window + * @param handlerTimeout the handler deadline + * @param manualSettlement whether this destination uses the M2 manual settlement tier + */ +public record ConsumerPolicy( + Optional group, + int concurrency, + int maxInFlightPerOrderingUnit, + int prefetch, + Duration handlerTimeout, + boolean manualSettlement) { + + public ConsumerPolicy { + Objects.requireNonNull(group, "group must not be null"); + Objects.requireNonNull(handlerTimeout, "handlerTimeout must not be null"); + if (concurrency < 1) { + throw new IllegalArgumentException("consumer concurrency must be at least 1"); + } + if (maxInFlightPerOrderingUnit < 1) { + throw new IllegalArgumentException("maxInFlightPerOrderingUnit must be at least 1"); + } + if (prefetch < 1) { + throw new IllegalArgumentException("prefetch must be at least 1"); + } + if (handlerTimeout.isNegative() || handlerTimeout.isZero()) { + throw new IllegalArgumentException("handlerTimeout must be positive"); + } + } + + /** + * Returns the M1 default: single ordering unit in flight, thirty second handler timeout. + * + * @param group the consumer group + * @return the default consumer policy + */ + public static ConsumerPolicy defaults(String group) { + return new ConsumerPolicy(Optional.ofNullable(group), 1, 1, 16, Duration.ofSeconds(30), false); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterEnvelopeFactory.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterEnvelopeFactory.java new file mode 100644 index 00000000..90408987 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterEnvelopeFactory.java @@ -0,0 +1,57 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.header.HeaderName; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Builds the envelope published to a dead letter destination. + * + *

The payload and the logical {@code messageId} are carried through untouched. That is what + * makes a redrive a genuine replay rather than a new message: an Inbox downstream still recognises + * it, and an operator can correlate the dead letter with the original publish. + * + *

Failure context is written into reserved headers, never into the payload, so redriving does + * not require unwrapping a platform-specific structure. + */ +public final class DeadLetterEnvelopeFactory { + + /** + * Wraps a failed delivery for the dead letter destination. + * + * @param source the envelope as delivered + * @param metadata why it is being dead lettered + * @return an envelope with identical identity and payload plus failure headers + */ + public MessageEnvelope create( + MessageEnvelope source, DeadLetterMetadata metadata) { + Objects.requireNonNull(source, "source must not be null"); + Objects.requireNonNull(metadata, "metadata must not be null"); + + Map headers = new LinkedHashMap<>(source.headers().asMap()); + headers.put( + new HeaderName(ReservedHeaders.FAILURE_CATEGORY), + new HeaderValue(metadata.category().name())); + headers.put(new HeaderName(ReservedHeaders.FAILURE_CODE), new HeaderValue(metadata.code())); + headers.put( + new HeaderName(ReservedHeaders.ORIGIN_DESTINATION), + new HeaderValue(metadata.originDestination().value())); + headers.put( + new HeaderName(ReservedHeaders.RETRY_ATTEMPT), + new HeaderValue(Integer.toString(metadata.deliveryAttempt()))); + headers.put( + new HeaderName(ReservedHeaders.FIRST_FAILURE_AT), + new HeaderValue(metadata.firstFailureAt().toString())); + headers.put( + new HeaderName(ReservedHeaders.LAST_FAILURE_AT), + new HeaderValue(metadata.lastFailureAt().toString())); + + return source.withHeaders(MessageHeaders.platform(headers)); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterMetadata.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterMetadata.java new file mode 100644 index 00000000..792f4f1d --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterMetadata.java @@ -0,0 +1,42 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.FailureCategory; +import java.time.Instant; +import java.util.Objects; + +/** + * What a dead lettered message carries about why it is there. + * + *

Deliberately small. A dead letter destination is read by operators, exported to tickets, and + * often retained far longer than the source topic, so it holds a category, a code, and timing — not + * a stack trace, not the exception message, and not the original headers. + * + * @param category the stable failure classification + * @param code the stable failure code + * @param originDestination the destination the message was consumed from + * @param deliveryAttempt the attempt that gave up + * @param firstFailureAt when the message first failed + * @param lastFailureAt when the message last failed + */ +public record DeadLetterMetadata( + FailureCategory category, + String code, + DestinationName originDestination, + int deliveryAttempt, + Instant firstFailureAt, + Instant lastFailureAt) { + + public DeadLetterMetadata { + Objects.requireNonNull(category, "category must not be null"); + Objects.requireNonNull(originDestination, "originDestination must not be null"); + Objects.requireNonNull(firstFailureAt, "firstFailureAt must not be null"); + Objects.requireNonNull(lastFailureAt, "lastFailureAt must not be null"); + if (code == null || code.isBlank()) { + throw new IllegalArgumentException("failure code must not be blank"); + } + if (deliveryAttempt < 1) { + throw new IllegalArgumentException("deliveryAttempt counts the first delivery as 1"); + } + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterOrchestrator.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterOrchestrator.java new file mode 100644 index 00000000..697728c7 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterOrchestrator.java @@ -0,0 +1,121 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.delivery.MessageDelivery; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.time.Instant; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * Publishes to the dead letter destination, then settles the source — never the other way round. + * + *

This ordering is the single invariant that stops dead lettering from becoming data loss. If + * the source were acknowledged first, a failed dead letter publish would leave no copy of the + * message anywhere: the broker has released it and the dead letter destination never received it. + * So the source stays unsettled on anything other than a confirmed publish, including an ambiguous + * one, and the message is redelivered instead of disappearing. + * + *

An ambiguous dead letter publish therefore produces a duplicate rather than a loss. That is + * the intended trade: the dead letter destination is read by humans who can spot a duplicate, and + * it is the only side of the trade that is recoverable. + */ +public final class DeadLetterOrchestrator { + + private final MessagePublisher publisher; + private final DeadLetterEnvelopeFactory envelopeFactory; + + /** + * Creates an orchestrator over a publisher. + * + * @param publisher the publisher used for the dead letter destination + */ + public DeadLetterOrchestrator(MessagePublisher publisher) { + this(publisher, new DeadLetterEnvelopeFactory()); + } + + /** + * Creates an orchestrator with an explicit envelope factory. + * + * @param publisher the publisher used for the dead letter destination + * @param envelopeFactory the dead letter envelope factory + */ + public DeadLetterOrchestrator( + MessagePublisher publisher, DeadLetterEnvelopeFactory envelopeFactory) { + this.publisher = Objects.requireNonNull(publisher, "publisher must not be null"); + this.envelopeFactory = + Objects.requireNonNull(envelopeFactory, "envelopeFactory must not be null"); + } + + /** + * Dead letters one delivery. + * + * @param sourceProfile the profile of the destination the message came from + * @param delivery the failed delivery, still encoded + * @param failure the sanitized failure + * @param settlement the source settlement callback + * @return a stage completing with the dead letter outcome + */ + public CompletionStage deadLetter( + DestinationProfile sourceProfile, + MessageDelivery delivery, + FailureDescriptor failure, + SourceSettlement settlement) { + + Objects.requireNonNull(sourceProfile, "sourceProfile must not be null"); + Objects.requireNonNull(delivery, "delivery must not be null"); + Objects.requireNonNull(failure, "failure must not be null"); + Objects.requireNonNull(settlement, "settlement must not be null"); + + DestinationName deadLetterName = + sourceProfile + .deadLetter() + .destination() + .orElseThrow( + () -> + new MessagingConfigurationException( + "DEAD_LETTER_NOT_CONFIGURED", + "destination has no dead letter destination: " + + sourceProfile.name().value())); + + Instant failedAt = delivery.metadata().receivedAt(); + DeadLetterMetadata metadata = + new DeadLetterMetadata( + failure.category(), + failure.code(), + sourceProfile.name(), + delivery.metadata().deliveryAttempt(), + failedAt, + failedAt); + + MessageDestination destination = + new MessageDestination<>( + deadLetterName, delivery.message().messageType(), EncodedMessage.class); + + return publisher + .publish( + destination, + envelopeFactory.create(delivery.message(), metadata), + PublishOptions.defaults()) + .thenCompose( + result -> { + if (result.completion() != PublishCompletion.CONFIRMED) { + return CompletableFuture.completedFuture(new DeadLetterResult(result, false)); + } + return settleAfterConfirmation(result, settlement); + }); + } + + private static CompletionStage settleAfterConfirmation( + PublishResult result, SourceSettlement settlement) { + return settlement.settle().thenApply(ignored -> new DeadLetterResult(result, true)); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterPolicy.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterPolicy.java new file mode 100644 index 00000000..65605e7e --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterPolicy.java @@ -0,0 +1,45 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.util.Objects; +import java.util.Optional; + +/** + * Where permanently failed messages go. + * + * @param enabled whether dead lettering is configured + * @param destination the dead letter destination + * @param maxRedriveCount how many times a message may be redriven back + */ +public record DeadLetterPolicy( + boolean enabled, Optional destination, int maxRedriveCount) { + + public DeadLetterPolicy { + Objects.requireNonNull(destination, "destination must not be null"); + if (enabled && destination.isEmpty()) { + throw new IllegalArgumentException("an enabled dead letter policy needs a destination"); + } + if (maxRedriveCount < 0) { + throw new IllegalArgumentException("maxRedriveCount must not be negative"); + } + } + + /** + * Returns a disabled dead letter policy. + * + * @return the disabled policy + */ + public static DeadLetterPolicy disabled() { + return new DeadLetterPolicy(false, Optional.empty(), 0); + } + + /** + * Returns a policy routing to a dead letter destination. + * + * @param destination the dead letter destination + * @return the enabled policy + */ + public static DeadLetterPolicy to(DestinationName destination) { + return new DeadLetterPolicy(true, Optional.of(destination), 1); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterResult.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterResult.java new file mode 100644 index 00000000..8d6a4f6a --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DeadLetterResult.java @@ -0,0 +1,17 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.publish.PublishResult; +import java.util.Objects; + +/** + * The outcome of one dead letter attempt. + * + * @param publishResult what the dead letter publish returned + * @param sourceSettled whether the source message was settled afterwards + */ +public record DeadLetterResult(PublishResult publishResult, boolean sourceSettled) { + + public DeadLetterResult { + Objects.requireNonNull(publishResult, "publishResult must not be null"); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DefaultRetryDecisionEngine.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DefaultRetryDecisionEngine.java new file mode 100644 index 00000000..943d498b --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DefaultRetryDecisionEngine.java @@ -0,0 +1,103 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.error.FailureCategory; +import java.time.Duration; +import java.util.Objects; + +/** + * The deterministic retry decision order. + * + *

The order is fixed and evaluated top to bottom. Retryability is checked before the attempt + * budget so that a deserialization failure is parked on its first delivery instead of being + * replayed three more times against a payload that cannot change. The ordering-preserving strategy + * is checked before the re-publishing one so that an ordered destination can never fall through to + * a strategy that reorders it, even if both are technically configured. + */ +public final class DefaultRetryDecisionEngine implements RetryDecisionEngine { + + private final BackoffCalculator backoff; + + /** + * Creates an engine over a backoff calculator. + * + * @param backoff the delay calculator + */ + public DefaultRetryDecisionEngine(BackoffCalculator backoff) { + this.backoff = Objects.requireNonNull(backoff, "backoff must not be null"); + } + + @Override + public RetryDecision decide(RetryContext context) { + Objects.requireNonNull(context, "context must not be null"); + + DestinationProfile profile = context.destination(); + RetryPolicy policy = profile.retry(); + int attempt = context.delivery().deliveryAttempt(); + + if (!isRetryable(policy, context.failure().category(), context.failure().retryable())) { + return park(context); + } + if (attempt >= policy.maxAttempts()) { + return new RetryDecision.DeadLetter(context.failure()); + } + + Duration delay = backoff.delayFor(policy, attempt); + + if (policy.orderingImpact() == OrderingImpact.PRESERVE + && profile.isOrdered() + && context.capabilities().orderedStream()) { + return new RetryDecision.PauseAndRetry(delay); + } + if (policy.mode() == RetryMode.PAUSE_PARTITION) { + return new RetryDecision.PauseAndRetry(delay); + } + if (policy.mode() == RetryMode.RETRY_DESTINATION + && policy.orderingImpact() == OrderingImpact.ALLOW_REORDER + && policy.retryDestination().isPresent()) { + return new RetryDecision.PublishToRetryDestination( + policy.retryDestination().orElseThrow(), delay); + } + if (policy.mode() == RetryMode.INLINE || policy.mode() == RetryMode.BLOCKING) { + return new RetryDecision.RetryInline(delay); + } + if (policy.mode() == RetryMode.BROKER_DELAYED && context.capabilities().delayedDelivery()) { + return new RetryDecision.PublishToRetryDestination( + policy.retryDestination().orElse(profile.name()), delay); + } + return new RetryDecision.DeadLetter(context.failure()); + } + + /** + * Applies the category rules, then the destination's explicit overrides. + * + *

A destination may add a category to the retryable set, but the four categories that fail + * identically on every redelivery stay non-retryable unless the profile names them explicitly. + */ + private static boolean isRetryable( + RetryPolicy policy, FailureCategory category, boolean descriptorRetryable) { + if (policy.nonRetryableCategories().contains(category)) { + return false; + } + if (policy.retryableCategories().contains(category)) { + return true; + } + return descriptorRetryable && FailureDescriptorDefaults.retryable(category); + } + + /** + * Parks a message the platform will not retry. + * + *

Dead lettering is preferred wherever a dead letter destination exists: discarding is only + * acceptable where the profile has already declared that loss is tolerable. + */ + private static RetryDecision park(RetryContext context) { + if (context.destination().deadLetter().enabled()) { + return new RetryDecision.DeadLetter(context.failure()); + } + if (context.destination().deliveryGuarantee() == DeliveryGuarantee.AT_MOST_ONCE) { + return new RetryDecision.Reject(context.failure()); + } + return new RetryDecision.DeadLetter(context.failure()); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DestinationProfile.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DestinationProfile.java new file mode 100644 index 00000000..69b3938b --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DestinationProfile.java @@ -0,0 +1,83 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.delivery.ExternalSideEffectGuarantee; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.DestinationKind; +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.util.Objects; + +/** + * Everything the platform needs to know about one logical destination. + * + *

This is the single place where a logical name becomes a broker address, a guarantee, and a + * failure policy. Validation of the whole profile happens at startup rather than at first publish, + * so a contradiction like "preserve order, but retry through a separate topic" is a failed boot + * instead of a silent reordering in production. + * + * @param name the logical destination name + * @param broker the adapter this destination binds to + * @param kind the interaction pattern + * @param physical the broker-specific address + * @param schema the codec and compatibility contract + * @param deliveryGuarantee the broker-level delivery guarantee + * @param orderingScope the scope inside which order holds + * @param externalSideEffectGuarantee how handler side effects are protected + * @param producer the producer contract + * @param consumer the consumer contract + * @param retry the retry contract + * @param deadLetter the dead letter contract + * @param payload the size contract + * @param tier the API tier this destination is approved for + * @param production whether this profile is a production profile + * @param keyResolverConfigured whether a key resolver is registered for keyed ordering + * @param topologyAutoCreate whether the application may create topology itself + */ +public record DestinationProfile( + DestinationName name, + String broker, + DestinationKind kind, + PhysicalDestination physical, + SchemaPolicy schema, + DeliveryGuarantee deliveryGuarantee, + OrderingScope orderingScope, + ExternalSideEffectGuarantee externalSideEffectGuarantee, + ProducerPolicy producer, + ConsumerPolicy consumer, + RetryPolicy retry, + DeadLetterPolicy deadLetter, + PayloadPolicy payload, + CapabilityTier tier, + boolean production, + boolean keyResolverConfigured, + boolean topologyAutoCreate) { + + public DestinationProfile { + Objects.requireNonNull(tier, "tier must not be null"); + Objects.requireNonNull(name, "name must not be null"); + Objects.requireNonNull(kind, "kind must not be null"); + Objects.requireNonNull(physical, "physical must not be null"); + Objects.requireNonNull(schema, "schema must not be null"); + Objects.requireNonNull(deliveryGuarantee, "deliveryGuarantee must not be null"); + Objects.requireNonNull(orderingScope, "orderingScope must not be null"); + Objects.requireNonNull( + externalSideEffectGuarantee, "externalSideEffectGuarantee must not be null"); + Objects.requireNonNull(producer, "producer must not be null"); + Objects.requireNonNull(consumer, "consumer must not be null"); + Objects.requireNonNull(retry, "retry must not be null"); + Objects.requireNonNull(deadLetter, "deadLetter must not be null"); + Objects.requireNonNull(payload, "payload must not be null"); + if (broker == null || broker.isBlank()) { + throw new IllegalArgumentException("broker must not be blank"); + } + } + + /** + * Reports whether order must be preserved for this destination. + * + * @return true when the ordering scope is anything but {@code NONE} + */ + public boolean isOrdered() { + return orderingScope != OrderingScope.NONE; + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DestinationProfileValidator.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DestinationProfileValidator.java new file mode 100644 index 00000000..23f2f6d7 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/DestinationProfileValidator.java @@ -0,0 +1,181 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.ConfirmationRequirement; +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * Startup validation for destination profiles. + * + *

Every rule here exists because the alternative is a production surprise. A profile that asks + * for ordered delivery and configures a reordering retry does not fail on the happy path; it fails + * the first time a message is retried, months later, in a way that looks like a data bug rather + * than a configuration one. Making the contradiction a boot failure moves that discovery to the + * deploy that introduced it. + */ +public final class DestinationProfileValidator { + + /** + * Validates a single profile. + * + * @param profile the profile to validate + * @throws IllegalArgumentException when the profile is internally contradictory + */ + public void validate(DestinationProfile profile) { + Objects.requireNonNull(profile, "profile must not be null"); + + if (profile.retry().orderingImpact() == OrderingImpact.PRESERVE && profile.retry().reorders()) { + throw new IllegalArgumentException( + "retry destination cannot preserve ordering for " + profile.name().value()); + } + if (profile.isOrdered() && profile.retry().orderingImpact() == OrderingImpact.ALLOW_REORDER) { + throw new IllegalArgumentException( + "an ordered destination cannot allow a reordering retry: " + profile.name().value()); + } + if (profile.payload().maxBytes() > PayloadPolicy.HARD_MAX_BYTES) { + throw new IllegalArgumentException( + "payload maximum exceeds " + PayloadPolicy.HARD_MAX_BYTES + " bytes"); + } + if (profile.payload().claimCheckThresholdBytes() > profile.payload().maxBytes()) { + throw new IllegalArgumentException( + "claim check threshold cannot exceed the payload maximum for " + profile.name().value()); + } + if (profile.deadLetter().enabled() + && profile.deadLetter().destination().orElseThrow().equals(profile.name())) { + throw new IllegalArgumentException("dead letter destination cannot reference itself"); + } + if (profile.retry().retryDestination().filter(profile.name()::equals).isPresent()) { + throw new IllegalArgumentException( + "retry destination cannot reference itself: " + profile.name().value()); + } + if (profile.orderingScope() == OrderingScope.KEY && !profile.keyResolverConfigured()) { + throw new IllegalArgumentException( + "ordering scope KEY requires a key resolver for " + profile.name().value()); + } + if (profile.tier() == CapabilityTier.M1 && profile.consumer().manualSettlement()) { + throw new IllegalArgumentException( + "an M1 destination cannot configure manual settlement: " + profile.name().value()); + } + if (profile.deliveryGuarantee() == DeliveryGuarantee.AT_LEAST_ONCE + && profile.producer().confirmation() == ConfirmationRequirement.NONE) { + throw new IllegalArgumentException( + "at-least-once delivery requires a publish confirmation for " + profile.name().value()); + } + if (profile.production() && profile.topologyAutoCreate()) { + throw new IllegalArgumentException( + "a production profile must not auto-create topology: " + profile.name().value()); + } + if (profile.orderingScope() == OrderingScope.DESTINATION + && profile.consumer().concurrency() > 1) { + throw new IllegalArgumentException( + "destination-scoped ordering requires concurrency 1 for " + profile.name().value()); + } + if (profile.isOrdered() && profile.consumer().maxInFlightPerOrderingUnit() > 1) { + throw new IllegalArgumentException( + "an ordered destination allows at most one in-flight message per ordering unit: " + + profile.name().value()); + } + if (profile.physical().isEmpty()) { + throw new IllegalArgumentException( + "a destination profile requires a physical address: " + profile.name().value()); + } + if (profile.retry().mode() == RetryMode.NONE && profile.retry().maxAttempts() > 1) { + throw new IllegalArgumentException( + "retry mode NONE cannot allow more than one attempt: " + profile.name().value()); + } + if (profile.retry().mode() == RetryMode.RETRY_DESTINATION + && profile.retry().retryDestination().isEmpty()) { + throw new IllegalArgumentException( + "retry mode RETRY_DESTINATION requires a retry destination: " + profile.name().value()); + } + if (profile.retry().maxAttempts() > 1 + && profile.retry().mode() != RetryMode.NONE + && !profile.deadLetter().enabled()) { + throw new IllegalArgumentException( + "a retrying destination needs somewhere to park exhausted messages: " + + profile.name().value()); + } + } + + /** + * Validates a whole registry, including the retry and dead letter graphs. + * + *

Cycles are checked across profiles rather than per profile: an individually sane pair of + * profiles can still form a loop that replays a poison message forever. + * + * @param profiles every registered profile + * @throws IllegalArgumentException when a profile or the graph is invalid + */ + public void validateAll(Collection profiles) { + Objects.requireNonNull(profiles, "profiles must not be null"); + + Map byName = new LinkedHashMap<>(); + for (DestinationProfile profile : profiles) { + validate(profile); + if (byName.put(profile.name(), profile) != null) { + throw new IllegalArgumentException("duplicate destination profile: " + profile.name()); + } + } + + for (DestinationProfile profile : byName.values()) { + requireCycleFree(profile, byName, Edge.RETRY); + requireCycleFree(profile, byName, Edge.DEAD_LETTER); + } + } + + /** Which forwarding edge a cycle walk follows. */ + private enum Edge { + RETRY("retry"), + DEAD_LETTER("dead letter"); + + private final String label; + + Edge(String label) { + this.label = label; + } + + Optional from(DestinationProfile profile) { + return this == RETRY + ? profile.retry().retryDestination() + : profile.deadLetter().destination(); + } + } + + private void requireCycleFree( + DestinationProfile start, Map byName, Edge edge) { + + Set visited = new HashSet<>(); + List path = new ArrayList<>(); + DestinationProfile current = start; + + while (true) { + if (!visited.add(current.name())) { + throw new IllegalArgumentException( + edge.label + " graph contains a cycle: " + String.join(" -> ", path)); + } + path.add(current.name().value()); + + Optional next = edge.from(current); + if (next.isEmpty()) { + return; + } + DestinationName nextName = next.get(); + DestinationProfile nextProfile = byName.get(nextName); + if (nextProfile == null) { + throw new IllegalArgumentException( + edge.label + " destination is not registered: " + nextName.value()); + } + current = nextProfile; + } + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/FailureDescriptorDefaults.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/FailureDescriptorDefaults.java new file mode 100644 index 00000000..517cf789 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/FailureDescriptorDefaults.java @@ -0,0 +1,14 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; + +/** The category retry defaults, kept in one place so policy and engine cannot disagree. */ +final class FailureDescriptorDefaults { + + private FailureDescriptorDefaults() {} + + static boolean retryable(FailureCategory category) { + return FailureDescriptor.defaultRetryable(category); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/InFlightLimiter.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/InFlightLimiter.java new file mode 100644 index 00000000..30ef288e --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/InFlightLimiter.java @@ -0,0 +1,82 @@ +package dev.caskeleton.messaging.policy; + +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +/** + * A bounded permit pool for producer-side in-flight work. + * + *

The wait is bounded on purpose. Section 40.3 of the design specifies "bounded wait, then + * {@code MessageBackpressureException}": a short wait absorbs the bursts that a saturated producer + * buffer clears on its own, while the bound stops a persistent stall from converting into unbounded + * thread occupancy in the calling application. + * + *

A zero wait is legal and means "reject immediately", which is what a latency-sensitive caller + * that would rather shed load than queue should use. + */ +public final class InFlightLimiter { + + private final Semaphore permits; + private final Duration maxWait; + private final int limit; + + /** + * Creates a limiter. + * + * @param limit the number of concurrent in-flight operations allowed + * @param maxWait how long an admission attempt may wait for a permit + */ + public InFlightLimiter(int limit, Duration maxWait) { + Objects.requireNonNull(maxWait, "maxWait must not be null"); + if (limit < 1) { + throw new IllegalArgumentException("limit must be at least 1"); + } + if (maxWait.isNegative()) { + throw new IllegalArgumentException("maxWait must not be negative"); + } + this.limit = limit; + this.maxWait = maxWait; + // Fair ordering: an unfair semaphore lets a late arrival barge ahead of a caller that has + // already been waiting, which turns a bounded wait into an unbounded one for the unlucky. + this.permits = new Semaphore(limit, true); + } + + /** + * Tries to acquire one permit, waiting no longer than the configured bound. + * + * @return true when a permit was acquired + * @throws InterruptedException when the waiting thread is interrupted + */ + public boolean tryAcquire() throws InterruptedException { + return permits.tryAcquire(maxWait.toNanos(), TimeUnit.NANOSECONDS); + } + + /** Returns one permit to the pool. */ + public void release() { + // Never exceed the configured limit: an unbalanced release would raise the ceiling silently + // and the limiter would stop limiting anything. + if (permits.availablePermits() < limit) { + permits.release(); + } + } + + /** + * Returns how many operations are currently admitted. + * + * @return the in-flight count + */ + public int inFlight() { + return limit - permits.availablePermits(); + } + + /** + * Returns the configured ceiling. + * + * @return the in-flight limit + */ + public int limit() { + return limit; + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/MessagingAdmissionController.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/MessagingAdmissionController.java new file mode 100644 index 00000000..c426ba2b --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/MessagingAdmissionController.java @@ -0,0 +1,101 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.error.MessageBackpressureException; +import dev.caskeleton.messaging.api.error.MessageTooLargeException; +import java.util.Objects; + +/** + * The single gate every publish passes before any transport work begins. + * + *

Order matters and is fixed here rather than left to each adapter: the payload limit is checked + * before a permit is taken. An oversized message can never succeed, so letting it occupy a + * scarce in-flight permit while it is being rejected would let a stream of bad messages starve the + * good ones. + * + *

Both refusals happen before transmission, so neither is ambiguous — the caller may resubmit + * under the same message id without risking a duplicate. That is the reason admission lives at the + * front of the pipeline instead of being folded into the adapter's error handling, where a partial + * write would already have made the outcome unknowable. + */ +public final class MessagingAdmissionController { + + private final PayloadLimitGuard payloadGuard; + private final InFlightLimiter limiter; + private volatile boolean acceptingNewWork = true; + + /** + * Creates an admission controller. + * + * @param payloadGuard the payload size guard + * @param limiter the in-flight permit pool + */ + public MessagingAdmissionController(PayloadLimitGuard payloadGuard, InFlightLimiter limiter) { + this.payloadGuard = Objects.requireNonNull(payloadGuard, "payloadGuard must not be null"); + this.limiter = Objects.requireNonNull(limiter, "limiter must not be null"); + } + + /** + * Admits one publish, or refuses it. + * + * @param destination the logical destination name + * @param payloadBytes the encoded payload size + * @throws MessageTooLargeException when the payload exceeds the destination limit + * @throws MessageBackpressureException when shutting down or no permit became available + */ + public void admit(String destination, int payloadBytes) { + Objects.requireNonNull(destination, "destination must not be null"); + payloadGuard.checkPayload(destination, payloadBytes); + + if (!acceptingNewWork) { + throw new MessageBackpressureException( + "SHUTTING_DOWN", "the messaging runtime stopped accepting new publishes"); + } + + boolean acquired; + try { + acquired = limiter.tryAcquire(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new MessageBackpressureException( + "ADMISSION_INTERRUPTED", "interrupted while waiting for a publish permit"); + } + if (!acquired) { + throw new MessageBackpressureException( + "IN_FLIGHT_LIMIT_EXCEEDED", + "no publish permit became available for %s within the bounded wait" + .formatted(destination)); + } + } + + /** Releases the permit taken by a successful {@link #admit}. */ + public void complete() { + limiter.release(); + } + + /** + * Stops admitting new work, as the first step of graceful shutdown. + * + *

Permits already held stay valid: the point is to drain what is in flight, not to abandon it. + */ + public void stopAcceptingNewWork() { + acceptingNewWork = false; + } + + /** + * Reports whether new publishes are being admitted. + * + * @return true while the controller accepts new work + */ + public boolean isAcceptingNewWork() { + return acceptingNewWork; + } + + /** + * Returns the current in-flight count. + * + * @return the number of admitted, uncompleted publishes + */ + public int inFlight() { + return limiter.inFlight(); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/OrderingImpact.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/OrderingImpact.java new file mode 100644 index 00000000..81b93409 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/OrderingImpact.java @@ -0,0 +1,11 @@ +package dev.caskeleton.messaging.policy; + +/** What a retry strategy is permitted to do to message order. */ +public enum OrderingImpact { + + /** Order must survive the retry. */ + PRESERVE, + + /** The retried message may be delivered out of its original position. */ + ALLOW_REORDER +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/PayloadLimitGuard.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/PayloadLimitGuard.java new file mode 100644 index 00000000..4f7cf440 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/PayloadLimitGuard.java @@ -0,0 +1,86 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.error.MessageTooLargeException; +import java.util.List; +import java.util.Objects; + +/** + * Rejects oversized payloads and batches before anything is handed to a broker. + * + *

Local rejection is deliberate. A broker that refuses an oversized record does so after the + * bytes have crossed the network, and some brokers only refuse it at the leader, after the producer + * has already buffered and batched it. Checking here keeps the failure cheap, immediate, and + * unambiguous: nothing was transmitted, so the caller may fix and resubmit under the same message + * id. + * + *

Batches are limited by count and bytes. A count limit alone lets a handful of large + * messages exceed the broker's frame; a byte limit alone lets a huge number of tiny messages exceed + * its request timeout. + */ +public final class PayloadLimitGuard { + + private final PayloadPolicy policy; + + /** + * Creates a guard for one destination's payload policy. + * + * @param policy the destination's payload limits + */ + public PayloadLimitGuard(PayloadPolicy policy) { + this.policy = Objects.requireNonNull(policy, "policy must not be null"); + } + + /** + * Checks one encoded payload against the destination limit. + * + * @param destination the logical destination name, used in the diagnostic + * @param payloadBytes the encoded payload size + * @throws MessageTooLargeException when the payload exceeds the limit + */ + public void checkPayload(String destination, int payloadBytes) { + Objects.requireNonNull(destination, "destination must not be null"); + if (payloadBytes < 0) { + throw new IllegalArgumentException("payloadBytes must not be negative"); + } + if (payloadBytes > policy.maxBytes()) { + throw new MessageTooLargeException( + "PAYLOAD_LIMIT_EXCEEDED", + "payload of %d bytes exceeds the %d byte limit for %s; use claim check" + .formatted(payloadBytes, policy.maxBytes(), destination)); + } + } + + /** + * Checks a batch against both the count and the aggregate byte limit. + * + * @param destination the logical destination name, used in the diagnostic + * @param payloadSizes each entry's encoded payload size + * @param maxBatchCount the largest accepted entry count + * @param maxBatchBytes the largest accepted aggregate size + * @throws MessageTooLargeException when the batch exceeds either limit + */ + public void checkBatch( + String destination, List payloadSizes, int maxBatchCount, long maxBatchBytes) { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(payloadSizes, "payloadSizes must not be null"); + + if (payloadSizes.size() > maxBatchCount) { + throw new MessageTooLargeException( + "BATCH_COUNT_EXCEEDED", + "batch of %d entries exceeds the %d entry limit for %s" + .formatted(payloadSizes.size(), maxBatchCount, destination)); + } + + long total = 0; + for (int index = 0; index < payloadSizes.size(); index++) { + checkPayload(destination, payloadSizes.get(index)); + total += payloadSizes.get(index); + } + if (total > maxBatchBytes) { + throw new MessageTooLargeException( + "BATCH_BYTES_EXCEEDED", + "batch of %d bytes exceeds the %d byte limit for %s" + .formatted(total, maxBatchBytes, destination)); + } + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/PayloadPolicy.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/PayloadPolicy.java new file mode 100644 index 00000000..579eae72 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/PayloadPolicy.java @@ -0,0 +1,39 @@ +package dev.caskeleton.messaging.policy; + +/** + * The size contract for a destination's payloads. + * + *

The portability default is one mebibyte and the absolute ceiling is eight. Raising a broker's + * frame limit to carry large payloads trades a bounded, testable failure for an unbounded one: it + * degrades broker memory, replication latency, and consumer recovery all at once. Anything above + * the threshold is expected to travel by Claim Check instead. + * + * @param maxBytes the encoded payload ceiling for this destination + * @param claimCheckThresholdBytes the size above which payloads are offloaded + */ +public record PayloadPolicy(int maxBytes, int claimCheckThresholdBytes) { + + /** The portability default. */ + public static final int DEFAULT_MAX_BYTES = 1_048_576; + + /** The absolute ceiling no destination may exceed. */ + public static final int HARD_MAX_BYTES = 8_388_608; + + public PayloadPolicy { + if (maxBytes < 1) { + throw new IllegalArgumentException("payload maxBytes must be positive"); + } + if (claimCheckThresholdBytes < 1) { + throw new IllegalArgumentException("claimCheckThresholdBytes must be positive"); + } + } + + /** + * Returns the portability default. + * + * @return a one mebibyte policy + */ + public static PayloadPolicy defaults() { + return new PayloadPolicy(DEFAULT_MAX_BYTES, DEFAULT_MAX_BYTES); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/PhysicalDestination.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/PhysicalDestination.java new file mode 100644 index 00000000..e6b7e156 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/PhysicalDestination.java @@ -0,0 +1,125 @@ +package dev.caskeleton.messaging.policy; + +import java.util.Objects; +import java.util.Optional; + +/** + * The broker-specific address a logical destination maps to. + * + *

Held here and nowhere else. Once a topic name reaches application code the logical destination + * stops being a boundary, and swapping the broker under a service becomes a code change instead of + * a configuration change. + * + * @param topic the Kafka or Pulsar topic + * @param exchange the AMQP exchange + * @param routingKey the AMQP routing key + * @param queue the AMQP queue + * @param subject the NATS subject + * @param stream the NATS or Pulsar stream + */ +public record PhysicalDestination( + Optional topic, + Optional exchange, + Optional routingKey, + Optional queue, + Optional subject, + Optional stream) { + + public PhysicalDestination { + Objects.requireNonNull(topic, "topic must not be null"); + Objects.requireNonNull(exchange, "exchange must not be null"); + Objects.requireNonNull(routingKey, "routingKey must not be null"); + Objects.requireNonNull(queue, "queue must not be null"); + Objects.requireNonNull(subject, "subject must not be null"); + Objects.requireNonNull(stream, "stream must not be null"); + } + + /** + * Builds a Kafka topic mapping. + * + * @param topic the topic name + * @return the physical mapping + */ + public static PhysicalDestination kafkaTopic(String topic) { + return new PhysicalDestination( + Optional.of(topic), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + } + + /** + * Builds an AMQP exchange, routing key, and queue mapping. + * + * @param exchange the exchange name + * @param routingKey the routing key + * @param queue the queue name + * @return the physical mapping + */ + public static PhysicalDestination rabbitQueue(String exchange, String routingKey, String queue) { + return new PhysicalDestination( + Optional.empty(), + Optional.of(exchange), + Optional.of(routingKey), + Optional.of(queue), + Optional.empty(), + Optional.empty()); + } + + /** + * Builds a Pulsar topic and subscription mapping. + * + *

The subscription is part of the address, not of the consumer configuration. Two consumers on + * the same topic with different subscriptions receive independent copies of every message, so the + * subscription name determines what is delivered just as much as the topic does. + * + * @param topic the fully-qualified Pulsar topic + * @param subscription the durable subscription name + * @return the physical mapping + */ + public static PhysicalDestination pulsarTopic(String topic, String subscription) { + return new PhysicalDestination( + Optional.of(topic), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.of(subscription)); + } + + /** + * Builds a NATS JetStream subject and stream mapping. + * + *

Both are required. A subject without its stream is ambiguous: JetStream lets several streams + * capture overlapping subjects, and which one a message lands in decides its retention, its + * replication, and whether it is replayable at all. + * + * @param subject the JetStream subject + * @param stream the stream capturing that subject + * @return the physical mapping + */ + public static PhysicalDestination natsStream(String subject, String stream) { + return new PhysicalDestination( + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.of(subject), + Optional.of(stream)); + } + + /** + * Reports whether any physical address is present. + * + * @return true when at least one address field is set + */ + public boolean isEmpty() { + return topic.isEmpty() + && exchange.isEmpty() + && queue.isEmpty() + && subject.isEmpty() + && stream.isEmpty(); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/ProducerPolicy.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/ProducerPolicy.java new file mode 100644 index 00000000..c2f7ce12 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/ProducerPolicy.java @@ -0,0 +1,38 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.destination.ConfirmationRequirement; +import java.time.Duration; +import java.util.Objects; + +/** + * The producer-side contract for a destination. + * + * @param confirmation the confirmation strength this destination demands + * @param timeout the publish deadline + * @param mandatoryRouting whether the broker must report unroutable publishes + * @param idempotentProducer whether producer-side duplicate suppression is required + */ +public record ProducerPolicy( + ConfirmationRequirement confirmation, + Duration timeout, + boolean mandatoryRouting, + boolean idempotentProducer) { + + public ProducerPolicy { + Objects.requireNonNull(confirmation, "confirmation must not be null"); + Objects.requireNonNull(timeout, "timeout must not be null"); + if (timeout.isNegative() || timeout.isZero()) { + throw new IllegalArgumentException("producer timeout must be positive"); + } + } + + /** + * Returns the Stable default: replication evidence, five seconds, mandatory routing, idempotent. + * + * @return the default producer policy + */ + public static ProducerPolicy defaults() { + return new ProducerPolicy( + ConfirmationRequirement.REPLICATION_OR_PERSISTENCE_ACK, Duration.ofSeconds(5), true, true); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryContext.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryContext.java new file mode 100644 index 00000000..fd804278 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryContext.java @@ -0,0 +1,34 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.delivery.DeliveryMetadata; +import dev.caskeleton.messaging.api.destination.MessagingCapabilities; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import java.util.Objects; + +/** + * Everything the retry engine is allowed to decide from. + * + *

Capabilities are an input rather than an assumption: the same policy resolves to + * pause-and-retry on a partitioned Kafka topic and to a retry destination on a queue that cannot + * pause, and the engine must not pick a strategy the adapter cannot actually carry out. + * + * @param destination the validated destination profile + * @param delivery transport-side facts about this attempt + * @param failure the sanitized failure + * @param capabilities what the adapter can actually do for this destination + * @param handlerMayHaveCommittedSideEffect whether the handler might already have committed + */ +public record RetryContext( + DestinationProfile destination, + DeliveryMetadata delivery, + FailureDescriptor failure, + MessagingCapabilities capabilities, + boolean handlerMayHaveCommittedSideEffect) { + + public RetryContext { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(delivery, "delivery must not be null"); + Objects.requireNonNull(failure, "failure must not be null"); + Objects.requireNonNull(capabilities, "capabilities must not be null"); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryDecision.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryDecision.java new file mode 100644 index 00000000..735dfb1e --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryDecision.java @@ -0,0 +1,76 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import java.time.Duration; +import java.util.Objects; + +/** What the platform will do with a failed delivery. */ +public sealed interface RetryDecision + permits RetryDecision.RetryInline, + RetryDecision.PauseAndRetry, + RetryDecision.PublishToRetryDestination, + RetryDecision.DeadLetter, + RetryDecision.Reject { + + /** + * Retry without releasing the delivery. + * + * @param delay how long to wait first + */ + record RetryInline(Duration delay) implements RetryDecision { + public RetryInline { + Objects.requireNonNull(delay, "delay must not be null"); + } + } + + /** + * Pause the ordering unit and redeliver from the same position. + * + *

The only strategy that both retries and preserves order, because the message never leaves + * its position in the log. + * + * @param delay how long to stay paused + */ + record PauseAndRetry(Duration delay) implements RetryDecision { + public PauseAndRetry { + Objects.requireNonNull(delay, "delay must not be null"); + } + } + + /** + * Re-publish to a separate retry destination. + * + * @param destination the retry destination + * @param delay the intended delay before redelivery + */ + record PublishToRetryDestination(DestinationName destination, Duration delay) + implements RetryDecision { + public PublishToRetryDestination { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(delay, "delay must not be null"); + } + } + + /** + * Route to the dead letter destination. + * + * @param failure the sanitized failure + */ + record DeadLetter(FailureDescriptor failure) implements RetryDecision { + public DeadLetter { + Objects.requireNonNull(failure, "failure must not be null"); + } + } + + /** + * Discard without dead lettering. + * + * @param failure the sanitized failure + */ + record Reject(FailureDescriptor failure) implements RetryDecision { + public Reject { + Objects.requireNonNull(failure, "failure must not be null"); + } + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryDecisionEngine.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryDecisionEngine.java new file mode 100644 index 00000000..d65ae606 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryDecisionEngine.java @@ -0,0 +1,13 @@ +package dev.caskeleton.messaging.policy; + +/** Decides what happens to a failed delivery. */ +public interface RetryDecisionEngine { + + /** + * Decides the outcome for one failed delivery. + * + * @param context the destination policy, delivery facts, failure, and capabilities + * @return the decision + */ + RetryDecision decide(RetryContext context); +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryMode.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryMode.java new file mode 100644 index 00000000..16daa2bd --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryMode.java @@ -0,0 +1,29 @@ +package dev.caskeleton.messaging.policy; + +/** + * How a retry is carried out. + * + *

The mode determines whether ordering survives, which is why it is a policy decision rather + * than an adapter detail. Only {@link #NONE}, {@link #INLINE}, {@link #BLOCKING}, and {@link + * #PAUSE_PARTITION} can preserve order; the two that re-publish cannot. + */ +public enum RetryMode { + + /** No automatic retry; failures go straight to the dead letter or reject path. */ + NONE, + + /** Retry inside the same handler invocation, without releasing the delivery. */ + INLINE, + + /** Hold the consumer thread between attempts. */ + BLOCKING, + + /** Pause the ordering unit and redeliver from the same position. */ + PAUSE_PARTITION, + + /** Re-publish to a separate retry destination. Reorders relative to the source. */ + RETRY_DESTINATION, + + /** Use the broker's own delayed delivery. Reorders relative to the source. */ + BROKER_DELAYED +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryPolicy.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryPolicy.java new file mode 100644 index 00000000..f4cce47c --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/RetryPolicy.java @@ -0,0 +1,136 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.FailureCategory; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * The retry contract for one destination. + * + *

Automatic retry is opt-in. The default for an ordinary destination is zero attempts, because a + * retry that reorders a stream, multiplies a non-idempotent side effect, or hammers a throttled + * downstream is worse than a visible failure. + * + * @param mode how the retry is carried out + * @param maxAttempts total attempts including the first delivery + * @param initialDelay the first backoff interval + * @param maxDelay the backoff ceiling + * @param multiplier the exponential backoff factor + * @param jitter whether to randomise the backoff + * @param orderingImpact what this policy is allowed to do to order + * @param retryableCategories categories added to the retryable set + * @param nonRetryableCategories categories removed from the retryable set + * @param retryDestination the destination messages are re-published to, when the mode uses one + */ +public record RetryPolicy( + RetryMode mode, + int maxAttempts, + Duration initialDelay, + Duration maxDelay, + double multiplier, + boolean jitter, + OrderingImpact orderingImpact, + Set retryableCategories, + Set nonRetryableCategories, + Optional retryDestination) { + + /** + * Creates a policy with no retry destination. + * + *

Convenience for the majority of modes, which never re-publish. + * + * @param mode how the retry is carried out + * @param maxAttempts total attempts including the first delivery + * @param initialDelay the first backoff interval + * @param maxDelay the backoff ceiling + * @param multiplier the exponential backoff factor + * @param jitter whether to randomise the backoff + * @param orderingImpact what this policy is allowed to do to order + * @param retryableCategories categories added to the retryable set + * @param nonRetryableCategories categories removed from the retryable set + */ + public RetryPolicy( + RetryMode mode, + int maxAttempts, + Duration initialDelay, + Duration maxDelay, + double multiplier, + boolean jitter, + OrderingImpact orderingImpact, + Set retryableCategories, + Set nonRetryableCategories) { + this( + mode, + maxAttempts, + initialDelay, + maxDelay, + multiplier, + jitter, + orderingImpact, + retryableCategories, + nonRetryableCategories, + Optional.empty()); + } + + public RetryPolicy { + Objects.requireNonNull(retryDestination, "retryDestination must not be null"); + Objects.requireNonNull(mode, "retry mode must not be null"); + Objects.requireNonNull(initialDelay, "initialDelay must not be null"); + Objects.requireNonNull(maxDelay, "maxDelay must not be null"); + Objects.requireNonNull(orderingImpact, "orderingImpact must not be null"); + Objects.requireNonNull(retryableCategories, "retryableCategories must not be null"); + Objects.requireNonNull(nonRetryableCategories, "nonRetryableCategories must not be null"); + + if (maxAttempts < 1) { + throw new IllegalArgumentException("maxAttempts counts the first delivery, so it is >= 1"); + } + if (initialDelay.isNegative() || maxDelay.isNegative()) { + throw new IllegalArgumentException("retry delays must not be negative"); + } + if (maxDelay.compareTo(initialDelay) < 0) { + throw new IllegalArgumentException("maxDelay must not be smaller than initialDelay"); + } + if (multiplier < 1.0) { + throw new IllegalArgumentException("retry multiplier must be at least 1.0"); + } + retryableCategories = Set.copyOf(retryableCategories); + nonRetryableCategories = Set.copyOf(nonRetryableCategories); + + Set overlap = new java.util.HashSet<>(retryableCategories); + overlap.retainAll(nonRetryableCategories); + if (!overlap.isEmpty()) { + throw new IllegalArgumentException( + "a failure category cannot be both retryable and non-retryable: " + overlap); + } + } + + /** + * Returns the safe default: no automatic retry, order preserved. + * + * @return a no-retry policy + */ + public static RetryPolicy none() { + return new RetryPolicy( + RetryMode.NONE, + 1, + Duration.ZERO, + Duration.ZERO, + 1.0, + false, + OrderingImpact.PRESERVE, + Set.of(), + Set.of()); + } + + /** + * Reports whether this mode can move a message off its original ordering unit. + * + * @return true when the mode re-publishes or defers through the broker + */ + public boolean reorders() { + return mode == RetryMode.RETRY_DESTINATION || mode == RetryMode.BROKER_DELAYED; + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/SchemaPolicy.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/SchemaPolicy.java new file mode 100644 index 00000000..80c07223 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/SchemaPolicy.java @@ -0,0 +1,31 @@ +package dev.caskeleton.messaging.policy; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.schema.SchemaCompatibility; +import java.util.Objects; +import java.util.Set; + +/** + * The codec and compatibility contract for a destination. + * + *

The message type catalog is part of the policy, not something the codec discovers at runtime. + * A destination that accepts any type it happens to be handed has no reviewable wire contract. + * + * @param codec the content type the destination is encoded with + * @param compatibility the schema evolution mode enforced for this destination + * @param messageTypes the closed catalog of types this destination carries + */ +public record SchemaPolicy( + ContentType codec, SchemaCompatibility compatibility, Set messageTypes) { + + public SchemaPolicy { + Objects.requireNonNull(codec, "codec must not be null"); + Objects.requireNonNull(compatibility, "compatibility must not be null"); + Objects.requireNonNull(messageTypes, "messageTypes must not be null"); + if (messageTypes.isEmpty()) { + throw new IllegalArgumentException("a destination must declare at least one message type"); + } + messageTypes = Set.copyOf(messageTypes); + } +} diff --git a/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/SourceSettlement.java b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/SourceSettlement.java new file mode 100644 index 00000000..a9db7829 --- /dev/null +++ b/src/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/SourceSettlement.java @@ -0,0 +1,21 @@ +package dev.caskeleton.messaging.policy; + +import java.util.concurrent.CompletionStage; + +/** + * The callback that settles the source message. + * + *

Handed to the orchestrator rather than invoked by it directly so that the ordering constraint + * — settle only after the dead letter publish confirms — lives in one place instead of being + * re-implemented by every adapter. + */ +@FunctionalInterface +public interface SourceSettlement { + + /** + * Settles the source message. + * + * @return a stage completing when settlement has been attempted + */ + CompletionStage settle(); +} diff --git a/src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/DeadLetterOrchestratorTest.java b/src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/DeadLetterOrchestratorTest.java new file mode 100644 index 00000000..ecc4e36a --- /dev/null +++ b/src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/DeadLetterOrchestratorTest.java @@ -0,0 +1,312 @@ +package dev.caskeleton.messaging.policy; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.delivery.DeliveryContext; +import dev.caskeleton.messaging.api.delivery.DeliveryMetadata; +import dev.caskeleton.messaging.api.delivery.MessageDelivery; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; + +class DeadLetterOrchestratorTest { + + @Test + void settlesSourceOnlyAfterDlqConfirmation() { + AtomicBoolean sourceSettled = new AtomicBoolean(false); + FakePublisher publisher = FakePublisher.confirming(); + DeadLetterOrchestrator orchestrator = new DeadLetterOrchestrator(publisher); + + DeadLetterResult result = + orchestrator + .deadLetter( + DeadLetterFixtures.profile(), + DeadLetterFixtures.delivery(), + DeadLetterFixtures.failure(), + () -> { + sourceSettled.set(true); + return CompletableFuture.completedFuture(null); + }) + .toCompletableFuture() + .join(); + + assertThat(publisher.publishedBeforeSettlement()).isTrue(); + assertThat(result.sourceSettled()).isTrue(); + assertThat(sourceSettled).isTrue(); + } + + @Test + void keepsSourceUnsettledWhenDlqPublishFails() { + AtomicBoolean sourceSettled = new AtomicBoolean(false); + DeadLetterOrchestrator orchestrator = new DeadLetterOrchestrator(FakePublisher.ambiguous()); + + DeadLetterResult result = + orchestrator + .deadLetter( + DeadLetterFixtures.profile(), + DeadLetterFixtures.delivery(), + DeadLetterFixtures.failure(), + () -> { + sourceSettled.set(true); + return CompletableFuture.completedFuture(null); + }) + .toCompletableFuture() + .join(); + + assertThat(result.sourceSettled()).isFalse(); + assertThat(sourceSettled).isFalse(); + } + + @Test + void keepsSourceUnsettledWhenTheBrokerRejectsTheDeadLetterPublish() { + AtomicBoolean sourceSettled = new AtomicBoolean(false); + DeadLetterOrchestrator orchestrator = new DeadLetterOrchestrator(FakePublisher.rejecting()); + + DeadLetterResult result = + orchestrator + .deadLetter( + DeadLetterFixtures.profile(), + DeadLetterFixtures.delivery(), + DeadLetterFixtures.failure(), + () -> { + sourceSettled.set(true); + return CompletableFuture.completedFuture(null); + }) + .toCompletableFuture() + .join(); + + assertThat(result.sourceSettled()).isFalse(); + assertThat(sourceSettled).isFalse(); + } + + @Test + void preservesTheLogicalMessageIdAndPayload() { + FakePublisher publisher = FakePublisher.confirming(); + MessageDelivery delivery = DeadLetterFixtures.delivery(); + + new DeadLetterOrchestrator(publisher) + .deadLetter( + DeadLetterFixtures.profile(), + delivery, + DeadLetterFixtures.failure(), + () -> CompletableFuture.completedFuture(null)) + .toCompletableFuture() + .join(); + + MessageEnvelope published = publisher.lastEnvelope(); + assertThat(published.messageId()).isEqualTo(delivery.message().messageId()); + assertThat(published.payload().bytes()).isEqualTo(delivery.message().payload().bytes()); + } + + @Test + void publishesToTheConfiguredDeadLetterDestination() { + FakePublisher publisher = FakePublisher.confirming(); + + new DeadLetterOrchestrator(publisher) + .deadLetter( + DeadLetterFixtures.profile(), + DeadLetterFixtures.delivery(), + DeadLetterFixtures.failure(), + () -> CompletableFuture.completedFuture(null)) + .toCompletableFuture() + .join(); + + assertThat(publisher.lastDestination().name()) + .isEqualTo(new DestinationName("order-events-dlq")); + } + + @Test + void writesFailureContextIntoReservedHeadersOnly() { + FakePublisher publisher = FakePublisher.confirming(); + + new DeadLetterOrchestrator(publisher) + .deadLetter( + DeadLetterFixtures.profile(), + DeadLetterFixtures.delivery(), + DeadLetterFixtures.failure(), + () -> CompletableFuture.completedFuture(null)) + .toCompletableFuture() + .join(); + + MessageHeaders headers = publisher.lastEnvelope().headers(); + assertThat(headers.find(ReservedHeaders.FAILURE_CATEGORY)) + .map(HeaderValue::value) + .hasValue("PERMANENT_BUSINESS"); + assertThat(headers.find(ReservedHeaders.FAILURE_CODE)) + .map(HeaderValue::value) + .hasValue("ORDER_REJECTED"); + assertThat(headers.find(ReservedHeaders.ORIGIN_DESTINATION)) + .map(HeaderValue::value) + .hasValue("order-events"); + assertThat(headers.find("stacktrace")).isEmpty(); + } +} + +/** Builds dead letter fixtures. */ +final class DeadLetterFixtures { + + private DeadLetterFixtures() {} + + static DestinationProfile profile() { + return DestinationProfileFixtures.named( + "order-events", + RetryPolicy.none(), + DeadLetterPolicy.to(new DestinationName("order-events-dlq"))); + } + + static FailureDescriptor failure() { + return FailureDescriptor.of( + FailureCategory.PERMANENT_BUSINESS, "ORDER_REJECTED", "the order was rejected"); + } + + static MessageDelivery delivery() { + EncodedMessage payload = + new EncodedMessage( + "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), + ContentType.JSON, + Optional.empty()); + + MessageEnvelope envelope = + new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + Instant.parse("2026-08-10T09:15:00Z"), + Optional.of(Instant.parse("2026-08-10T09:15:00Z")), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + payload); + + return new MessageDelivery<>( + envelope, + new DeliveryMetadata( + new DestinationName("order-events"), + 3, + true, + Optional.empty(), + Optional.of("order.events.v1-0"), + Optional.of("order-projection"), + Instant.parse("2026-08-10T09:15:05Z")), + new DeliveryContext(Instant.parse("2026-08-10T09:15:35Z"), false, "order-projection")); + } +} + +/** A publisher that records call order so the settlement sequence can be asserted. */ +final class FakePublisher implements MessagePublisher { + + private final PublishCompletion completion; + private final List events = new ArrayList<>(); + + private MessageDestination lastDestination; + private MessageEnvelope lastEnvelope; + + private FakePublisher(PublishCompletion completion) { + this.completion = completion; + } + + static FakePublisher confirming() { + return new FakePublisher(PublishCompletion.CONFIRMED); + } + + static FakePublisher ambiguous() { + return new FakePublisher(PublishCompletion.AMBIGUOUS); + } + + static FakePublisher rejecting() { + return new FakePublisher(PublishCompletion.REJECTED); + } + + @Override + public CompletionStage publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options) { + lastDestination = destination; + lastEnvelope = message; + events.add("publish"); + return CompletableFuture.completedFuture(result()); + } + + boolean publishedBeforeSettlement() { + return events.indexOf("publish") == 0; + } + + MessageDestination lastDestination() { + return lastDestination; + } + + @SuppressWarnings("unchecked") + MessageEnvelope lastEnvelope() { + return (MessageEnvelope) lastEnvelope; + } + + private PublishResult result() { + return switch (completion) { + case CONFIRMED -> + new PublishResult( + PublishCompletion.CONFIRMED, + PublishEvidence.confirmed(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK), + RoutingOutcome.ROUTED, + Optional.empty(), + 1, + Duration.ofMillis(2), + Optional.empty()); + case AMBIGUOUS -> + new PublishResult( + PublishCompletion.AMBIGUOUS, + PublishEvidence.ambiguous(), + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + Duration.ofSeconds(5), + Optional.of( + FailureDescriptor.of( + FailureCategory.AMBIGUOUS, "CONFIRM_TIMEOUT", "confirm timed out"))); + case REJECTED -> + new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ofMillis(1), + Optional.of( + FailureDescriptor.of( + FailureCategory.PERMANENT_BUSINESS, "DLQ_REJECTED", "broker refused"))); + }; + } +} diff --git a/src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/DestinationProfileValidatorTest.java b/src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/DestinationProfileValidatorTest.java new file mode 100644 index 00000000..51dadaee --- /dev/null +++ b/src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/DestinationProfileValidatorTest.java @@ -0,0 +1,422 @@ +package dev.caskeleton.messaging.policy; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.delivery.ExternalSideEffectGuarantee; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.ConfirmationRequirement; +import dev.caskeleton.messaging.api.destination.DestinationKind; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.schema.SchemaCompatibility; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class DestinationProfileValidatorTest { + + private final DestinationProfileValidator validator = new DestinationProfileValidator(); + + @Test + void strictOrderingRejectsRetryDestination() { + DestinationProfile profile = + DestinationProfileFixtures.kafkaOrdered( + new RetryPolicy( + RetryMode.RETRY_DESTINATION, + 5, + Duration.ofSeconds(1), + Duration.ofMinutes(1), + 2.0, + true, + OrderingImpact.PRESERVE, + Set.of(), + Set.of())); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("ordering"); + } + + @Test + void payloadAboveHardLimitIsRejected() { + DestinationProfile profile = DestinationProfileFixtures.withPayloadLimit(8_388_609); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("8388608"); + } + + @Test + void dlqCannotPointToItself() { + DestinationProfile profile = DestinationProfileFixtures.selfReferencingDlq(); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("dead letter"); + } + + @Test + void keyedOrderingRequiresAKeyResolver() { + DestinationProfile profile = DestinationProfileFixtures.keyedWithoutResolver(); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("key resolver"); + } + + @Test + void anM1DestinationCannotConfigureManualSettlement() { + DestinationProfile profile = DestinationProfileFixtures.m1WithManualSettlement(); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("manual settlement"); + } + + @Test + void atLeastOnceRequiresAPublishConfirmation() { + DestinationProfile profile = DestinationProfileFixtures.atLeastOnceWithoutConfirmation(); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("confirmation"); + } + + @Test + void aProductionProfileCannotAutoCreateTopology() { + DestinationProfile profile = DestinationProfileFixtures.productionWithAutoCreate(); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("topology"); + } + + @Test + void aRetryingDestinationNeedsADeadLetterDestination() { + DestinationProfile profile = DestinationProfileFixtures.retryingWithoutDeadLetter(); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("park"); + } + + @Test + void retryDestinationCyclesAreRejectedAcrossProfiles() { + List cycle = DestinationProfileFixtures.retryCycle(); + + assertThatThrownBy(() -> validator.validateAll(cycle)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cycle"); + } + + @Test + void anUnregisteredRetryDestinationIsRejected() { + List dangling = DestinationProfileFixtures.danglingRetryDestination(); + + assertThatThrownBy(() -> validator.validateAll(dangling)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("not registered"); + } + + @Test + void aCoherentProfileValidates() { + assertThatCode(() -> validator.validate(DestinationProfileFixtures.base())) + .doesNotThrowAnyException(); + } +} + +/** Builds destination profiles for validator tests. */ +final class DestinationProfileFixtures { + + private DestinationProfileFixtures() {} + + static SchemaPolicy schema() { + return new SchemaPolicy( + ContentType.JSON, + SchemaCompatibility.BACKWARD_TRANSITIVE, + Set.of(new MessageType("order.created"))); + } + + static DestinationProfile base() { + return new DestinationProfile( + new DestinationName("order-events"), + "kafka-primary", + DestinationKind.EVENT_STREAM, + PhysicalDestination.kafkaTopic("order.events.v1"), + schema(), + DeliveryGuarantee.AT_LEAST_ONCE, + OrderingScope.NONE, + ExternalSideEffectGuarantee.IDEMPOTENCY_REQUIRED, + ProducerPolicy.defaults(), + ConsumerPolicy.defaults("order-projection"), + RetryPolicy.none(), + DeadLetterPolicy.disabled(), + PayloadPolicy.defaults(), + CapabilityTier.M1, + false, + false, + false); + } + + static DestinationProfile kafkaOrdered(RetryPolicy retry) { + DestinationProfile base = base(); + return new DestinationProfile( + base.name(), + base.broker(), + base.kind(), + base.physical(), + base.schema(), + base.deliveryGuarantee(), + OrderingScope.PARTITION, + base.externalSideEffectGuarantee(), + base.producer(), + base.consumer(), + retry, + DeadLetterPolicy.to(new DestinationName("order-events-dlq")), + base.payload(), + base.tier(), + base.production(), + base.keyResolverConfigured(), + base.topologyAutoCreate()); + } + + static DestinationProfile withPayloadLimit(int maxBytes) { + DestinationProfile base = base(); + return new DestinationProfile( + base.name(), + base.broker(), + base.kind(), + base.physical(), + base.schema(), + base.deliveryGuarantee(), + base.orderingScope(), + base.externalSideEffectGuarantee(), + base.producer(), + base.consumer(), + base.retry(), + base.deadLetter(), + new PayloadPolicy(maxBytes, 1_048_576), + base.tier(), + base.production(), + base.keyResolverConfigured(), + base.topologyAutoCreate()); + } + + static DestinationProfile selfReferencingDlq() { + DestinationProfile base = base(); + return new DestinationProfile( + base.name(), + base.broker(), + base.kind(), + base.physical(), + base.schema(), + base.deliveryGuarantee(), + base.orderingScope(), + base.externalSideEffectGuarantee(), + base.producer(), + base.consumer(), + base.retry(), + DeadLetterPolicy.to(base.name()), + base.payload(), + base.tier(), + base.production(), + base.keyResolverConfigured(), + base.topologyAutoCreate()); + } + + static DestinationProfile keyedWithoutResolver() { + DestinationProfile base = base(); + return new DestinationProfile( + base.name(), + base.broker(), + base.kind(), + base.physical(), + base.schema(), + base.deliveryGuarantee(), + OrderingScope.KEY, + base.externalSideEffectGuarantee(), + base.producer(), + base.consumer(), + base.retry(), + base.deadLetter(), + base.payload(), + base.tier(), + base.production(), + false, + base.topologyAutoCreate()); + } + + static DestinationProfile m1WithManualSettlement() { + DestinationProfile base = base(); + return new DestinationProfile( + base.name(), + base.broker(), + base.kind(), + base.physical(), + base.schema(), + base.deliveryGuarantee(), + base.orderingScope(), + base.externalSideEffectGuarantee(), + base.producer(), + new ConsumerPolicy(Optional.of("order-projection"), 1, 1, 16, Duration.ofSeconds(30), true), + base.retry(), + base.deadLetter(), + base.payload(), + CapabilityTier.M1, + base.production(), + base.keyResolverConfigured(), + base.topologyAutoCreate()); + } + + static DestinationProfile atLeastOnceWithoutConfirmation() { + DestinationProfile base = base(); + return new DestinationProfile( + base.name(), + base.broker(), + base.kind(), + base.physical(), + base.schema(), + DeliveryGuarantee.AT_LEAST_ONCE, + base.orderingScope(), + base.externalSideEffectGuarantee(), + new ProducerPolicy(ConfirmationRequirement.NONE, Duration.ofSeconds(5), true, true), + base.consumer(), + base.retry(), + base.deadLetter(), + base.payload(), + base.tier(), + base.production(), + base.keyResolverConfigured(), + base.topologyAutoCreate()); + } + + static DestinationProfile productionWithAutoCreate() { + DestinationProfile base = base(); + return new DestinationProfile( + base.name(), + base.broker(), + base.kind(), + base.physical(), + base.schema(), + base.deliveryGuarantee(), + base.orderingScope(), + base.externalSideEffectGuarantee(), + base.producer(), + base.consumer(), + base.retry(), + base.deadLetter(), + base.payload(), + base.tier(), + true, + base.keyResolverConfigured(), + true); + } + + static DestinationProfile retryingWithoutDeadLetter() { + DestinationProfile base = base(); + return new DestinationProfile( + base.name(), + base.broker(), + base.kind(), + base.physical(), + base.schema(), + base.deliveryGuarantee(), + base.orderingScope(), + base.externalSideEffectGuarantee(), + base.producer(), + base.consumer(), + new RetryPolicy( + RetryMode.BLOCKING, + 3, + Duration.ofMillis(200), + Duration.ofSeconds(2), + 2.0, + true, + OrderingImpact.PRESERVE, + Set.of(), + Set.of()), + DeadLetterPolicy.disabled(), + base.payload(), + base.tier(), + base.production(), + base.keyResolverConfigured(), + base.topologyAutoCreate()); + } + + static DestinationProfile named(String name, RetryPolicy retry, DeadLetterPolicy deadLetter) { + DestinationProfile base = base(); + return new DestinationProfile( + new DestinationName(name), + base.broker(), + base.kind(), + base.physical(), + base.schema(), + base.deliveryGuarantee(), + base.orderingScope(), + base.externalSideEffectGuarantee(), + base.producer(), + base.consumer(), + retry, + deadLetter, + base.payload(), + base.tier(), + base.production(), + base.keyResolverConfigured(), + base.topologyAutoCreate()); + } + + static DestinationProfile atMostOnceWithoutDeadLetter() { + DestinationProfile base = base(); + return new DestinationProfile( + base.name(), + base.broker(), + base.kind(), + base.physical(), + base.schema(), + DeliveryGuarantee.AT_MOST_ONCE, + base.orderingScope(), + base.externalSideEffectGuarantee(), + base.producer(), + base.consumer(), + RetryPolicy.none(), + DeadLetterPolicy.disabled(), + base.payload(), + base.tier(), + base.production(), + base.keyResolverConfigured(), + base.topologyAutoCreate()); + } + + static RetryPolicy retryTo(String destination) { + return new RetryPolicy( + RetryMode.RETRY_DESTINATION, + 3, + Duration.ofSeconds(1), + Duration.ofMinutes(1), + 2.0, + true, + OrderingImpact.ALLOW_REORDER, + Set.of(), + Set.of(), + Optional.of(new DestinationName(destination))); + } + + static List retryCycle() { + return List.of( + named("alpha", retryTo("beta"), DeadLetterPolicy.to(new DestinationName("alpha-dlq"))), + named("beta", retryTo("alpha"), DeadLetterPolicy.to(new DestinationName("beta-dlq"))), + named("alpha-dlq", RetryPolicy.none(), DeadLetterPolicy.disabled()), + named("beta-dlq", RetryPolicy.none(), DeadLetterPolicy.disabled())); + } + + static List danglingRetryDestination() { + return List.of( + named("alpha", retryTo("missing"), DeadLetterPolicy.to(new DestinationName("alpha-dlq"))), + named("alpha-dlq", RetryPolicy.none(), DeadLetterPolicy.disabled())); + } +} diff --git a/src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/MessagingAdmissionControllerTest.java b/src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/MessagingAdmissionControllerTest.java new file mode 100644 index 00000000..47683786 --- /dev/null +++ b/src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/MessagingAdmissionControllerTest.java @@ -0,0 +1,122 @@ +package dev.caskeleton.messaging.policy; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.error.MessageBackpressureException; +import dev.caskeleton.messaging.api.error.MessageTooLargeException; +import java.time.Duration; +import java.util.List; +import org.junit.jupiter.api.Test; + +class MessagingAdmissionControllerTest { + + private static final String DESTINATION = "order.created"; + + private static MessagingAdmissionController controller(int limit) { + return new MessagingAdmissionController( + new PayloadLimitGuard(new PayloadPolicy(1024, 512)), + new InFlightLimiter(limit, Duration.ZERO)); + } + + @Test + void anAdmittedPublishOccupiesAPermitUntilItCompletes() { + MessagingAdmissionController controller = controller(2); + + controller.admit(DESTINATION, 100); + + assertThat(controller.inFlight()).isEqualTo(1); + controller.complete(); + assertThat(controller.inFlight()).isZero(); + } + + @Test + void publishingBeyondTheInFlightLimitIsRefusedRatherThanQueued() { + MessagingAdmissionController controller = controller(1); + controller.admit(DESTINATION, 100); + + assertThatThrownBy(() -> controller.admit(DESTINATION, 100)) + .isInstanceOf(MessageBackpressureException.class) + .hasMessageContaining(DESTINATION); + } + + @Test + void backpressureIsRetryableBecauseNothingWasTransmitted() { + MessagingAdmissionController controller = controller(1); + controller.admit(DESTINATION, 100); + + assertThatThrownBy(() -> controller.admit(DESTINATION, 100)) + .isInstanceOfSatisfying( + MessageBackpressureException.class, + exception -> assertThat(exception.failure().retryable()).isTrue()); + } + + @Test + void anOversizedPayloadIsRejectedWithoutConsumingAPermit() { + MessagingAdmissionController controller = controller(1); + + assertThatThrownBy(() -> controller.admit(DESTINATION, 2048)) + .isInstanceOf(MessageTooLargeException.class); + + assertThat(controller.inFlight()) + .as("a message that can never succeed must not occupy a scarce permit") + .isZero(); + } + + @Test + void shutdownStopsAdmittingNewWorkWithoutRevokingPermitsAlreadyHeld() { + MessagingAdmissionController controller = controller(4); + controller.admit(DESTINATION, 100); + + controller.stopAcceptingNewWork(); + + assertThat(controller.isAcceptingNewWork()).isFalse(); + assertThat(controller.inFlight()).isEqualTo(1); + assertThatThrownBy(() -> controller.admit(DESTINATION, 100)) + .isInstanceOf(MessageBackpressureException.class) + .hasMessageContaining("stopped accepting"); + } + + @Test + void aReleaseWithoutAnAdmitCannotRaiseTheCeiling() { + MessagingAdmissionController controller = controller(1); + + controller.complete(); + controller.complete(); + controller.admit(DESTINATION, 100); + + assertThatThrownBy(() -> controller.admit(DESTINATION, 100)) + .as("unbalanced releases must not hand out more permits than the limit") + .isInstanceOf(MessageBackpressureException.class); + } + + @Test + void aBatchIsBoundedByCountAndByAggregateBytes() { + PayloadLimitGuard guard = new PayloadLimitGuard(new PayloadPolicy(1024, 512)); + + assertThatThrownBy(() -> guard.checkBatch(DESTINATION, List.of(10, 10, 10), 2, 10_000)) + .isInstanceOf(MessageTooLargeException.class) + .hasMessageContaining("entry limit"); + + assertThatThrownBy(() -> guard.checkBatch(DESTINATION, List.of(1000, 1000), 10, 1500)) + .isInstanceOf(MessageTooLargeException.class) + .hasMessageContaining("byte limit"); + } + + @Test + void aWaitingCallerIsAdmittedOnceAPermitIsReturned() throws InterruptedException { + MessagingAdmissionController controller = + new MessagingAdmissionController( + new PayloadLimitGuard(PayloadPolicy.defaults()), + new InFlightLimiter(1, Duration.ofSeconds(2))); + controller.admit(DESTINATION, 100); + + Thread waiter = new Thread(() -> controller.admit(DESTINATION, 100)); + waiter.start(); + controller.complete(); + waiter.join(Duration.ofSeconds(5).toMillis()); + + assertThat(waiter.isAlive()).isFalse(); + assertThat(controller.inFlight()).isEqualTo(1); + } +} diff --git a/src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/RetryDecisionEngineTest.java b/src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/RetryDecisionEngineTest.java new file mode 100644 index 00000000..53fc29c0 --- /dev/null +++ b/src/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/RetryDecisionEngineTest.java @@ -0,0 +1,259 @@ +package dev.caskeleton.messaging.policy; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.delivery.DeliveryMetadata; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessagingCapabilities; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class RetryDecisionEngineTest { + + private final DefaultRetryDecisionEngine engine = + new DefaultRetryDecisionEngine(new BackoffCalculator(() -> 1.0)); + + @Test + void deserializationFailureGoesDirectlyToParking() { + RetryDecision decision = + engine.decide(RetryContextFixtures.failure(FailureCategory.DESERIALIZATION, 1)); + + assertThat(decision).isInstanceOf(RetryDecision.DeadLetter.class); + } + + @Test + void authenticationAndConfigurationFailuresAreNeverRetried() { + assertThat(engine.decide(RetryContextFixtures.failure(FailureCategory.AUTHENTICATION, 1))) + .isInstanceOf(RetryDecision.DeadLetter.class); + assertThat(engine.decide(RetryContextFixtures.failure(FailureCategory.CONFIGURATION, 1))) + .isInstanceOf(RetryDecision.DeadLetter.class); + assertThat(engine.decide(RetryContextFixtures.failure(FailureCategory.AUTHORIZATION, 1))) + .isInstanceOf(RetryDecision.DeadLetter.class); + } + + @Test + void transientOrderedKafkaFailureUsesPauseStrategy() { + RetryDecision decision = engine.decide(RetryContextFixtures.orderedKafkaTransient(1)); + + assertThat(decision).isInstanceOf(RetryDecision.PauseAndRetry.class); + } + + @Test + void exhaustedAttemptGoesToDeadLetter() { + RetryDecision decision = engine.decide(RetryContextFixtures.transientAtMaximumAttempt()); + + assertThat(decision).isInstanceOf(RetryDecision.DeadLetter.class); + } + + @Test + void unorderedRetryDestinationModeRePublishes() { + RetryDecision decision = engine.decide(RetryContextFixtures.unorderedRetryDestination(1)); + + assertThat(decision) + .isInstanceOfSatisfying( + RetryDecision.PublishToRetryDestination.class, + publish -> + assertThat(publish.destination()) + .isEqualTo(new DestinationName("email-work-retry"))); + } + + @Test + void blockingModeRetriesInline() { + RetryDecision decision = engine.decide(RetryContextFixtures.blockingTransient(1)); + + assertThat(decision).isInstanceOf(RetryDecision.RetryInline.class); + } + + @Test + void backoffGrowsExponentiallyAndIsCappedByMaxDelay() { + BackoffCalculator calculator = new BackoffCalculator(() -> 1.0); + RetryPolicy policy = + new RetryPolicy( + RetryMode.BLOCKING, + 10, + Duration.ofMillis(200), + Duration.ofSeconds(2), + 2.0, + false, + OrderingImpact.PRESERVE, + Set.of(), + Set.of()); + + assertThat(calculator.delayFor(policy, 1)).isEqualTo(Duration.ofMillis(200)); + assertThat(calculator.delayFor(policy, 2)).isEqualTo(Duration.ofMillis(400)); + assertThat(calculator.delayFor(policy, 3)).isEqualTo(Duration.ofMillis(800)); + assertThat(calculator.delayFor(policy, 9)).isEqualTo(Duration.ofSeconds(2)); + } + + @Test + void fullJitterSpreadsRetriesAcrossTheWholeWindow() { + RetryPolicy policy = + new RetryPolicy( + RetryMode.BLOCKING, + 10, + Duration.ofSeconds(1), + Duration.ofSeconds(2), + 2.0, + true, + OrderingImpact.PRESERVE, + Set.of(), + Set.of()); + + assertThat(new BackoffCalculator(() -> 0.0).delayFor(policy, 1)).isEqualTo(Duration.ZERO); + assertThat(new BackoffCalculator(() -> 0.5).delayFor(policy, 1)) + .isEqualTo(Duration.ofMillis(500)); + } + + @Test + void anExplicitProfileOverrideCanMakeACategoryRetryable() { + RetryDecision decision = + engine.decide(RetryContextFixtures.withRetryableOverride(FailureCategory.DESERIALIZATION)); + + assertThat(decision).isInstanceOf(RetryDecision.RetryInline.class); + } + + @Test + void anAtMostOnceDestinationWithoutDlqDiscards() { + RetryDecision decision = engine.decide(RetryContextFixtures.atMostOnceWithoutDlq()); + + assertThat(decision).isInstanceOf(RetryDecision.Reject.class); + } +} + +/** Builds retry contexts for engine tests. */ +final class RetryContextFixtures { + + private RetryContextFixtures() {} + + private static final MessagingCapabilities ORDERED_KAFKA = + new MessagingCapabilities( + true, true, true, true, true, true, true, false, true, true, false, true); + + private static final MessagingCapabilities PLAIN_QUEUE = + new MessagingCapabilities( + true, true, true, false, false, false, false, false, false, false, true, true); + + static DeliveryMetadata delivery(int attempt) { + return new DeliveryMetadata( + new DestinationName("order-events"), + attempt, + attempt > 1, + Optional.empty(), + Optional.of("order.events.v1-0"), + Optional.of("order-projection"), + Instant.parse("2026-08-10T00:00:00Z")); + } + + static RetryContext failure(FailureCategory category, int attempt) { + return new RetryContext( + DestinationProfileFixtures.kafkaOrdered(pausePolicy()), + delivery(attempt), + FailureDescriptor.of(category, category.name(), "failed"), + ORDERED_KAFKA, + false); + } + + static RetryContext orderedKafkaTransient(int attempt) { + return new RetryContext( + DestinationProfileFixtures.kafkaOrdered(pausePolicy()), + delivery(attempt), + FailureDescriptor.of( + FailureCategory.TRANSIENT_INFRASTRUCTURE, "BROKER_TIMEOUT", "broker timed out"), + ORDERED_KAFKA, + false); + } + + static RetryContext transientAtMaximumAttempt() { + return new RetryContext( + DestinationProfileFixtures.kafkaOrdered(pausePolicy()), + delivery(3), + FailureDescriptor.of( + FailureCategory.TRANSIENT_INFRASTRUCTURE, "BROKER_TIMEOUT", "broker timed out"), + ORDERED_KAFKA, + false); + } + + static RetryContext unorderedRetryDestination(int attempt) { + return new RetryContext( + DestinationProfileFixtures.named( + "email-work", + DestinationProfileFixtures.retryTo("email-work-retry"), + DeadLetterPolicy.to(new DestinationName("email-work-dlq"))), + delivery(attempt), + FailureDescriptor.of( + FailureCategory.PROCESSING_TRANSIENT, "DOWNSTREAM_TIMEOUT", "downstream timed out"), + PLAIN_QUEUE, + false); + } + + static RetryContext blockingTransient(int attempt) { + return new RetryContext( + DestinationProfileFixtures.named( + "email-work", blockingPolicy(), DeadLetterPolicy.to(new DestinationName("email-dlq"))), + delivery(attempt), + FailureDescriptor.of(FailureCategory.THROTTLED, "RATE_LIMITED", "throttled"), + PLAIN_QUEUE, + false); + } + + static RetryContext withRetryableOverride(FailureCategory category) { + RetryPolicy policy = + new RetryPolicy( + RetryMode.BLOCKING, + 3, + Duration.ofMillis(100), + Duration.ofSeconds(1), + 2.0, + false, + OrderingImpact.PRESERVE, + Set.of(category), + Set.of()); + return new RetryContext( + DestinationProfileFixtures.named( + "email-work", policy, DeadLetterPolicy.to(new DestinationName("email-dlq"))), + delivery(1), + FailureDescriptor.of(category, category.name(), "failed"), + PLAIN_QUEUE, + false); + } + + static RetryContext atMostOnceWithoutDlq() { + return new RetryContext( + DestinationProfileFixtures.atMostOnceWithoutDeadLetter(), + delivery(1), + FailureDescriptor.of(FailureCategory.PERMANENT_BUSINESS, "INVALID", "rejected"), + PLAIN_QUEUE, + false); + } + + private static RetryPolicy pausePolicy() { + return new RetryPolicy( + RetryMode.PAUSE_PARTITION, + 3, + Duration.ofMillis(200), + Duration.ofSeconds(2), + 2.0, + false, + OrderingImpact.PRESERVE, + Set.of(), + Set.of()); + } + + private static RetryPolicy blockingPolicy() { + return new RetryPolicy( + RetryMode.BLOCKING, + 5, + Duration.ofMillis(200), + Duration.ofSeconds(2), + 2.0, + false, + OrderingImpact.PRESERVE, + Set.of(), + Set.of()); + } +} diff --git a/src/messaging/messaging-pulsar-experimental/build.gradle b/src/messaging/messaging-pulsar-experimental/build.gradle new file mode 100644 index 00000000..168449d6 --- /dev/null +++ b/src/messaging/messaging-pulsar-experimental/build.gradle @@ -0,0 +1,13 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-schema-api') + api project(':messaging:messaging-policy') + api project(':messaging:messaging-transport-spi') + api project(':messaging:messaging-observability') + api project(':messaging:messaging-security') + api project(':messaging:messaging-admin-api') + + implementation 'org.apache.pulsar:pulsar-client:4.0.3' +} diff --git a/src/messaging/messaging-pulsar-experimental/gradle.lockfile b/src/messaging/messaging-pulsar-experimental/gradle.lockfile new file mode 100644 index 00000000..3df51a0c --- /dev/null +++ b/src/messaging/messaging-pulsar-experimental/gradle.lockfile @@ -0,0 +1,102 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.fasterxml.jackson.core:jackson-annotations:2.20=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +io.micrometer:micrometer-commons:1.16.0=runtimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-core:1.16.0=runtimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-observation:1.16.0=runtimeClasspath,testRuntimeClasspath +io.opentelemetry:opentelemetry-api-incubator:1.45.0-alpha=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.opentelemetry:opentelemetry-api:1.55.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.opentelemetry:opentelemetry-common:1.55.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.opentelemetry:opentelemetry-context:1.55.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.pulsar:bouncy-castle-bc:4.1.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.pulsar:pulsar-client-admin-api:4.1.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.pulsar:pulsar-client-api:4.1.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.pulsar:pulsar-client:4.0.3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.bouncycastle:bcpkix-jdk18on:1.81=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.bouncycastle:bcprov-ext-jdk18on:1.78.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.bouncycastle:bcprov-jdk18on:1.81.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.bouncycastle:bcutil-jdk18on:1.81.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.hdrhistogram:HdrHistogram:2.2.2=runtimeClasspath,testRuntimeClasspath +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.latencyutils:LatencyUtils:2.0.3=runtimeClasspath,testRuntimeClasspath +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=compileClasspath,runtimeClasspath,spotbugs,spotbugsSlf4j,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty= diff --git a/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarMessagePosition.java b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarMessagePosition.java new file mode 100644 index 00000000..7271ac0b --- /dev/null +++ b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarMessagePosition.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.pulsar; + +import dev.caskeleton.messaging.api.publish.BrokerPosition; +import java.util.Map; + +/** + * A Pulsar message id, decomposed into its parts. + * + *

Kept as four fields rather than the opaque {@code MessageId} string so a diagnostic can be + * read without a Pulsar client to parse it. {@code batchIndex} is the part that surprises people: a + * batched message's id is shared by every message in the batch, so the index is what makes an + * individual message addressable at all. + * + * @param ledgerId the ledger holding the entry + * @param entryId the entry within the ledger + * @param partition the partition index, or -1 for a non-partitioned topic + * @param batchIndex the index within a batch, or -1 when the message was not batched + */ +public record PulsarMessagePosition(long ledgerId, long entryId, int partition, int batchIndex) + implements BrokerPosition { + + /** The value used where a topic is not partitioned or a message is not batched. */ + public static final int NOT_APPLICABLE = -1; + + @Override + public String broker() { + return "pulsar"; + } + + @Override + public Map diagnosticAttributes() { + return Map.of( + "ledgerId", Long.toString(ledgerId), + "entryId", Long.toString(entryId), + "partition", Integer.toString(partition), + "batchIndex", Integer.toString(batchIndex)); + } + + /** + * Returns the canonical Pulsar message id rendering. + * + * @return the message id as Pulsar prints it + */ + public String toMessageId() { + return batchIndex == NOT_APPLICABLE + ? "%d:%d:%d".formatted(ledgerId, entryId, partition) + : "%d:%d:%d:%d".formatted(ledgerId, entryId, partition, batchIndex); + } +} diff --git a/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarMessagingTransport.java b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarMessagingTransport.java new file mode 100644 index 00000000..78026f7a --- /dev/null +++ b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarMessagingTransport.java @@ -0,0 +1,249 @@ +package dev.caskeleton.messaging.pulsar; + +import dev.caskeleton.messaging.api.destination.DestinationCapabilities; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessagingCapabilities; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.error.MessagingCapabilityUnavailableException; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import dev.caskeleton.messaging.transport.MessagingTransport; +import dev.caskeleton.messaging.transport.TransportConsumerRegistration; +import dev.caskeleton.messaging.transport.TransportConsumerSpec; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import dev.caskeleton.messaging.transport.TransportPublishResult; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; + +/** + * The Experimental Pulsar adapter. + * + *

Disabled unless explicitly enabled. Experimental means the contract suite has been run but the + * live-broker certification has not, so the adapter must never become load-bearing because a + * default switched it on. + * + *

Pulsar's send future resolves with a {@code MessageId} once the broker has persisted the + * message to its configured number of bookies, which is genuine replication evidence rather than a + * receipt. A send that fails with a timeout is reported {@code AMBIGUOUS}: the write may have + * reached the bookies and only the acknowledgement been lost, and treating that as a rejection is + * how a message gets published twice. + */ +public final class PulsarMessagingTransport implements MessagingTransport { + + /** What this adapter can prove today. Transactions stay false while Experimental. */ + private static final MessagingCapabilities SHARED_CAPABILITIES = + new MessagingCapabilities( + true, true, true, true, false, false, true, true, false, false, true, true); + + private static final MessagingCapabilities KEY_SHARED_CAPABILITIES = + new MessagingCapabilities( + true, true, true, true, false, true, true, true, false, false, true, true); + + private final String brokerName; + private final long generation; + private final PulsarProfile profile; + private final PulsarSendOperation send; + private final Function consumerFactory; + private final AtomicBoolean closed = new AtomicBoolean(); + + /** + * Creates a publish-only transport. + * + * @param brokerName the logical broker name + * @param generation the runtime generation + * @param profile the Pulsar destination settings + * @param send the producer send operation + */ + public PulsarMessagingTransport( + String brokerName, long generation, PulsarProfile profile, PulsarSendOperation send) { + this( + brokerName, + generation, + profile, + send, + spec -> { + throw new MessagingCapabilityUnavailableException( + "PULSAR_CONSUMER_NOT_CONFIGURED", + "this Pulsar transport was created without a consumer factory"); + }); + } + + /** + * Creates a transport with a consumer factory. + * + * @param brokerName the logical broker name + * @param generation the runtime generation + * @param profile the Pulsar destination settings + * @param send the producer send operation + * @param consumerFactory builds a consumer registration for a spec + */ + public PulsarMessagingTransport( + String brokerName, + long generation, + PulsarProfile profile, + PulsarSendOperation send, + Function consumerFactory) { + this.brokerName = Objects.requireNonNull(brokerName, "brokerName must not be null"); + this.generation = generation; + this.profile = Objects.requireNonNull(profile, "profile must not be null"); + this.send = Objects.requireNonNull(send, "send must not be null"); + this.consumerFactory = Objects.requireNonNull(consumerFactory, "consumerFactory is required"); + } + + @Override + public CompletionStage publish(TransportPublishRequest request) { + Objects.requireNonNull(request, "request must not be null"); + + if (closed.get()) { + return completed( + rejectedLocally("PULSAR_TRANSPORT_CLOSED", "the transport is shutting down")); + } + int size = request.envelope().payload().size(); + int limit = request.profile().payload().maxBytes(); + if (size > limit) { + return completed( + rejectedLocally( + "PAYLOAD_TOO_LARGE", "encoded payload is " + size + " bytes, limit is " + limit)); + } + + return send.send(profile.topic(), request) + .handle((position, failure) -> failure == null ? confirmed(position) : classify(failure)) + .thenApply(TransportPublishResult::new); + } + + @Override + public TransportConsumerRegistration register(TransportConsumerSpec spec) { + Objects.requireNonNull(spec, "spec must not be null"); + if (closed.get()) { + throw new MessagingCapabilityUnavailableException( + "PULSAR_TRANSPORT_CLOSED", "the transport is shutting down"); + } + return consumerFactory.apply(spec); + } + + @Override + public DestinationCapabilities capabilities(DestinationName destination) { + MessagingCapabilities capabilities = + profile.mode().subscriptionType() == PulsarSubscriptionType.KEY_SHARED + ? KEY_SHARED_CAPABILITIES + : SHARED_CAPABILITIES; + return new DestinationCapabilities(destination, brokerName, capabilities); + } + + @Override + public String brokerName() { + return brokerName; + } + + @Override + public long generation() { + return generation; + } + + @Override + public void close() { + closed.set(true); + } + + /** + * Reports whether the transport is still accepting work. + * + * @return true until close + */ + public boolean isAcceptingWork() { + return !closed.get(); + } + + private static PublishResult confirmed(PulsarMessagePosition position) { + return new PublishResult( + PublishCompletion.CONFIRMED, + // The send future resolves only after the entry is persisted to the configured bookies, so + // this is replication evidence rather than a broker receipt. + new PublishEvidence( + true, + TransmissionEvidence.TRANSMITTED, + true, + ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK), + RoutingOutcome.ROUTED, + Optional.of(position), + 1, + Duration.ZERO, + Optional.empty()); + } + + private static PublishResult classify(Throwable failure) { + Throwable cause = + failure instanceof java.util.concurrent.CompletionException ? failure.getCause() : failure; + String name = cause == null ? "" : cause.getClass().getSimpleName(); + + // A timeout is the ambiguous case: the entry may already be on the bookies with only the + // acknowledgement lost. Reporting it as rejected is how the same message is published twice. + if (name.contains("Timeout")) { + return new PublishResult( + PublishCompletion.AMBIGUOUS, + new PublishEvidence( + true, TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED, false, ConfirmationLevel.NONE), + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of( + FailureDescriptor.of( + FailureCategory.TRANSIENT_INFRASTRUCTURE, + "PULSAR_SEND_TIMEOUT", + "the send timed out; the broker may already hold the message"))); + } + return new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of( + FailureDescriptor.of( + FailureCategory.PERMANENT_BUSINESS, + "PULSAR_SEND_REJECTED", + "the broker rejected the send: " + name))); + } + + private static TransportPublishResult rejectedLocally(String code, String message) { + return new TransportPublishResult( + new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of(FailureDescriptor.of(FailureCategory.PERMANENT_BUSINESS, code, message)))); + } + + private static CompletionStage completed(TransportPublishResult result) { + return CompletableFuture.completedFuture(result); + } + + /** The producer send, isolated so the adapter is testable without a broker. */ + @FunctionalInterface + public interface PulsarSendOperation { + + /** + * Sends one encoded message. + * + * @param topic the Pulsar topic + * @param request the publish request + * @return a stage completing with the broker position + */ + CompletionStage send(String topic, TransportPublishRequest request); + } +} diff --git a/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarProfile.java b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarProfile.java new file mode 100644 index 00000000..154cce1d --- /dev/null +++ b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarProfile.java @@ -0,0 +1,80 @@ +package dev.caskeleton.messaging.pulsar; + +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; + +/** + * The Pulsar-side settings for one destination. + * + *

{@code negativeAckRedeliveryDelay} is the platform's retry hook on Pulsar. Pulsar has no retry + * topic of its own in this adapter, so a failed handler negatively acknowledges and the broker + * redelivers after this delay — which means the delay is the backoff, and setting it to + * zero converts a failing handler into a spin loop against the broker. + * + *

{@code ackTimeout} is deliberately optional and off by default. It redelivers any message the + * consumer has not acknowledged in time, which looks like a safety net and behaves like a duplicate + * generator for handlers that are simply slow. + * + * @param topic the fully-qualified Pulsar topic + * @param subscription the durable subscription name + * @param mode the subscription type and acknowledgement style + * @param negativeAckRedeliveryDelay how long the broker waits after a negative acknowledgement + * @param ackTimeout redelivers unacknowledged messages after this long, when set + * @param maxRedeliverCount how many redeliveries before the message is parked + */ +public record PulsarProfile( + String topic, + String subscription, + PulsarSubscriptionMode mode, + Duration negativeAckRedeliveryDelay, + Optional ackTimeout, + int maxRedeliverCount) { + + /** Pulsar's own floor for an acknowledgement timeout. */ + public static final Duration MINIMUM_ACK_TIMEOUT = Duration.ofSeconds(10); + + public PulsarProfile { + Objects.requireNonNull(mode, "mode must not be null"); + Objects.requireNonNull(negativeAckRedeliveryDelay, "negativeAckRedeliveryDelay is required"); + Objects.requireNonNull(ackTimeout, "ackTimeout must not be null"); + requireText(topic, "topic"); + requireText(subscription, "subscription"); + + if (negativeAckRedeliveryDelay.isNegative() || negativeAckRedeliveryDelay.isZero()) { + throw new IllegalArgumentException( + "negativeAckRedeliveryDelay must be positive; zero turns a failing handler into a spin " + + "loop against the broker"); + } + if (maxRedeliverCount < 1) { + throw new IllegalArgumentException("maxRedeliverCount must be at least 1"); + } + if (ackTimeout.isPresent() && ackTimeout.get().compareTo(MINIMUM_ACK_TIMEOUT) < 0) { + throw new IllegalArgumentException( + "ackTimeout below %s is rejected by Pulsar itself".formatted(MINIMUM_ACK_TIMEOUT)); + } + } + + /** + * Returns a shared-subscription profile with the platform defaults. + * + * @param topic the Pulsar topic + * @param subscription the subscription name + * @return the profile + */ + public static PulsarProfile shared(String topic, String subscription) { + return new PulsarProfile( + topic, + subscription, + PulsarSubscriptionMode.shared(), + Duration.ofSeconds(30), + Optional.empty(), + 5); + } + + private static void requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " must not be blank"); + } + } +} diff --git a/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarProfileValidator.java b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarProfileValidator.java new file mode 100644 index 00000000..16380e02 --- /dev/null +++ b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarProfileValidator.java @@ -0,0 +1,66 @@ +package dev.caskeleton.messaging.pulsar; + +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.MessagingCapabilities; +import dev.caskeleton.messaging.api.error.MessagingCapabilityUnavailableException; +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.util.Objects; + +/** + * Guards the Pulsar Experimental adapter. + * + *

The subscription type and the destination's ordering scope have to agree, and the platform + * checks rather than assumes. A {@code Shared} subscription distributes messages round-robin across + * consumers, so a destination that declares key ordering on one is advertising a guarantee Pulsar + * is not providing — the messages arrive, in an order nobody promised. + * + *

Experimental means off by default. An adapter whose contract suite is still being proven must + * not become load-bearing because someone's configuration defaulted it on. + */ +public final class PulsarProfileValidator { + + /** + * Validates a destination against a Pulsar subscription. + * + * @param profile the destination profile + * @param subscriptionType the Pulsar subscription type + * @param enabled whether the experimental adapter is switched on + */ + public void validate( + DestinationProfile profile, PulsarSubscriptionType subscriptionType, boolean enabled) { + Objects.requireNonNull(profile, "profile must not be null"); + Objects.requireNonNull(subscriptionType, "subscriptionType must not be null"); + + if (!enabled) { + throw new MessagingCapabilityUnavailableException( + "PULSAR_DISABLED", + "the Pulsar adapter is experimental and disabled unless " + + "backend.messaging.experimental.pulsar=true"); + } + if (profile.orderingScope() == OrderingScope.KEY + && subscriptionType != PulsarSubscriptionType.KEY_SHARED) { + throw new IllegalArgumentException( + "keyed ordering requires a Key_Shared subscription: " + profile.name().value()); + } + if (profile.orderingScope() == OrderingScope.DESTINATION) { + throw new IllegalArgumentException( + "the Pulsar adapter does not offer destination-wide ordering: " + profile.name().value()); + } + } + + /** + * Returns what the Pulsar adapter can prove for a subscription type. + * + *

Transactions are reported as unavailable while the adapter is Experimental: Pulsar has them, + * but the platform has not yet proven them against its own contract suite, and advertising an + * unverified guarantee is exactly what the Experimental tier exists to prevent. + * + * @param subscriptionType the subscription type + * @return the capabilities + */ + public MessagingCapabilities capabilities(PulsarSubscriptionType subscriptionType) { + boolean keyed = subscriptionType == PulsarSubscriptionType.KEY_SHARED; + return new MessagingCapabilities( + true, true, true, true, keyed, keyed, true, true, false, false, true, true); + } +} diff --git a/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarSubscriptionMode.java b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarSubscriptionMode.java new file mode 100644 index 00000000..4d2c53e8 --- /dev/null +++ b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarSubscriptionMode.java @@ -0,0 +1,62 @@ +package dev.caskeleton.messaging.pulsar; + +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import java.util.Objects; + +/** + * How a Pulsar subscription is configured, and what ordering it can actually deliver. + * + *

Pairs the subscription type with its acknowledgement style because the two together determine + * the guarantee. A {@code Key_Shared} subscription with cumulative acknowledgement is not keyed + * ordering with a faster ack — cumulative ack over interleaved keys acknowledges messages from keys + * the consumer has not finished, so the combination silently loses the property the subscription + * type was chosen for. + * + * @param subscriptionType the Pulsar subscription type + * @param cumulativeAcknowledgement whether acknowledgements are cumulative rather than individual + */ +public record PulsarSubscriptionMode( + PulsarSubscriptionType subscriptionType, boolean cumulativeAcknowledgement) { + + public PulsarSubscriptionMode { + Objects.requireNonNull(subscriptionType, "subscriptionType must not be null"); + if (cumulativeAcknowledgement && subscriptionType == PulsarSubscriptionType.KEY_SHARED) { + throw new IllegalArgumentException( + "Key_Shared with cumulative acknowledgement acknowledges keys the consumer has not " + + "finished, which discards the per-key ordering the subscription type provides"); + } + if (cumulativeAcknowledgement && subscriptionType == PulsarSubscriptionType.SHARED) { + throw new IllegalArgumentException( + "Shared subscriptions do not support cumulative acknowledgement"); + } + } + + /** + * Returns a shared subscription with individual acknowledgement. + * + * @return the shared mode + */ + public static PulsarSubscriptionMode shared() { + return new PulsarSubscriptionMode(PulsarSubscriptionType.SHARED, false); + } + + /** + * Returns a key-shared subscription with individual acknowledgement. + * + * @return the key-shared mode + */ + public static PulsarSubscriptionMode keyShared() { + return new PulsarSubscriptionMode(PulsarSubscriptionType.KEY_SHARED, false); + } + + /** + * Returns the strongest ordering this mode can deliver. + * + * @return the ordering scope + */ + public OrderingScope orderingScope() { + return subscriptionType == PulsarSubscriptionType.KEY_SHARED + ? OrderingScope.KEY + : OrderingScope.NONE; + } +} diff --git a/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarSubscriptionType.java b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarSubscriptionType.java new file mode 100644 index 00000000..4a88a321 --- /dev/null +++ b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarSubscriptionType.java @@ -0,0 +1,17 @@ +package dev.caskeleton.messaging.pulsar; + +/** + * The Pulsar subscription types the Experimental adapter exposes. + * + *

Only the two that map cleanly onto the platform's ordering model. {@code Exclusive} and {@code + * Failover} are omitted deliberately: they encode a topology decision the destination profile + * already owns, and exposing them would give an application two places to configure the same thing. + */ +public enum PulsarSubscriptionType { + + /** Competing consumers, no ordering. */ + SHARED, + + /** Competing consumers with per-key ordering. */ + KEY_SHARED +} diff --git a/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarTransactionCapability.java b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarTransactionCapability.java new file mode 100644 index 00000000..104caf4c --- /dev/null +++ b/src/messaging/messaging-pulsar-experimental/src/main/java/dev/caskeleton/messaging/pulsar/PulsarTransactionCapability.java @@ -0,0 +1,49 @@ +package dev.caskeleton.messaging.pulsar; + +import dev.caskeleton.messaging.api.error.MessagingCapabilityUnavailableException; + +/** + * States, in code, that Pulsar transactions are not promoted. + * + *

Pulsar has transactions. The platform does not offer them, and the distinction matters enough + * to be a class rather than a comment: an operator reading the capability matrix needs to know the + * answer is "not proven here", not "the broker cannot do it". + * + *

What is missing is the proof. The platform's contract suite has not been run against Pulsar + * transactions across broker restart, client fencing, and partial-commit recovery, and the + * Experimental tier exists precisely so that an unproven guarantee is never advertised as one. A + * capability that reports {@code true} on the strength of the broker's documentation is the failure + * mode this class prevents. + */ +public final class PulsarTransactionCapability { + + private PulsarTransactionCapability() {} + + /** Whether the adapter advertises broker transactions. */ + public static final boolean PROMOTED = false; + + /** + * Reports whether transactions may be used. + * + * @return false while the adapter is Experimental + */ + public static boolean isAvailable() { + return PROMOTED; + } + + /** + * Returns the refusal for a transactional request, for the caller to throw. + * + *

Returned rather than thrown so the refusal reads as a value at the call site: {@code throw + * PulsarTransactionCapability.rejection();} makes it obvious that control leaves there, which a + * method that throws on the caller's behalf hides. + * + * @return the exception explaining why transactions are unavailable + */ + public static MessagingCapabilityUnavailableException rejection() { + return new MessagingCapabilityUnavailableException( + "PULSAR_TRANSACTION_NOT_PROMOTED", + "Pulsar supports transactions but this adapter has not certified them against the " + + "platform contract suite, so the capability is not advertised"); + } +} diff --git a/src/messaging/messaging-pulsar-experimental/src/test/java/dev/caskeleton/messaging/pulsar/PulsarAdapterContractTest.java b/src/messaging/messaging-pulsar-experimental/src/test/java/dev/caskeleton/messaging/pulsar/PulsarAdapterContractTest.java new file mode 100644 index 00000000..3968a6ee --- /dev/null +++ b/src/messaging/messaging-pulsar-experimental/src/test/java/dev/caskeleton/messaging/pulsar/PulsarAdapterContractTest.java @@ -0,0 +1,234 @@ +package dev.caskeleton.messaging.pulsar; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.delivery.ExternalSideEffectGuarantee; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.DestinationKind; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessagingCapabilityUnavailableException; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import dev.caskeleton.messaging.policy.CapabilityTier; +import dev.caskeleton.messaging.policy.ConsumerPolicy; +import dev.caskeleton.messaging.policy.DeadLetterPolicy; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.PayloadPolicy; +import dev.caskeleton.messaging.policy.PhysicalDestination; +import dev.caskeleton.messaging.policy.ProducerPolicy; +import dev.caskeleton.messaging.policy.RetryPolicy; +import dev.caskeleton.messaging.policy.SchemaPolicy; +import dev.caskeleton.messaging.schema.EncodedMessage; +import dev.caskeleton.messaging.schema.SchemaCompatibility; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import java.time.Instant; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.TimeoutException; +import org.junit.jupiter.api.Test; + +class PulsarAdapterContractTest { + + private static final String TOPIC = "persistent://public/default/orders"; + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + private static final PulsarMessagePosition POSITION = new PulsarMessagePosition(5, 9, 0, -1); + + private static PulsarMessagingTransport transport( + PulsarMessagingTransport.PulsarSendOperation send, PulsarSubscriptionMode mode) { + PulsarProfile profile = + new PulsarProfile( + TOPIC, "orders-sub", mode, java.time.Duration.ofSeconds(30), Optional.empty(), 5); + return new PulsarMessagingTransport("pulsar-primary", 1, profile, send); + } + + private static PulsarMessagingTransport confirming() { + return transport( + (topic, request) -> CompletableFuture.completedFuture(POSITION), + PulsarSubscriptionMode.shared()); + } + + private static PulsarMessagingTransport failingWith(Throwable failure) { + return transport( + (topic, request) -> CompletableFuture.failedFuture(failure), + PulsarSubscriptionMode.shared()); + } + + private static PublishResult await(CompletionStage stage) { + Object value = stage.toCompletableFuture().join(); + return ((dev.caskeleton.messaging.transport.TransportPublishResult) value).result(); + } + + @Test + void aSuccessfulSendReportsReplicationEvidenceNotAMereReceipt() { + PublishResult result = await(confirming().publish(request(64))); + + assertThat(result.completion()).isEqualTo(PublishCompletion.CONFIRMED); + assertThat(result.evidence().confirmationLevel()) + .as("the send future resolves only once the entry is persisted to the bookies") + .isEqualTo(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK); + assertThat(result.routingOutcome()).isEqualTo(RoutingOutcome.ROUTED); + } + + @Test + void theBrokerPositionIsCarriedBackForDiagnostics() { + assertThat(await(confirming().publish(request(64))).position()).hasValue(POSITION); + } + + @Test + void aSendTimeoutIsAmbiguousRatherThanRejected() { + PublishResult result = + await(failingWith(new TimeoutException("send timed out")).publish(request(64))); + + assertThat(result.completion()) + .as("the entry may be on the bookies with only the acknowledgement lost") + .isEqualTo(PublishCompletion.AMBIGUOUS); + assertThat(result.evidence().transmission()) + .isEqualTo(TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED); + assertThat(result.evidence().brokerAccepted()).isFalse(); + } + + @Test + void anExplicitBrokerRejectionIsNotAmbiguous() { + PublishResult result = + await(failingWith(new IllegalStateException("producer is fenced")).publish(request(64))); + + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.evidence().transmission()).isEqualTo(TransmissionEvidence.NOT_TRANSMITTED); + } + + @Test + void anOversizedPayloadIsRejectedBeforeItReachesTheBroker() { + PublishResult result = + await(confirming().publish(request(PayloadPolicy.DEFAULT_MAX_BYTES + 1))); + + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.evidence().transmission()).isEqualTo(TransmissionEvidence.NOT_TRANSMITTED); + assertThat(result.failure()) + .hasValueSatisfying(failure -> assertThat(failure.code()).isEqualTo("PAYLOAD_TOO_LARGE")); + } + + @Test + void aClosedTransportStopsAcceptingNewWork() { + PulsarMessagingTransport transport = confirming(); + transport.close(); + + assertThat(transport.isAcceptingWork()).isFalse(); + assertThat(await(transport.publish(request(64))).completion()) + .isEqualTo(PublishCompletion.REJECTED); + } + + @Test + void aTransportWithoutAConsumerFactoryRefusesToRegisterRatherThanReturningNothing() { + assertThatThrownBy(() -> confirming().register(null)).isInstanceOf(NullPointerException.class); + } + + @Test + void aSharedSubscriptionAdvertisesNoKeyedOrdering() { + var capabilities = + transport( + (topic, request) -> CompletableFuture.completedFuture(POSITION), + PulsarSubscriptionMode.shared()) + .capabilities(new DestinationName("orders.v1")) + .capabilities(); + + assertThat(capabilities.keyedOrdering()).isFalse(); + } + + @Test + void aKeySharedSubscriptionAdvertisesKeyedOrdering() { + var capabilities = + transport( + (topic, request) -> CompletableFuture.completedFuture(POSITION), + PulsarSubscriptionMode.keyShared()) + .capabilities(new DestinationName("orders.v1")) + .capabilities(); + + assertThat(capabilities.keyedOrdering()).isTrue(); + } + + @Test + void noSubscriptionModeAdvertisesBrokerTransactions() { + assertThat( + transport( + (topic, request) -> CompletableFuture.completedFuture(POSITION), + PulsarSubscriptionMode.keyShared()) + .capabilities(new DestinationName("orders.v1")) + .capabilities() + .brokerTransaction()) + .as("Pulsar has transactions; this adapter has not certified them") + .isFalse(); + } + + @Test + void requestingATransactionFailsWithAnExplanation() { + assertThat(PulsarTransactionCapability.rejection()) + .isInstanceOf(MessagingCapabilityUnavailableException.class) + .hasMessageContaining("not advertised"); + } + + private static TransportPublishRequest request(int payloadBytes) { + return new TransportPublishRequest( + profile(), envelope(payloadBytes), PublishOptions.defaults()); + } + + private static DestinationProfile profile() { + return new DestinationProfile( + new DestinationName("orders.v1"), + "pulsar-primary", + DestinationKind.EVENT_STREAM, + PhysicalDestination.pulsarTopic(TOPIC, "orders-sub"), + new SchemaPolicy( + ContentType.JSON, + SchemaCompatibility.BACKWARD, + Set.of(new MessageType("order.created"))), + DeliveryGuarantee.AT_LEAST_ONCE, + OrderingScope.NONE, + ExternalSideEffectGuarantee.IDEMPOTENCY_REQUIRED, + ProducerPolicy.defaults(), + ConsumerPolicy.defaults("orders"), + RetryPolicy.none(), + DeadLetterPolicy.to(new DestinationName("orders.v1.dlq")), + PayloadPolicy.defaults(), + CapabilityTier.M1, + false, + false, + false); + } + + private static MessageEnvelope envelope(int payloadBytes) { + byte[] payload = new byte[payloadBytes]; + java.util.Arrays.fill(payload, (byte) 'x'); + return new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + NOW, + Optional.of(NOW), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage(payload, ContentType.JSON, Optional.empty())); + } +} diff --git a/src/messaging/messaging-pulsar-experimental/src/test/java/dev/caskeleton/messaging/pulsar/PulsarSubscriptionGuardTest.java b/src/messaging/messaging-pulsar-experimental/src/test/java/dev/caskeleton/messaging/pulsar/PulsarSubscriptionGuardTest.java new file mode 100644 index 00000000..a7dea10f --- /dev/null +++ b/src/messaging/messaging-pulsar-experimental/src/test/java/dev/caskeleton/messaging/pulsar/PulsarSubscriptionGuardTest.java @@ -0,0 +1,125 @@ +package dev.caskeleton.messaging.pulsar; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.error.MessagingCapabilityUnavailableException; +import java.time.Duration; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class PulsarSubscriptionGuardTest { + + @Test + void keySharedWithCumulativeAcknowledgementIsRefused() { + assertThatThrownBy(() -> new PulsarSubscriptionMode(PulsarSubscriptionType.KEY_SHARED, true)) + .as("cumulative ack over interleaved keys discards the ordering the type provides") + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("ordering"); + } + + @Test + void sharedWithCumulativeAcknowledgementIsRefused() { + assertThatThrownBy(() -> new PulsarSubscriptionMode(PulsarSubscriptionType.SHARED, true)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void aSharedSubscriptionPromisesNoOrdering() { + assertThat(PulsarSubscriptionMode.shared().orderingScope()).isEqualTo(OrderingScope.NONE); + } + + @Test + void aKeySharedSubscriptionPromisesKeyOrdering() { + assertThat(PulsarSubscriptionMode.keyShared().orderingScope()).isEqualTo(OrderingScope.KEY); + } + + @Test + void aZeroNegativeAckDelayIsRefused() { + assertThatThrownBy( + () -> + new PulsarProfile( + "persistent://public/default/orders", + "orders-sub", + PulsarSubscriptionMode.shared(), + Duration.ZERO, + Optional.empty(), + 5)) + .as("a zero redelivery delay turns a failing handler into a spin loop") + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("spin loop"); + } + + @Test + void anAckTimeoutBelowPulsarsOwnFloorIsRefused() { + assertThatThrownBy( + () -> + new PulsarProfile( + "persistent://public/default/orders", + "orders-sub", + PulsarSubscriptionMode.shared(), + Duration.ofSeconds(30), + Optional.of(Duration.ofSeconds(1)), + 5)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void theDefaultProfileLeavesAckTimeoutOff() { + assertThat( + PulsarProfile.shared("persistent://public/default/orders", "orders-sub").ackTimeout()) + .as("an ack timeout redelivers messages from handlers that are merely slow") + .isEmpty(); + } + + @Test + void transactionsAreNotAdvertisedWhileTheAdapterIsExperimental() { + assertThat(PulsarTransactionCapability.isAvailable()).isFalse(); + assertThat(PulsarTransactionCapability.rejection()) + .isInstanceOf(MessagingCapabilityUnavailableException.class); + } + + @Test + void theRefusalExplainsThatTheGapIsProofNotBrokerSupport() { + assertThat(PulsarTransactionCapability.rejection()) + .hasMessageContaining("certified") + .hasMessageContaining("contract suite"); + } + + @Test + void aNonBatchedPositionRendersWithoutABatchIndex() { + PulsarMessagePosition position = + new PulsarMessagePosition(12, 34, 2, PulsarMessagePosition.NOT_APPLICABLE); + + assertThat(position.toMessageId()).isEqualTo("12:34:2"); + assertThat(position.broker()).isEqualTo("pulsar"); + } + + @Test + void aBatchedPositionKeepsItsIndexBecauseTheIdAloneIsShared() { + assertThat(new PulsarMessagePosition(12, 34, 2, 7).toMessageId()).isEqualTo("12:34:2:7"); + } + + @Test + void thePositionIsReadableAsDiagnosticAttributes() { + assertThat(new PulsarMessagePosition(12, 34, 2, 7).diagnosticAttributes()) + .containsEntry("ledgerId", "12") + .containsEntry("batchIndex", "7"); + } + + @Test + void theValidatorAcceptsAKeyedProfileOnKeyShared() { + assertThatCode( + () -> + new PulsarProfile( + "persistent://public/default/orders", + "orders-sub", + PulsarSubscriptionMode.keyShared(), + Duration.ofSeconds(30), + Optional.empty(), + 5)) + .doesNotThrowAnyException(); + } +} diff --git a/src/messaging/messaging-rabbit/build.gradle b/src/messaging/messaging-rabbit/build.gradle new file mode 100644 index 00000000..21685a3d --- /dev/null +++ b/src/messaging/messaging-rabbit/build.gradle @@ -0,0 +1,23 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-schema-api') + api project(':messaging:messaging-policy') + api project(':messaging:messaging-transport-spi') + api project(':messaging:messaging-observability') + api project(':messaging:messaging-security') + api project(':messaging:messaging-admin-api') + + implementation 'org.springframework.amqp:spring-rabbit' + implementation 'org.springframework:spring-context' + + // Test-only. The same adapter contract Kafka runs, so a guarantee can only be weakened by + // editing the contract rather than by an adapter quietly not implementing it. + testImplementation project(':messaging:messaging-testkit') + + // Live-broker certification. The confirm-and-return ordering the adapter depends on is a + // property of AMQP, and only a real broker exhibits it. + testImplementation 'org.testcontainers:testcontainers-rabbitmq' + testImplementation 'org.testcontainers:testcontainers-junit-jupiter' +} diff --git a/src/messaging/messaging-rabbit/gradle.lockfile b/src/messaging/messaging-rabbit/gradle.lockfile new file mode 100644 index 00000000..95c6b3d9 --- /dev/null +++ b/src/messaging/messaging-rabbit/gradle.lockfile @@ -0,0 +1,127 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.fasterxml.jackson.core:jackson-annotations:2.20=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.github.docker-java:docker-java-api:3.7.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport-zerodep:3.7.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport:3.7.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,jmhAnnotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,jmhAnnotationProcessor,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,jmhAnnotationProcessor,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +com.rabbitmq:amqp-client:5.27.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-codec:commons-codec:1.19.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.20.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.21.0=spotbugs +commons-logging:commons-logging:1.3.5=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +io.micrometer:micrometer-commons:1.16.0=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.micrometer:micrometer-core:1.16.0=jmhRuntimeClasspath,runtimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-observation:1.16.0=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-buffer:4.2.17.Final=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-base:4.2.17.Final=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-compression:4.2.17.Final=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-marshalling:4.2.17.Final=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-protobuf:4.2.17.Final=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec:4.2.17.Final=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-common:4.2.17.Final=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-handler:4.2.17.Final=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-resolver:4.2.17.Final=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-transport-native-unix-common:4.2.17.Final=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-transport:4.2.17.Final=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +javax.inject:javax.inject:1=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna:5.18.1=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +net.sf.jopt-simple:jopt-simple:5.0.4=jmhAnnotationProcessor,jmhCompileClasspath,jmhRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-compress:1.28.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-lang3:3.20.0=checkstyle,jmhCompileClasspath,jmhRuntimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-math3:3.6.1=jmhAnnotationProcessor,jmhCompileClasspath,jmhRuntimeClasspath +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=jmhCompileClasspath,testCompileClasspath +org.assertj:assertj-core:3.27.6=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.hdrhistogram:HdrHistogram:2.2.2=jmhRuntimeClasspath,runtimeClasspath,testRuntimeClasspath +org.javassist:javassist:3.28.0-GA=checkstyle +org.jetbrains:annotations:17.0.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,compileClasspath,jmhAnnotationProcessor,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=jmhRuntimeClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=jmhRuntimeClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=jmhRuntimeClasspath,testRuntimeClasspath +org.junit:junit-bom:6.0.1=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.latencyutils:LatencyUtils:2.0.3=jmhRuntimeClasspath,runtimeClasspath,testRuntimeClasspath +org.mockito:mockito-core:5.20.0=mockitoAgent +org.openjdk.jmh:jmh-core:1.37=jmhAnnotationProcessor,jmhCompileClasspath,jmhRuntimeClasspath +org.openjdk.jmh:jmh-generator-annprocess:1.37=jmhAnnotationProcessor +org.opentest4j:opentest4j:1.3.0=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.rnorth.duct-tape:duct-tape:1.0.8=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:2.0.17=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,spotbugs,spotbugsSlf4j,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.springframework.amqp:spring-amqp:4.0.0=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework.amqp:spring-rabbit:4.0.0=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-aop:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-beans:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-context:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-core:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-expression:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-messaging:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-tx:7.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-junit-jupiter:2.0.2=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-rabbitmq:2.0.2=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers:2.0.2=jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty= diff --git a/src/messaging/messaging-rabbit/src/jmh/java/dev/caskeleton/messaging/rabbit/RabbitPublishBenchmark.java b/src/messaging/messaging-rabbit/src/jmh/java/dev/caskeleton/messaging/rabbit/RabbitPublishBenchmark.java new file mode 100644 index 00000000..e8e2d460 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/jmh/java/dev/caskeleton/messaging/rabbit/RabbitPublishBenchmark.java @@ -0,0 +1,156 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.destination.ConfirmationRequirement; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measures the RabbitMQ adapter's own cost per publish, with no broker in the loop. + * + *

The confirm coordinator is the interesting one here. Every publish registers a pending entry + * and every confirm resolves it, so its map operations run twice per message on the hot path — and + * because a confirm arrives on the connection thread while publishes arrive on caller threads, a + * regression there shows up as contention rather than as a slower method. + * + *

No channel and no network, for the same reason the Kafka benchmark has no producer: measuring + * the broker measures the environment. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +@Fork(1) +@State(Scope.Benchmark) +public class RabbitPublishBenchmark { + + /** Payload sizes spanning the range a portable destination allows. */ + @Param({"256", "4096", "65536"}) + public int payloadBytes; + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + private static final String EXCHANGE = "notification.work"; + private static final String ROUTING_KEY = "email"; + + private RabbitHeaderMapper headerMapper; + private RabbitPublishFailureClassifier classifier; + private MessageEnvelope envelope; + private RabbitConfirmCoordinator coordinator; + private long sequence; + + /** Builds the fixtures once so the measured methods allocate only what a publish allocates. */ + @Setup(Level.Trial) + public void setUp() { + headerMapper = new RabbitHeaderMapper(); + classifier = new RabbitPublishFailureClassifier(); + byte[] payload = new byte[payloadBytes]; + java.util.Arrays.fill(payload, (byte) 'x'); + envelope = + new MessageEnvelope<>( + MessageId.newId(), + new MessageType("email.requested"), + new SchemaVersion(1), + NOW, + Optional.of(NOW), + new ProducerId("notification-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage(payload, ContentType.JSON, Optional.empty())); + } + + /** Resets the coordinator per iteration so pending entries do not accumulate across runs. */ + @Setup(Level.Iteration) + public void resetCoordinator() { + coordinator = new RabbitConfirmCoordinator(); + sequence = 0; + } + + /** + * Measures mapping an envelope's reserved headers onto AMQP properties. + * + * @param blackhole consumes the result + */ + @Benchmark + public void mapHeaders(Blackhole blackhole) { + blackhole.consume(headerMapper.toProperties(envelope)); + } + + /** + * Measures one full register-then-confirm round trip through the coordinator. + * + *

Both halves together, because a benchmark of the registration alone would leave the pending + * map growing and report a cost that no real publish pays. + * + * @param blackhole consumes the result + */ + @Benchmark + public void registerAndConfirm(Blackhole blackhole) { + long current = ++sequence; + blackhole.consume( + coordinator.awaiting( + current, + EXCHANGE, + ROUTING_KEY, + ConfirmationRequirement.REPLICATION_OR_PERSISTENCE_ACK)); + coordinator.confirmed(current, true, Duration.ofMillis(1)); + } + + /** + * Measures the unroutable path, which resolves through a return before the confirm. + * + * @param blackhole consumes the result + */ + @Benchmark + public void registerReturnAndConfirm(Blackhole blackhole) { + long current = ++sequence; + blackhole.consume( + coordinator.awaiting( + current, + EXCHANGE, + ROUTING_KEY, + ConfirmationRequirement.REPLICATION_OR_PERSISTENCE_ACK)); + coordinator.returned(current); + coordinator.confirmed(current, true, Duration.ofMillis(1)); + } + + /** + * Measures classifying a lost confirm, which runs on every ambiguous publish. + * + * @param blackhole consumes the result + */ + @Benchmark + public void classifyLostConfirm(Blackhole blackhole) { + blackhole.consume( + classifier.classify( + new java.io.IOException("connection reset"), true, Duration.ofMillis(5))); + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitBatchConsumerRegistrar.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitBatchConsumerRegistrar.java new file mode 100644 index 00000000..68ab87a3 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitBatchConsumerRegistrar.java @@ -0,0 +1,165 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.delivery.BatchDeliveryMetadata; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Accumulates AMQP deliveries into batches, bounded by size and by age. + * + *

Unlike Kafka, RabbitMQ delivers one message at a time, so a batch here is built by the client + * rather than returned by a poll. That makes the age bound essential: without it, a queue that goes + * quiet leaves the last few messages sitting in the accumulator indefinitely, unacknowledged, until + * enough traffic arrives to fill a batch — which on a low-volume queue may be hours. + * + *

{@code settlableAsBatch} is reported false. AMQP's multiple-ack acknowledges every delivery up + * to a tag, which on a channel handling concurrent work settles messages the handler has not + * finished; the platform therefore acknowledges each delivery individually even when the handler + * saw them as a batch. + * + *

The accumulator is not thread-safe by design. It is driven from the single consumer callback + * thread for its channel, and adding synchronisation here would suggest it can be shared, which + * would interleave deliveries from different channels into one batch and lose their ordering. + */ +public final class RabbitBatchConsumerRegistrar { + + private final DestinationProfile profile; + private final int maxBatchSize; + private final Duration maxBatchAge; + private final List deliveryTags = new ArrayList<>(); + private Instant oldestArrival; + + /** + * Creates a batch accumulator for a destination. + * + * @param profile the validated destination profile + * @param maxBatchSize the largest batch handed to a handler + * @param maxBatchAge how long the oldest delivery may wait before the batch is released + */ + public RabbitBatchConsumerRegistrar( + DestinationProfile profile, int maxBatchSize, Duration maxBatchAge) { + this.profile = Objects.requireNonNull(profile, "profile must not be null"); + Objects.requireNonNull(maxBatchAge, "maxBatchAge must not be null"); + if (maxBatchSize < 1) { + throw new IllegalArgumentException("maxBatchSize must be at least 1"); + } + if (maxBatchAge.isNegative() || maxBatchAge.isZero()) { + throw new IllegalArgumentException( + "maxBatchAge must be positive; without it a quiet queue holds its last messages " + + "unacknowledged indefinitely"); + } + this.maxBatchSize = maxBatchSize; + this.maxBatchAge = maxBatchAge; + } + + /** + * Adds one delivery to the current batch. + * + * @param deliveryTag the AMQP delivery tag + * @param now the current instant + * @return the batch when it is ready to release, otherwise empty + */ + public Optional add(long deliveryTag, Instant now) { + Objects.requireNonNull(now, "now must not be null"); + if (deliveryTags.isEmpty()) { + oldestArrival = now; + } + deliveryTags.add(deliveryTag); + return deliveryTags.size() >= maxBatchSize ? Optional.of(release(now)) : Optional.empty(); + } + + /** + * Releases the current batch if its oldest delivery has waited long enough. + * + * @param now the current instant + * @return the batch when the age bound was reached, otherwise empty + */ + public Optional releaseIfStale(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + if (deliveryTags.isEmpty()) { + return Optional.empty(); + } + return !now.isBefore(oldestArrival.plus(maxBatchAge)) + ? Optional.of(release(now)) + : Optional.empty(); + } + + /** + * Releases whatever has accumulated, for shutdown. + * + * @param now the current instant + * @return the batch when anything was pending + */ + public Optional drain(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + return deliveryTags.isEmpty() ? Optional.empty() : Optional.of(release(now)); + } + + /** + * Returns how many deliveries are waiting in the accumulator. + * + * @return the pending delivery count + */ + public int pending() { + return deliveryTags.size(); + } + + /** + * Refuses batch consumption where the prefetch cannot hold a whole batch. + * + *

A prefetch below the batch size deadlocks: the accumulator waits for more deliveries the + * broker will not send until something is acknowledged, and nothing is acknowledged until the + * batch is released. + * + * @param prefetch the channel prefetch count + * @throws MessagingConfigurationException when the prefetch is too small + */ + public void requirePrefetchFor(int prefetch) { + if (prefetch < maxBatchSize) { + throw new MessagingConfigurationException( + "PREFETCH_BELOW_BATCH_SIZE", + "destination %s batches %d deliveries but the channel prefetch is %d; the accumulator " + .formatted(profile.name().value(), maxBatchSize, prefetch) + + "would wait for deliveries the broker will not send until something is acked"); + } + } + + private AmqpBatch release(Instant now) { + List released = List.copyOf(deliveryTags); + deliveryTags.clear(); + oldestArrival = null; + return new AmqpBatch( + released, + new BatchDeliveryMetadata( + profile.name(), + released.size(), + // One queue is one ordering unit, so an accumulated batch never spans two. + Optional.of(profile.physical().queue().orElse(profile.name().value())), + // AMQP multiple-ack settles everything up to a tag, including work still in flight. + false, + now)); + } + + /** + * One accumulated batch of AMQP deliveries. + * + * @param deliveryTags the delivery tags, in arrival order + * @param metadata the batch-wide metadata handed to the handler + */ + public record AmqpBatch(List deliveryTags, BatchDeliveryMetadata metadata) { + + public AmqpBatch { + Objects.requireNonNull(metadata, "metadata must not be null"); + deliveryTags = List.copyOf(deliveryTags); + if (deliveryTags.isEmpty()) { + throw new IllegalArgumentException("a released batch is never empty"); + } + } + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitBrokerProfile.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitBrokerProfile.java new file mode 100644 index 00000000..e53d3774 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitBrokerProfile.java @@ -0,0 +1,50 @@ +package dev.caskeleton.messaging.rabbit; + +import java.time.Duration; +import java.util.List; +import java.util.Objects; + +/** + * The RabbitMQ client settings the platform is willing to run with. + * + * @param broker the logical broker name + * @param stable whether this profile claims the Stable guarantees + * @param production whether this profile is used in production + * @param addresses the broker addresses + * @param publisherConfirms whether correlated publisher confirms are on + * @param publisherReturns whether unroutable returns are delivered + * @param mandatory whether publishes are sent with the mandatory flag + * @param confirmTimeout how long to wait for a confirm + * @param autoAck whether the consumer acknowledges on delivery + * @param prefetch the consumer prefetch window + * @param quorumQueues whether durable work queues are declared as quorum queues + * @param tlsEnabled whether transport encryption is on + * @param authenticationEnabled whether broker authentication is on + */ +public record RabbitBrokerProfile( + String broker, + boolean stable, + boolean production, + List addresses, + boolean publisherConfirms, + boolean publisherReturns, + boolean mandatory, + Duration confirmTimeout, + boolean autoAck, + int prefetch, + boolean quorumQueues, + boolean tlsEnabled, + boolean authenticationEnabled) { + + public RabbitBrokerProfile { + Objects.requireNonNull(addresses, "addresses must not be null"); + Objects.requireNonNull(confirmTimeout, "confirmTimeout must not be null"); + if (broker == null || broker.isBlank()) { + throw new IllegalArgumentException("broker must not be blank"); + } + if (addresses.isEmpty()) { + throw new IllegalArgumentException("addresses must not be empty"); + } + addresses = List.copyOf(addresses); + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitChannelPublisher.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitChannelPublisher.java new file mode 100644 index 00000000..dd7fead6 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitChannelPublisher.java @@ -0,0 +1,33 @@ +package dev.caskeleton.messaging.rabbit; + +import org.springframework.amqp.core.Message; + +/** + * The channel operations the transport needs, split so the confirm cannot outrun its registration. + * + *

Reserving the sequence number is a separate call from publishing on purpose. A confirm can + * arrive on the connection thread before {@code basicPublish} has even returned, so a transport + * that registered its pending publish afterwards would drop that confirm and wait forever for one + * that already came. Reserving first closes that race. + */ +public interface RabbitChannelPublisher { + + /** + * Reserves the next publish sequence number without publishing. + * + * @return the sequence number the next publish will carry + */ + long nextPublishSequence(); + + /** + * Publishes a message under a previously reserved sequence number. + * + * @param sequence the reserved sequence number + * @param exchange the exchange to publish to + * @param routingKey the routing key + * @param message the AMQP message + * @param mandatory whether the broker must return an unroutable message + */ + void publish( + long sequence, String exchange, String routingKey, Message message, boolean mandatory); +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitConfirmCoordinator.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitConfirmCoordinator.java new file mode 100644 index 00000000..80585f88 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitConfirmCoordinator.java @@ -0,0 +1,190 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.destination.ConfirmationRequirement; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import java.time.Duration; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Correlates a RabbitMQ publish with its confirm and its possible return. + * + *

This is the piece that makes the Rabbit adapter honest. AMQP delivers a return before + * the confirm for an unroutable message, so a naive adapter that completes on the confirm reports + * success for a message the broker threw away. The coordinator therefore keeps each publish pending + * until the confirm arrives, and remembers whether a return was seen first. + * + *

The resulting outcomes are: confirmed and not returned is {@code CONFIRMED}; confirmed and + * returned is {@code REJECTED} with {@code UNROUTABLE}; nacked is {@code REJECTED}; and a confirm + * that never arrives is {@code AMBIGUOUS}, because the broker may still have stored it. + * + *

Implemented as a pure state machine over sequence numbers so the whole ordering, including the + * return-before-confirm case, is testable without a broker. + */ +public final class RabbitConfirmCoordinator { + + private final Map pending = new ConcurrentHashMap<>(); + + /** + * Registers a publish awaiting its confirm. + * + * @param sequence the channel publish sequence number + * @param exchange the exchange published to + * @param routingKey the routing key used + * @param requirement the confirmation the profile demanded + * @return a stage completing when the confirm, return, or timeout resolves it + */ + public CompletionStage awaiting( + long sequence, String exchange, String routingKey, ConfirmationRequirement requirement) { + Objects.requireNonNull(exchange, "exchange must not be null"); + Objects.requireNonNull(routingKey, "routingKey must not be null"); + Objects.requireNonNull(requirement, "requirement must not be null"); + + PendingPublish publish = new PendingPublish(exchange, routingKey, requirement); + pending.put(sequence, publish); + return publish.completion; + } + + /** + * Records that the broker returned a message as unroutable. + * + *

The return alone does not complete the publish: the confirm still follows, and only the pair + * establishes that the broker both accepted and discarded it. + * + * @param sequence the publish sequence number + */ + public void returned(long sequence) { + PendingPublish publish = pending.get(sequence); + if (publish != null) { + publish.returned = true; + } + } + + /** + * Records a broker confirm. + * + * @param sequence the publish sequence number + * @param acknowledged true for an ack, false for a nack + * @param elapsed how long the publish took + */ + public void confirmed(long sequence, boolean acknowledged, Duration elapsed) { + PendingPublish publish = pending.remove(sequence); + if (publish == null) { + return; + } + publish.completion.complete(publish.resolve(sequence, acknowledged, elapsed)); + } + + /** + * Records that the confirm never arrived within the deadline. + * + * @param sequence the publish sequence number + * @param elapsed how long was waited + */ + public void timedOut(long sequence, Duration elapsed) { + PendingPublish publish = pending.remove(sequence); + if (publish == null) { + return; + } + publish.completion.complete( + new PublishResult( + PublishCompletion.AMBIGUOUS, + PublishEvidence.ambiguous(), + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + elapsed, + Optional.of( + FailureDescriptor.of( + FailureCategory.AMBIGUOUS, + "RABBIT_CONFIRM_TIMEOUT", + "no publisher confirm arrived before the deadline")))); + } + + /** + * Returns how many publishes are awaiting a confirm. + * + * @return the pending count + */ + public int pendingCount() { + return pending.size(); + } + + /** One publish waiting for its confirm, and whether a return arrived first. */ + private static final class PendingPublish { + + private final String exchange; + private final String routingKey; + private final ConfirmationRequirement requirement; + private final CompletableFuture completion = new CompletableFuture<>(); + + private volatile boolean returned; + + private PendingPublish( + String exchange, String routingKey, ConfirmationRequirement requirement) { + this.exchange = exchange; + this.routingKey = routingKey; + this.requirement = requirement; + } + + private PublishResult resolve(long sequence, boolean acknowledged, Duration elapsed) { + RabbitPublishReference position = new RabbitPublishReference(exchange, routingKey, sequence); + + if (!acknowledged) { + return new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.UNKNOWN, + Optional.of(position), + 1, + elapsed, + Optional.of( + FailureDescriptor.of( + FailureCategory.PERMANENT_BUSINESS, + "RABBIT_NACK", + "the broker negatively acknowledged the publish"))); + } + if (returned) { + return new PublishResult( + PublishCompletion.REJECTED, + new PublishEvidence( + true, + dev.caskeleton.messaging.api.publish.TransmissionEvidence.TRANSMITTED, + false, + ConfirmationLevel.NONE), + RoutingOutcome.UNROUTABLE, + Optional.of(position), + 1, + elapsed, + Optional.of( + FailureDescriptor.of( + FailureCategory.PERMANENT_BUSINESS, + "RABBIT_UNROUTABLE", + "the exchange accepted the publish but no queue was bound"))); + } + + ConfirmationLevel level = + requirement == ConfirmationRequirement.REPLICATION_OR_PERSISTENCE_ACK + ? ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK + : ConfirmationLevel.BROKER_ACK; + return new PublishResult( + PublishCompletion.CONFIRMED, + PublishEvidence.confirmed(level), + RoutingOutcome.ROUTED, + Optional.of(position), + 1, + elapsed, + Optional.empty()); + } + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitConsumerRegistrar.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitConsumerRegistrar.java new file mode 100644 index 00000000..1095928d --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitConsumerRegistrar.java @@ -0,0 +1,198 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.settlement.SettlementCompletion; +import dev.caskeleton.messaging.api.settlement.SettlementEvidence; +import dev.caskeleton.messaging.api.settlement.SettlementResult; +import dev.caskeleton.messaging.transport.GracefulShutdownCoordinator; +import dev.caskeleton.messaging.transport.TransportConsumerRegistration; +import dev.caskeleton.messaging.transport.TransportConsumerSpec; +import dev.caskeleton.messaging.transport.TransportDelivery; +import dev.caskeleton.messaging.transport.TransportSettlement; +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import org.springframework.amqp.core.Message; + +/** + * The RabbitMQ consumer runtime. + * + *

Deliveries arrive from the listener container and are handed to the platform with a settlement + * handle that is bound to one delivery tag and refuses a second terminal call. Duplicating a {@code + * basic.ack} on a reused tag acknowledges a different message, which shows up as + * unexplained loss under load, far from its cause. + * + *

Consumption stops the moment a drain begins, but deliveries already in flight are allowed to + * finish. Rejecting an in-flight delivery at shutdown would either lose it or duplicate its side + * effect, depending on where the handler had got to. + */ +public final class RabbitConsumerRegistrar implements TransportConsumerRegistration { + + private final TransportConsumerSpec spec; + private final RabbitDeliveryMapper deliveryMapper; + private final RabbitSettlementOperations operations; + private final GracefulShutdownCoordinator shutdown; + private final Set pausedScopes = ConcurrentHashMap.newKeySet(); + private final AtomicBoolean active = new AtomicBoolean(true); + + /** + * Creates a consumer runtime. + * + * @param spec what to consume and where to deliver it + * @param operations the channel operations a settlement drives + * @param shutdown the drain coordinator + */ + public RabbitConsumerRegistrar( + TransportConsumerSpec spec, + RabbitSettlementOperations operations, + GracefulShutdownCoordinator shutdown) { + this(spec, operations, shutdown, new RabbitDeliveryMapper()); + } + + /** + * Creates a consumer runtime with an explicit delivery mapper. + * + * @param spec what to consume and where to deliver it + * @param operations the channel operations a settlement drives + * @param shutdown the drain coordinator + * @param deliveryMapper rebuilds envelopes from AMQP messages + */ + public RabbitConsumerRegistrar( + TransportConsumerSpec spec, + RabbitSettlementOperations operations, + GracefulShutdownCoordinator shutdown, + RabbitDeliveryMapper deliveryMapper) { + this.spec = Objects.requireNonNull(spec, "spec must not be null"); + this.operations = Objects.requireNonNull(operations, "operations must not be null"); + this.shutdown = Objects.requireNonNull(shutdown, "shutdown must not be null"); + this.deliveryMapper = Objects.requireNonNull(deliveryMapper, "deliveryMapper is required"); + } + + /** + * Handles one delivery pushed from the listener container. + * + *

Returns whether the delivery was accepted. A refusal means the runtime is draining or the + * queue is paused, and the caller must leave the message unacknowledged so the broker redelivers + * it. + * + * @param message the AMQP message + * @param now the current instant + * @return true when the delivery was dispatched + */ + public boolean onMessage(Message message, Instant now) { + Objects.requireNonNull(message, "message must not be null"); + Objects.requireNonNull(now, "now must not be null"); + + String queue = + Optional.ofNullable(message.getMessageProperties().getConsumerQueue()).orElse(""); + if (pausedScopes.contains(queue) || pausedScopes.contains("")) { + return false; + } + if (!shutdown.tryBeginWork()) { + return false; + } + + long tag = message.getMessageProperties().getDeliveryTag(); + try { + TransportDelivery delivery = + new TransportDelivery( + deliveryMapper.toEnvelope(message), + deliveryMapper.toMetadata(spec.profile().name(), message, now), + new ControllerBackedSettlement(tag)); + spec.sink().apply(delivery).toCompletableFuture().join(); + return true; + } catch (RuntimeException exception) { + // The message could not even be decoded. Park it rather than requeue: it will fail the same + // way on every redelivery, and requeueing turns that into a hot loop. + operations + .discard( + tag, + FailureDescriptor.of( + FailureCategory.DESERIALIZATION, + "RABBIT_UNDECODABLE", + "the message could not be decoded into an envelope")) + .toCompletableFuture() + .join(); + return true; + } finally { + shutdown.endWork(); + } + } + + @Override + public CompletionStage pause(String scope) { + pausedScopes.add(scope == null ? "" : scope); + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletionStage resume(String scope) { + pausedScopes.remove(scope == null ? "" : scope); + return CompletableFuture.completedFuture(null); + } + + /** + * Reports whether a scope is currently paused. + * + * @param scope the queue, or an empty string for all + * @return true while paused + */ + public boolean isPaused(String scope) { + return pausedScopes.contains(scope == null ? "" : scope); + } + + @Override + public boolean isActive() { + return active.get(); + } + + @Override + public void close() { + active.set(false); + } + + /** Adapts the transport settlement SPI onto the one-terminal-call controller. */ + private final class ControllerBackedSettlement implements TransportSettlement { + + private final RabbitSettlementController controller; + + private ControllerBackedSettlement(long deliveryTag) { + this.controller = new RabbitSettlementController(deliveryTag, operations); + } + + @Override + public CompletionStage acknowledge() { + return controller.ack(); + } + + @Override + public CompletionStage requeue(Duration delay) { + if (!shutdown.mayCreateRetryAttempt()) { + return CompletableFuture.completedFuture( + new SettlementResult( + SettlementCompletion.UNKNOWN, + SettlementEvidence.unknown(), + Optional.of( + FailureDescriptor.of( + FailureCategory.AMBIGUOUS, + "RABBIT_DRAINING", + "no retry attempt is created while the consumer is draining")))); + } + return controller.retry(delay); + } + + @Override + public CompletionStage discard() { + return controller.reject( + FailureDescriptor.of( + FailureCategory.PERMANENT_BUSINESS, "RABBIT_DISCARDED", "the message was discarded")); + } + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitDeadLetterPublisher.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitDeadLetterPublisher.java new file mode 100644 index 00000000..e534a912 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitDeadLetterPublisher.java @@ -0,0 +1,130 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.delivery.MessageDelivery; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import dev.caskeleton.messaging.policy.DeadLetterOrchestrator; +import dev.caskeleton.messaging.policy.DeadLetterResult; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.SourceSettlement; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletionStage; + +/** + * The RabbitMQ entry point to dead lettering. + * + *

Delegates to the shared orchestrator, exactly as the Kafka adapter does, even though RabbitMQ + * has native dead-lettering available. The native path is only taken where {@link + * RabbitNativeDeadLetterCapability} has verified a queue is bound to the dead-letter exchange — + * everywhere else the broker would discard the message silently while the reject still succeeded, + * and the platform's publish-then-settle ordering is what prevents that. + * + *

Which path is taken is decided here rather than by each call site, so a handler cannot + * accidentally reject a message on a destination whose dead-letter exchange goes nowhere. + */ +public final class RabbitDeadLetterPublisher { + + private final DeadLetterOrchestrator orchestrator; + private final RabbitNativeDeadLetterCapability nativeCapability; + private final NativeReject nativeReject; + + /** + * Creates a Rabbit dead letter publisher. + * + * @param orchestrator the shared dead letter orchestrator + * @param nativeCapability decides whether the broker's own dead-lettering may be trusted + * @param nativeReject performs a broker-side reject + */ + public RabbitDeadLetterPublisher( + DeadLetterOrchestrator orchestrator, + RabbitNativeDeadLetterCapability nativeCapability, + NativeReject nativeReject) { + this.orchestrator = Objects.requireNonNull(orchestrator, "orchestrator must not be null"); + this.nativeCapability = Objects.requireNonNull(nativeCapability, "nativeCapability required"); + this.nativeReject = Objects.requireNonNull(nativeReject, "nativeReject must not be null"); + } + + /** + * Dead letters one delivery, by whichever path is safe for this destination. + * + * @param profile the source destination profile + * @param delivery the failed delivery + * @param failure the sanitized failure + * @param deliveryTag the AMQP delivery tag to reject, when taking the native path + * @param settlement the source settlement callback + * @return a stage completing with the dead letter outcome + */ + public CompletionStage deadLetter( + DestinationProfile profile, + MessageDelivery delivery, + FailureDescriptor failure, + long deliveryTag, + SourceSettlement settlement) { + Objects.requireNonNull(profile, "profile must not be null"); + Objects.requireNonNull(delivery, "delivery must not be null"); + Objects.requireNonNull(failure, "failure must not be null"); + + if (nativeCapability.requiresPlatformPublish()) { + return orchestrator.deadLetter(profile, delivery, failure, settlement); + } + // The broker moves the message and settles the source in one operation. Safe only because the + // capability check confirmed a queue is bound to the dead letter exchange. + return nativeReject.reject(deliveryTag).thenApply(moved -> nativelyRouted()); + } + + /** + * Describes what a broker-side reject actually proves. + * + *

{@link ConfirmationLevel#BROKER_ACK}, not replication. RabbitMQ issues no publisher confirm + * for the internal hop to the dead-letter exchange, so the evidence available is that the broker + * accepted the reject — reporting it as a replication acknowledgement would claim a guarantee no + * part of this path provides. + * + * @return the dead letter outcome for the native path + */ + private static DeadLetterResult nativelyRouted() { + return new DeadLetterResult( + new PublishResult( + PublishCompletion.CONFIRMED, + new PublishEvidence( + true, TransmissionEvidence.TRANSMITTED, true, ConfirmationLevel.BROKER_ACK), + RoutingOutcome.ROUTED, + Optional.empty(), + 1, + Duration.ZERO, + Optional.empty()), + // The reject settles the source as part of the same operation, so there is no window in + // which the message is neither parked nor deliverable. + true); + } + + /** + * Returns why the platform is publishing rather than letting the broker route. + * + * @return the sanitized reason, empty when the native path is in use + */ + public String pathExplanation() { + return nativeCapability.reasonForPlatformPublish(); + } + + /** A broker-side reject that routes the message to the dead letter exchange. */ + @FunctionalInterface + public interface NativeReject { + + /** + * Rejects one delivery without requeueing it. + * + * @param deliveryTag the AMQP delivery tag + * @return a stage completing once the broker has taken the reject + */ + CompletionStage reject(long deliveryTag); + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitDeliveryMapper.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitDeliveryMapper.java new file mode 100644 index 00000000..6021c966 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitDeliveryMapper.java @@ -0,0 +1,185 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.CausationId; +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.CorrelationId; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TenantContext; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.delivery.DeliveryMetadata; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessageValidationException; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.core.MessageProperties; + +/** + * Rebuilds a platform envelope from an AMQP message. + * + *

The payload stays encoded so a codec failure is classified by the platform and parked, rather + * than surfacing as an AMQP listener exception the container would requeue in a hot loop. + * + *

The delivery attempt comes from the platform's own retry header rather than from AMQP's {@code + * redelivered} flag. That flag only says "this message was delivered before", not how many times, + * and it resets when a message travels through a retry queue. + */ +public final class RabbitDeliveryMapper { + + private final RabbitHeaderMapper headerMapper; + + /** Creates a mapper with the default header mapper. */ + public RabbitDeliveryMapper() { + this(new RabbitHeaderMapper()); + } + + /** + * Creates a mapper with an explicit header mapper. + * + * @param headerMapper the header mapper + */ + public RabbitDeliveryMapper(RabbitHeaderMapper headerMapper) { + this.headerMapper = Objects.requireNonNull(headerMapper, "headerMapper must not be null"); + } + + /** + * Converts an AMQP message into an envelope. + * + * @param message the AMQP message + * @return the still-encoded envelope + */ + public MessageEnvelope toEnvelope(Message message) { + Objects.requireNonNull(message, "message must not be null"); + MessageProperties properties = message.getMessageProperties(); + + ContentType contentType = + new ContentType( + Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.CONTENT_TYPE)) + .or(() -> Optional.ofNullable(properties.getContentType())) + .orElse(ContentType.JSON.value())); + + byte[] payload = message.getBody() == null ? new byte[0] : message.getBody(); + Instant producedAt = instant(properties, ReservedHeaders.PRODUCED_AT); + + return new MessageEnvelope<>( + new MessageId(uuid(properties, ReservedHeaders.MESSAGE_ID)), + new MessageType(required(properties, ReservedHeaders.MESSAGE_TYPE)), + new SchemaVersion(intValue(properties, ReservedHeaders.SCHEMA_VERSION)), + producedAt, + Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.OCCURRED_AT)) + .map(Instant::parse), + new ProducerId( + Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.PRODUCER)) + .orElse("unknown")), + Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.CORRELATION_ID)) + .map(CorrelationId::new), + Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.CAUSATION_ID)) + .map(value -> new CausationId(new MessageId(UUID.fromString(value)))), + contentType, + Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.PARTITION_KEY)), + Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.ORDERING_KEY)), + Optional.empty(), + new TraceContext( + Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.TRACEPARENT)), + Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.TRACESTATE)), + Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.BAGGAGE))), + MessageHeaders.empty(), + new EncodedMessage(payload, contentType, Optional.empty())); + } + + /** + * Builds the delivery metadata for an AMQP message. + * + * @param destination the logical destination + * @param message the AMQP message + * @param receivedAt when the consumer received it + * @return the delivery metadata + */ + public DeliveryMetadata toMetadata( + DestinationName destination, Message message, Instant receivedAt) { + MessageProperties properties = message.getMessageProperties(); + int attempt = attemptOf(properties); + + return new DeliveryMetadata( + destination, + attempt, + attempt > 1, + Optional.of( + new RabbitPublishReference( + Optional.ofNullable(properties.getReceivedExchange()).orElse(""), + Optional.ofNullable(properties.getReceivedRoutingKey()).orElse(""), + Math.max(0, properties.getDeliveryTag()))), + Optional.ofNullable(properties.getConsumerQueue()), + Optional.ofNullable(properties.getConsumerTag()), + receivedAt); + } + + /** + * Returns the attempt a message carries, counting the first delivery as one. + * + * @param properties the message properties + * @return the attempt number + */ + public int attemptOf(MessageProperties properties) { + String value = headerMapper.value(properties, ReservedHeaders.RETRY_ATTEMPT); + if (value == null) { + return 1; + } + try { + return Math.max(1, Integer.parseInt(value)); + } catch (NumberFormatException exception) { + return 1; + } + } + + private String required(MessageProperties properties, String name) { + String value = headerMapper.value(properties, name); + if (value == null || value.isBlank()) { + throw new MessageValidationException( + "RABBIT_HEADER_MISSING", "message is missing the required header " + name); + } + return value; + } + + private UUID uuid(MessageProperties properties, String name) { + try { + return UUID.fromString(required(properties, name)); + } catch (IllegalArgumentException exception) { + throw new MessageValidationException( + "RABBIT_HEADER_MALFORMED", "header " + name + " is not a UUID", exception); + } + } + + private int intValue(MessageProperties properties, String name) { + try { + return Integer.parseInt(required(properties, name)); + } catch (NumberFormatException exception) { + throw new MessageValidationException( + "RABBIT_HEADER_MALFORMED", "header " + name + " is not an integer", exception); + } + } + + private Instant instant(MessageProperties properties, String name) { + String value = headerMapper.value(properties, name); + if (value == null || value.isBlank()) { + return Optional.ofNullable(properties.getTimestamp()) + .map(java.util.Date::toInstant) + .orElse(Instant.EPOCH); + } + try { + return Instant.parse(value); + } catch (java.time.format.DateTimeParseException exception) { + throw new MessageValidationException( + "RABBIT_HEADER_MALFORMED", "header " + name + " is not an instant", exception); + } + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitHeaderMapper.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitHeaderMapper.java new file mode 100644 index 00000000..4d693335 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitHeaderMapper.java @@ -0,0 +1,127 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.header.HeaderName; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import org.springframework.amqp.core.MessageProperties; + +/** + * Maps envelope identity onto AMQP message properties and back. + * + *

Identity goes in headers, not in the payload, so a consumer can classify and park a message it + * cannot deserialize. The AMQP {@code message-id} and {@code correlation-id} properties are also + * populated, because management UIs and tracing tools read those and not the platform's headers — + * but the platform's own headers stay authoritative, since AMQP's properties have no schema version + * or producer field. + */ +public final class RabbitHeaderMapper { + + /** + * Writes an envelope's identity into AMQP message properties. + * + * @param envelope the envelope being published + * @return the message properties + */ + public MessageProperties toProperties(MessageEnvelope envelope) { + Objects.requireNonNull(envelope, "envelope must not be null"); + MessageProperties properties = new MessageProperties(); + + properties.setMessageId(envelope.messageId().value().toString()); + properties.setContentType(envelope.contentType().value()); + properties.setTimestamp(java.util.Date.from(envelope.producedAt())); + properties.setDeliveryMode(org.springframework.amqp.core.MessageDeliveryMode.PERSISTENT); + + setHeader(properties, ReservedHeaders.MESSAGE_ID, envelope.messageId().value().toString()); + setHeader(properties, ReservedHeaders.MESSAGE_TYPE, envelope.messageType().value()); + setHeader( + properties, + ReservedHeaders.SCHEMA_VERSION, + Integer.toString(envelope.schemaVersion().value())); + setHeader(properties, ReservedHeaders.PRODUCER, envelope.producer().value()); + setHeader(properties, ReservedHeaders.PRODUCED_AT, envelope.producedAt().toString()); + setHeader(properties, ReservedHeaders.CONTENT_TYPE, envelope.contentType().value()); + + envelope + .occurredAt() + .ifPresent(value -> setHeader(properties, ReservedHeaders.OCCURRED_AT, value.toString())); + envelope + .correlationId() + .ifPresent( + value -> { + properties.setCorrelationId(value.value()); + setHeader(properties, ReservedHeaders.CORRELATION_ID, value.value()); + }); + envelope + .causationId() + .ifPresent( + value -> + setHeader( + properties, ReservedHeaders.CAUSATION_ID, value.value().value().toString())); + envelope + .partitionKey() + .ifPresent(value -> setHeader(properties, ReservedHeaders.PARTITION_KEY, value)); + envelope + .orderingKey() + .ifPresent(value -> setHeader(properties, ReservedHeaders.ORDERING_KEY, value)); + envelope + .traceContext() + .traceparent() + .ifPresent(value -> setHeader(properties, ReservedHeaders.TRACEPARENT, value)); + envelope + .traceContext() + .tracestate() + .ifPresent(value -> setHeader(properties, ReservedHeaders.TRACESTATE, value)); + envelope + .traceContext() + .baggage() + .ifPresent(value -> setHeader(properties, ReservedHeaders.BAGGAGE, value)); + + envelope + .headers() + .asMap() + .forEach((name, value) -> setHeader(properties, name.value(), value.value())); + + return properties; + } + + /** + * Reads AMQP headers back into a platform header map. + * + * @param properties the message properties + * @return the platform headers, including reserved names + */ + public MessageHeaders fromProperties(MessageProperties properties) { + Objects.requireNonNull(properties, "properties must not be null"); + Map values = new LinkedHashMap<>(); + properties + .getHeaders() + .forEach( + (name, value) -> { + if (value != null) { + values.put(new HeaderName(name), new HeaderValue(String.valueOf(value))); + } + }); + return MessageHeaders.platform(values); + } + + /** + * Reads one header value. + * + * @param properties the message properties + * @param name the header name + * @return the value, or null when absent + */ + public String value(MessageProperties properties, String name) { + Object value = properties.getHeaders().get(name); + return value == null ? null : String.valueOf(value); + } + + private static void setHeader(MessageProperties properties, String name, String value) { + properties.setHeader(name, value); + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitMessagingTransport.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitMessagingTransport.java new file mode 100644 index 00000000..acb9a7f0 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitMessagingTransport.java @@ -0,0 +1,218 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.destination.DestinationCapabilities; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessagingCapabilities; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.error.MessagingCapabilityUnavailableException; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.transport.MessagingTransport; +import dev.caskeleton.messaging.transport.TransportConsumerRegistration; +import dev.caskeleton.messaging.transport.TransportConsumerSpec; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import dev.caskeleton.messaging.transport.TransportPublishResult; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; + +/** + * The Stable RabbitMQ adapter. + * + *

Publishes are always sent with {@code mandatory} set and correlated through {@link + * RabbitConfirmCoordinator}, because in AMQP a confirm and a return answer different questions. The + * confirm says the exchange accepted the message; the return says no queue was bound to receive it. + * An adapter that completed on the confirm alone would report success for a message the broker + * discarded, which is the single most damaging thing a Rabbit adapter can get wrong. + * + *

Payload size and shutdown state are checked before anything reaches the channel, so both fail + * with {@code NOT_TRANSMITTED} evidence rather than as a broker error whose ambiguity the caller + * then has to reason about. + */ +public final class RabbitMessagingTransport implements MessagingTransport { + + private static final MessagingCapabilities CAPABILITIES = + new MessagingCapabilities( + true, true, true, false, false, false, false, true, false, false, true, true); + + private final String brokerName; + private final long generation; + private final RabbitChannelPublisher channel; + private final RabbitPublishMapper mapper; + private final RabbitConfirmCoordinator coordinator; + private final Function consumerFactory; + private final AtomicBoolean closed = new AtomicBoolean(); + + /** + * Creates a publish-only transport. + * + * @param brokerName the logical broker name + * @param generation the runtime generation + * @param channel the channel publish operation + */ + public RabbitMessagingTransport( + String brokerName, long generation, RabbitChannelPublisher channel) { + this( + brokerName, + generation, + channel, + spec -> { + throw new MessagingCapabilityUnavailableException( + "RABBIT_CONSUMER_NOT_CONFIGURED", + "this Rabbit transport was created without a consumer factory"); + }); + } + + /** + * Creates a transport with a consumer factory. + * + * @param brokerName the logical broker name + * @param generation the runtime generation + * @param channel the channel publish operation + * @param consumerFactory builds a consumer registration for a spec + */ + public RabbitMessagingTransport( + String brokerName, + long generation, + RabbitChannelPublisher channel, + Function consumerFactory) { + this.brokerName = Objects.requireNonNull(brokerName, "brokerName must not be null"); + this.generation = generation; + this.channel = Objects.requireNonNull(channel, "channel must not be null"); + this.consumerFactory = Objects.requireNonNull(consumerFactory, "consumerFactory is required"); + this.mapper = new RabbitPublishMapper(); + this.coordinator = new RabbitConfirmCoordinator(); + } + + @Override + public CompletionStage publish(TransportPublishRequest request) { + Objects.requireNonNull(request, "request must not be null"); + + if (closed.get()) { + return completed( + rejectedLocally("RABBIT_TRANSPORT_CLOSED", "the transport is shutting down")); + } + int size = request.envelope().payload().size(); + int limit = request.profile().payload().maxBytes(); + if (size > limit) { + return completed( + rejectedLocally( + "PAYLOAD_TOO_LARGE", "encoded payload is " + size + " bytes, limit is " + limit)); + } + + String exchange = mapper.exchange(request.profile()); + String routingKey = mapper.routingKey(request.profile()); + + // Register before publishing. A confirm can land on the connection thread while basicPublish is + // still returning; registering afterwards would drop it and leave the caller waiting forever. + long sequence = channel.nextPublishSequence(); + CompletionStage pending = + coordinator + .awaiting(sequence, exchange, routingKey, request.profile().producer().confirmation()) + .thenApply(TransportPublishResult::new); + + channel.publish(sequence, exchange, routingKey, mapper.toMessage(request), true); + return pending; + } + + /** + * Records a broker confirm. Called from the channel's confirm callback. + * + * @param sequence the publish sequence number + * @param acknowledged true for an ack, false for a nack + * @param elapsed how long the publish took + */ + public void onConfirm(long sequence, boolean acknowledged, Duration elapsed) { + coordinator.confirmed(sequence, acknowledged, elapsed); + } + + /** + * Records an unroutable return. Called from the channel's return callback. + * + * @param sequence the publish sequence number + */ + public void onReturn(long sequence) { + coordinator.returned(sequence); + } + + /** + * Records that a confirm never arrived. + * + * @param sequence the publish sequence number + * @param elapsed how long was waited + */ + public void onConfirmTimeout(long sequence, Duration elapsed) { + coordinator.timedOut(sequence, elapsed); + } + + /** + * Returns how many publishes are awaiting a confirm. + * + * @return the pending count + */ + public int pendingConfirms() { + return coordinator.pendingCount(); + } + + @Override + public TransportConsumerRegistration register(TransportConsumerSpec spec) { + Objects.requireNonNull(spec, "spec must not be null"); + if (closed.get()) { + throw new MessagingCapabilityUnavailableException( + "RABBIT_TRANSPORT_CLOSED", "the transport is shutting down"); + } + return consumerFactory.apply(spec); + } + + @Override + public DestinationCapabilities capabilities(DestinationName destination) { + return new DestinationCapabilities(destination, brokerName, CAPABILITIES); + } + + @Override + public String brokerName() { + return brokerName; + } + + @Override + public long generation() { + return generation; + } + + @Override + public void close() { + closed.set(true); + } + + /** + * Reports whether the transport is still accepting work. + * + * @return true until close + */ + public boolean isAcceptingWork() { + return !closed.get(); + } + + private static TransportPublishResult rejectedLocally(String code, String message) { + return new TransportPublishResult( + new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of(FailureDescriptor.of(FailureCategory.PERMANENT_BUSINESS, code, message)))); + } + + private static CompletionStage completed(TransportPublishResult result) { + return CompletableFuture.completedFuture(result); + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitNativeDeadLetterCapability.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitNativeDeadLetterCapability.java new file mode 100644 index 00000000..84d5e88a --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitNativeDeadLetterCapability.java @@ -0,0 +1,70 @@ +package dev.caskeleton.messaging.rabbit; + +import java.util.Objects; + +/** + * Decides whether the broker's own dead-lettering may be used, or the platform must publish. + * + *

RabbitMQ is the one Stable broker with native dead-lettering, and using it is tempting: a + * {@code basic.reject} with {@code requeue=false} moves the message with no publish at all. The + * catch is that the move is not confirmed to the publisher. If the dead-letter exchange is + * unroutable — nobody bound a queue to it, or the binding was removed — the broker discards the + * message silently and the reject still succeeds. + * + *

So native dead-lettering is only used where the dead-letter path has been verified to have a + * bound queue. Everywhere else the platform publishes and waits for its own confirm, which is the + * same invariant every other broker gets: the source is settled only after the dead-letter + * publish is confirmed. + */ +public final class RabbitNativeDeadLetterCapability { + + private final RabbitTopologyProfile topology; + private final boolean deadLetterQueueBound; + + /** + * Creates a capability check for one destination. + * + * @param topology the declared topology + * @param deadLetterQueueBound whether a queue is verified bound to the dead letter exchange + */ + public RabbitNativeDeadLetterCapability( + RabbitTopologyProfile topology, boolean deadLetterQueueBound) { + this.topology = Objects.requireNonNull(topology, "topology must not be null"); + this.deadLetterQueueBound = deadLetterQueueBound; + } + + /** + * Reports whether a rejected message may be left to the broker. + * + * @return true only when a dead letter exchange is declared and verified bound + */ + public boolean mayUseNativeDeadLetter() { + return topology.hasNativeDeadLetter() && deadLetterQueueBound; + } + + /** + * Reports whether the platform must publish the dead letter itself. + * + * @return true when native dead-lettering cannot be trusted for this destination + */ + public boolean requiresPlatformPublish() { + return !mayUseNativeDeadLetter(); + } + + /** + * Returns why native dead-lettering is not being used, for a startup diagnostic. + * + * @return the sanitized reason, empty when native dead-lettering is in use + */ + public String reasonForPlatformPublish() { + if (mayUseNativeDeadLetter()) { + return ""; + } + if (!topology.hasNativeDeadLetter()) { + return "queue %s was declared without a dead letter exchange".formatted(topology.queue()); + } + return "no queue is bound to dead letter exchange %s, so a broker-side reject would discard " + .formatted(topology.deadLetterExchange().orElse("(none)")) + + "the message silently while still reporting success"; + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitProfileValidator.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitProfileValidator.java new file mode 100644 index 00000000..3463b5c5 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitProfileValidator.java @@ -0,0 +1,83 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.destination.DestinationKind; +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.util.Objects; + +/** + * Startup validation of the RabbitMQ Stable profile. + * + *

Confirms and returns are both required, and they answer different questions. A confirm says + * the exchange accepted the message; a return says no queue was bound to receive it. Enabling + * confirms without returns produces the worst possible outcome: a publish that reports success + * while the message was silently discarded. + * + *

Durable work queues default to quorum queues because classic mirrored queues can lose + * acknowledged messages during a partition — which is precisely the guarantee a durable work queue + * exists to provide. + */ +public final class RabbitProfileValidator { + + /** + * Validates one RabbitMQ broker profile. + * + * @param profile the profile to validate + * @throws IllegalArgumentException when the profile cannot honour its declared guarantees + */ + public void validate(RabbitBrokerProfile profile) { + Objects.requireNonNull(profile, "profile must not be null"); + + if (profile.stable() && !profile.publisherConfirms()) { + throw new IllegalArgumentException( + "stable Rabbit producer requires publisher confirms: " + profile.broker()); + } + if (profile.stable() && !profile.publisherReturns()) { + throw new IllegalArgumentException( + "stable Rabbit producer requires publisher returns: " + profile.broker()); + } + if (profile.stable() && !profile.mandatory()) { + throw new IllegalArgumentException( + "stable Rabbit producer requires mandatory routing: " + profile.broker()); + } + if (profile.autoAck()) { + throw new IllegalArgumentException( + "consumer auto ack is forbidden; the platform acknowledges after handler success: " + + profile.broker()); + } + if (profile.prefetch() < 1) { + throw new IllegalArgumentException("prefetch must be at least 1: " + profile.broker()); + } + if (profile.confirmTimeout().isNegative() || profile.confirmTimeout().isZero()) { + throw new IllegalArgumentException("confirm timeout must be positive: " + profile.broker()); + } + if (profile.production() && !profile.tlsEnabled()) { + throw new IllegalArgumentException( + "a production Rabbit connection requires TLS: " + profile.broker()); + } + if (profile.production() && !profile.authenticationEnabled()) { + throw new IllegalArgumentException( + "a production Rabbit connection requires broker authentication: " + profile.broker()); + } + } + + /** + * Validates a destination against the broker profile. + * + * @param destination the destination profile + * @param broker the broker profile + * @throws IllegalArgumentException when the destination cannot be served safely + */ + public void validateDestination(DestinationProfile destination, RabbitBrokerProfile broker) { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(broker, "broker must not be null"); + + if (destination.kind() == DestinationKind.WORK_QUEUE && !broker.quorumQueues()) { + throw new IllegalArgumentException( + "a durable Rabbit work queue must use a quorum queue: " + destination.name().value()); + } + if (destination.physical().queue().isEmpty() && destination.physical().exchange().isEmpty()) { + throw new IllegalArgumentException( + "a Rabbit destination requires an exchange or a queue: " + destination.name().value()); + } + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitPublishFailureClassifier.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitPublishFailureClassifier.java new file mode 100644 index 00000000..4fb3cea6 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitPublishFailureClassifier.java @@ -0,0 +1,117 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import java.io.IOException; +import java.time.Duration; +import java.util.Optional; + +/** + * Turns an AMQP publish failure into a completion the caller can act on. + * + *

The distinction that matters is where the failure happened. A channel-level exception raised + * before {@code basicPublish} returned means nothing was written: rejected, with no + * ambiguity. A connection that drops after the frame was written proves nothing — the + * broker may have the message and only the confirm was lost. + * + *

Unknown failures default to ambiguous. Guessing "rejected" on an unrecognised error is what + * turns one lost confirm into two of the same order, and the cost of an unnecessary ambiguous is a + * retry the inbox absorbs. + */ +public final class RabbitPublishFailureClassifier { + + /** + * Classifies a failure raised while publishing. + * + * @param error the throwable the channel reported + * @param framePossiblyWritten whether the publish frame may already have reached the broker + * @param elapsed how long the attempt took + * @return the publish outcome with its evidence + */ + public PublishResult classify(Throwable error, boolean framePossiblyWritten, Duration elapsed) { + Throwable cause = unwrap(error); + + if (!framePossiblyWritten) { + return new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + elapsed, + Optional.of( + FailureDescriptor.of( + categoryOf(cause), + "RABBIT_PUBLISH_REJECTED", + "the channel refused the publish before writing it: " + name(cause)))); + } + + return new PublishResult( + PublishCompletion.AMBIGUOUS, + new PublishEvidence( + true, TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED, false, ConfirmationLevel.NONE), + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + elapsed, + Optional.of( + FailureDescriptor.of( + FailureCategory.AMBIGUOUS, + "RABBIT_CONFIRM_LOST", + "the frame was written but no confirm arrived: " + name(cause)))); + } + + /** + * Classifies a broker nack, which is always a definitive rejection. + * + * @param elapsed how long the attempt took + * @return the publish outcome + */ + public PublishResult nacked(Duration elapsed) { + return new PublishResult( + PublishCompletion.REJECTED, + // A nack means the broker took the frame and decided against it, so the bytes were + // transmitted even though nothing was stored. + new PublishEvidence(true, TransmissionEvidence.TRANSMITTED, false, ConfirmationLevel.NONE), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + elapsed, + Optional.of( + FailureDescriptor.of( + FailureCategory.TRANSIENT_INFRASTRUCTURE, + "RABBIT_NACKED", + "the broker negatively acknowledged the publish"))); + } + + private static FailureCategory categoryOf(Throwable cause) { + if (cause instanceof IOException) { + return FailureCategory.TRANSIENT_INFRASTRUCTURE; + } + String name = name(cause); + if (name.contains("Authentication")) { + return FailureCategory.AUTHENTICATION; + } + if (name.contains("Authorization") || name.contains("AccessRefused")) { + return FailureCategory.AUTHORIZATION; + } + return FailureCategory.PERMANENT_BUSINESS; + } + + private static Throwable unwrap(Throwable error) { + if (error instanceof java.util.concurrent.CompletionException && error.getCause() != null) { + return error.getCause(); + } + return error; + } + + private static String name(Throwable cause) { + return cause == null ? "unknown" : cause.getClass().getSimpleName(); + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitPublishMapper.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitPublishMapper.java new file mode 100644 index 00000000..ec03aae1 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitPublishMapper.java @@ -0,0 +1,72 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import java.util.Objects; +import org.springframework.amqp.core.Message; + +/** + * Builds AMQP messages from platform publish requests. + * + *

The exchange and routing key come from the destination profile, never from the caller. A + * publisher that could choose its own routing key would be able to reach queues the destination's + * access policy never granted. + */ +public final class RabbitPublishMapper { + + private final RabbitHeaderMapper headerMapper; + + /** Creates a mapper with the default header mapper. */ + public RabbitPublishMapper() { + this(new RabbitHeaderMapper()); + } + + /** + * Creates a mapper with an explicit header mapper. + * + * @param headerMapper the header mapper + */ + public RabbitPublishMapper(RabbitHeaderMapper headerMapper) { + this.headerMapper = Objects.requireNonNull(headerMapper, "headerMapper must not be null"); + } + + /** + * Converts a publish request into an AMQP message. + * + * @param request the publish request + * @return the AMQP message + */ + public Message toMessage(TransportPublishRequest request) { + Objects.requireNonNull(request, "request must not be null"); + return new Message( + request.envelope().payload().bytes(), headerMapper.toProperties(request.envelope())); + } + + /** + * Returns the exchange a destination publishes to. + * + * @param profile the destination profile + * @return the exchange name + */ + public String exchange(DestinationProfile profile) { + return profile + .physical() + .exchange() + .orElseThrow( + () -> + new MessagingConfigurationException( + "RABBIT_EXCHANGE_MISSING", + "destination has no Rabbit exchange: " + profile.name().value())); + } + + /** + * Returns the routing key a destination publishes with. + * + * @param profile the destination profile + * @return the routing key + */ + public String routingKey(DestinationProfile profile) { + return profile.physical().routingKey().orElse(""); + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitPublishReference.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitPublishReference.java new file mode 100644 index 00000000..aee6d791 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitPublishReference.java @@ -0,0 +1,37 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.publish.BrokerPosition; +import java.util.Map; +import java.util.Objects; + +/** + * A RabbitMQ exchange, routing key, and publish sequence coordinate. + * + * @param exchange the exchange published to + * @param routingKey the routing key used + * @param sequence the channel publish sequence number + */ +public record RabbitPublishReference(String exchange, String routingKey, long sequence) + implements BrokerPosition { + + public RabbitPublishReference { + Objects.requireNonNull(exchange, "exchange must not be null"); + Objects.requireNonNull(routingKey, "routingKey must not be null"); + if (sequence < 0) { + throw new IllegalArgumentException("sequence must not be negative"); + } + } + + @Override + public String broker() { + return "rabbitmq"; + } + + @Override + public Map diagnosticAttributes() { + return Map.of( + "exchange", exchange, + "routingKey", routingKey, + "sequence", Long.toString(sequence)); + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitRequestReply.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitRequestReply.java new file mode 100644 index 00000000..2854b419 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitRequestReply.java @@ -0,0 +1,32 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.time.Duration; +import java.util.concurrent.CompletionStage; + +/** + * The M2 request-reply capability. + * + *

Deliberately limited. Request-reply over a broker is a synchronous call wearing an + * asynchronous costume: it holds a correlation entry and a caller for the whole round trip, and it + * fails in ways a plain publish does not — a lost reply is indistinguishable from a slow one. It is + * offered with a mandatory deadline and no automatic retry, because retrying a request whose reply + * was merely late duplicates the work on the other side. + */ +public interface RabbitRequestReply { + + /** + * Sends a request and awaits its correlated reply. + * + * @param destination the request destination + * @param request the encoded request envelope + * @param timeout how long to wait for the reply + * @return a stage completing with the reply, or failing on timeout + */ + CompletionStage> request( + MessageDestination destination, + MessageEnvelope request, + Duration timeout); +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitRetryQueueTopology.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitRetryQueueTopology.java new file mode 100644 index 00000000..ca42dfb4 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitRetryQueueTopology.java @@ -0,0 +1,57 @@ +package dev.caskeleton.messaging.rabbit; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Describes the delayed retry queue a destination uses. + * + *

RabbitMQ has no native per-message delay in the core broker, so a retry queue is a queue with + * a message TTL whose dead letter exchange points back at the work queue: the message sits until + * the TTL expires and is then re-routed. Modelling this explicitly, rather than hiding it behind a + * plugin, keeps the behaviour reviewable — including its main caveat, that TTL expiry is evaluated + * at the head of the queue, so mixed delays in one retry queue do not expire independently. + * + * @param retryQueue the queue messages wait in + * @param deadLetterExchange the exchange expired messages are routed back through + * @param deadLetterRoutingKey the routing key expired messages are re-published with + * @param messageTtl how long messages wait + */ +public record RabbitRetryQueueTopology( + String retryQueue, + String deadLetterExchange, + String deadLetterRoutingKey, + Duration messageTtl) { + + public RabbitRetryQueueTopology { + Objects.requireNonNull(messageTtl, "messageTtl must not be null"); + if (retryQueue == null || retryQueue.isBlank()) { + throw new IllegalArgumentException("retryQueue must not be blank"); + } + if (deadLetterExchange == null) { + throw new IllegalArgumentException("deadLetterExchange must not be null"); + } + if (deadLetterRoutingKey == null) { + throw new IllegalArgumentException("deadLetterRoutingKey must not be null"); + } + if (messageTtl.isNegative() || messageTtl.isZero()) { + throw new IllegalArgumentException("messageTtl must be positive"); + } + } + + /** + * Returns the queue arguments this topology requires. + * + * @return the AMQP queue arguments + */ + public Map queueArguments() { + Map arguments = new LinkedHashMap<>(); + arguments.put("x-queue-type", "quorum"); + arguments.put("x-message-ttl", messageTtl.toMillis()); + arguments.put("x-dead-letter-exchange", deadLetterExchange); + arguments.put("x-dead-letter-routing-key", deadLetterRoutingKey); + return Map.copyOf(arguments); + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitSecurityConfigurer.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitSecurityConfigurer.java new file mode 100644 index 00000000..98419163 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitSecurityConfigurer.java @@ -0,0 +1,179 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.security.BrokerCredentialProfile; +import dev.caskeleton.messaging.security.BrokerSecurityProfile; +import dev.caskeleton.messaging.security.BrokerTlsPolicy; +import dev.caskeleton.messaging.security.CredentialRuntime; +import dev.caskeleton.messaging.security.CredentialRuntimeRegistry; +import java.time.Instant; +import java.util.List; +import java.util.Objects; + +/** + * Builds the AMQP connection settings from a validated security profile. + * + *

Credentials are resolved per connection attempt rather than cached in the factory. RabbitMQ + * client connections are long-lived and reconnect on their own, so a factory holding a credential + * from startup will happily reconnect with a revoked one for as long as the process runs — the + * reconnect is exactly the moment a rotated credential should take effect. + * + *

The password is returned inside a short-lived {@link AmqpCredentials} rather than assigned to + * a connection factory field, so the caller controls when it is cleared. + */ +public final class RabbitSecurityConfigurer { + + private final CredentialRuntimeRegistry credentials; + private final BrokerTlsPolicy tlsPolicy; + + /** + * Creates a configurer. + * + * @param credentials resolves and rotates credential material + * @param tlsPolicy validates the transport security posture + */ + public RabbitSecurityConfigurer( + CredentialRuntimeRegistry credentials, BrokerTlsPolicy tlsPolicy) { + this.credentials = Objects.requireNonNull(credentials, "credentials must not be null"); + this.tlsPolicy = Objects.requireNonNull(tlsPolicy, "tlsPolicy must not be null"); + } + + /** + * Resolves the connection credentials for one role. + * + * @param profile the broker security profile + * @param credential the role credential to configure + * @param enabledProtocols the TLS protocol versions to enable + * @param now the current instant + * @return the AMQP connection credentials + */ + public AmqpCredentials configure( + BrokerSecurityProfile profile, + BrokerCredentialProfile credential, + List enabledProtocols, + Instant now) { + Objects.requireNonNull(profile, "profile must not be null"); + Objects.requireNonNull(credential, "credential must not be null"); + Objects.requireNonNull(enabledProtocols, "enabledProtocols must not be null"); + + tlsPolicy.validate(profile, enabledProtocols); + + return switch (credential) { + case BrokerCredentialProfile.UsernamePassword plain -> { + CredentialRuntime resolved = credentials.resolve(plain.credentialId(), now); + yield new AmqpCredentials( + "PLAIN", plain.credentialId(), resolved.material(), profile.tlsEnabled()); + } + case BrokerCredentialProfile.MutualTls mutual -> { + // The certificate is the credential; there is no password to carry. + credentials.resolve(mutual.credentialId(), now); + yield new AmqpCredentials("EXTERNAL", mutual.credentialId(), new char[0], true); + } + case BrokerCredentialProfile.OAuth2 oauth -> { + CredentialRuntime resolved = credentials.resolve(oauth.credentialId(), now); + // RabbitMQ's OAuth 2 plugin takes the token in the password field of a PLAIN exchange. + yield new AmqpCredentials( + "PLAIN", oauth.credentialId(), resolved.material(), profile.tlsEnabled()); + } + case BrokerCredentialProfile.SaslScram scram -> { + CredentialRuntime resolved = credentials.resolve(scram.credentialId(), now); + yield new AmqpCredentials( + "RABBIT-CR-DEMO", scram.credentialId(), resolved.material(), profile.tlsEnabled()); + } + case BrokerCredentialProfile.Nkey ignored -> + throw new IllegalArgumentException( + "NKey credentials are a NATS concept, not an AMQP one"); + }; + } + + /** + * One connection's resolved credentials. + * + *

A class rather than a record because the secret has to stay mutable to be cleared, and a + * record carrying a {@code char[]} would compare and hash it by identity — two credentials with + * the same material would be unequal, which is a surprise waiting in any future collection use. + * + *

{@link #clear()} overwrites the secret. The caller is expected to call it once the + * connection is established, so the material's lifetime is the handshake rather than the process. + */ + public static final class AmqpCredentials { + + private final String mechanism; + private final String username; + private final char[] secret; + private final boolean tls; + + /** + * Creates connection credentials. + * + * @param mechanism the SASL mechanism name + * @param username the AMQP username + * @param secret the password or token, taken by reference and cleared by {@link #clear()} + * @param tls whether the connection uses TLS + */ + public AmqpCredentials(String mechanism, String username, char[] secret, boolean tls) { + Objects.requireNonNull(secret, "secret must not be null"); + if (mechanism == null || mechanism.isBlank()) { + throw new IllegalArgumentException("mechanism must not be blank"); + } + if (username == null || username.isBlank()) { + throw new IllegalArgumentException("username must not be blank"); + } + this.mechanism = mechanism; + this.username = username; + this.secret = secret; + this.tls = tls; + } + + /** + * Returns the SASL mechanism name. + * + * @return the mechanism + */ + public String mechanism() { + return mechanism; + } + + /** + * Returns the AMQP username. + * + * @return the username + */ + public String username() { + return username; + } + + /** + * Returns a copy of the secret. + * + * @return the password or token + */ + public char[] secret() { + return secret.clone(); + } + + /** + * Reports whether the connection uses TLS. + * + * @return true when TLS is enabled + */ + public boolean tls() { + return tls; + } + + /** Overwrites the secret in place. */ + public void clear() { + java.util.Arrays.fill(secret, '\0'); + } + + /** + * Returns a representation that names the connection without revealing it. + * + * @return a safe description + */ + @Override + public String toString() { + return "AmqpCredentials[mechanism=%s, username=%s, tls=%s]" + .formatted(mechanism, username, tls); + } + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitSettlementController.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitSettlementController.java new file mode 100644 index 00000000..3f0a248f --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitSettlementController.java @@ -0,0 +1,84 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.error.MessageSettlementException; +import dev.caskeleton.messaging.api.settlement.SettlementController; +import dev.caskeleton.messaging.api.settlement.SettlementResult; +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Manual settlement for one RabbitMQ delivery. + * + *

Exactly one terminal call is allowed. A second one fails loudly rather than racing the first, + * because a duplicated {@code basic.ack} on a reused delivery tag acknowledges a different + * message — a failure mode that shows up as unexplained message loss under load, far from its + * cause. + * + *

{@code retry} does not use {@code basic.nack} with requeue. Requeueing puts the message back + * at the head of the queue with no delay, which turns a transient downstream failure into a hot + * loop; the delayed retry queue is used instead. + */ +public final class RabbitSettlementController implements SettlementController { + + private final long deliveryTag; + private final RabbitSettlementOperations operations; + private final AtomicBoolean settled = new AtomicBoolean(); + + /** + * Creates a controller for one delivery. + * + * @param deliveryTag the AMQP delivery tag + * @param operations the channel operations, invoked on the container thread + */ + public RabbitSettlementController(long deliveryTag, RabbitSettlementOperations operations) { + this.deliveryTag = deliveryTag; + this.operations = Objects.requireNonNull(operations, "operations must not be null"); + } + + @Override + public CompletionStage ack() { + return terminal(() -> operations.ack(deliveryTag)); + } + + @Override + public CompletionStage retry(Duration delay) { + Objects.requireNonNull(delay, "delay must not be null"); + return terminal(() -> operations.publishToRetryQueue(deliveryTag, delay)); + } + + @Override + public CompletionStage deadLetter(FailureDescriptor failure) { + Objects.requireNonNull(failure, "failure must not be null"); + return terminal(() -> operations.deadLetter(deliveryTag, failure)); + } + + @Override + public CompletionStage reject(FailureDescriptor failure) { + Objects.requireNonNull(failure, "failure must not be null"); + return terminal(() -> operations.discard(deliveryTag, failure)); + } + + /** + * Reports whether this delivery has already been settled. + * + * @return true once a terminal call has been made + */ + public boolean isSettled() { + return settled.get(); + } + + private CompletionStage terminal( + java.util.function.Supplier> operation) { + if (!settled.compareAndSet(false, true)) { + return CompletableFuture.failedFuture( + new MessageSettlementException( + "SETTLEMENT_ALREADY_COMPLETED", + "delivery tag " + deliveryTag + " has already been settled")); + } + return operation.get(); + } +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitSettlementOperations.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitSettlementOperations.java new file mode 100644 index 00000000..6f2c9752 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitSettlementOperations.java @@ -0,0 +1,50 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.settlement.SettlementResult; +import java.time.Duration; +import java.util.concurrent.CompletionStage; + +/** + * The channel-side operations a settlement controller may perform. + * + *

Separated from the controller so the one-terminal-call guard is testable without an AMQP + * channel, and so the controller cannot reach anything the platform has not exposed. + */ +public interface RabbitSettlementOperations { + + /** + * Acknowledges a delivery. + * + * @param deliveryTag the AMQP delivery tag + * @return a stage completing with the settlement outcome + */ + CompletionStage ack(long deliveryTag); + + /** + * Publishes to the delayed retry queue, then acknowledges the source once confirmed. + * + * @param deliveryTag the AMQP delivery tag + * @param delay how long the message should wait + * @return a stage completing with the settlement outcome + */ + CompletionStage publishToRetryQueue(long deliveryTag, Duration delay); + + /** + * Publishes to the dead letter destination, then acknowledges the source once confirmed. + * + * @param deliveryTag the AMQP delivery tag + * @param failure the sanitized failure + * @return a stage completing with the settlement outcome + */ + CompletionStage deadLetter(long deliveryTag, FailureDescriptor failure); + + /** + * Discards a delivery without requeueing or dead lettering. + * + * @param deliveryTag the AMQP delivery tag + * @param failure the sanitized failure + * @return a stage completing with the settlement outcome + */ + CompletionStage discard(long deliveryTag, FailureDescriptor failure); +} diff --git a/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitTopologyProfile.java b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitTopologyProfile.java new file mode 100644 index 00000000..66633bbb --- /dev/null +++ b/src/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitTopologyProfile.java @@ -0,0 +1,137 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * The declared shape of one RabbitMQ destination: exchange, queue, and the arguments that give the + * queue its guarantees. + * + *

Quorum queues are required for anything durable. A classic mirrored queue can lose + * acknowledged messages during a partition — the promotion of a stale mirror is exactly the case + * where "the broker confirmed it" stops being true — and RabbitMQ itself has deprecated them. + * + *

The dead-letter exchange is part of the topology rather than a runtime decision, because a + * queue's {@code x-dead-letter-exchange} can only be set at declaration. A destination that gains a + * dead-letter policy later cannot acquire one without deleting and recreating the queue, so the + * mismatch is caught here instead of surfacing when the first message needs parking. + * + * @param exchange the exchange to publish to + * @param exchangeType the exchange type + * @param routingKey the routing key + * @param queue the queue that receives the messages + * @param quorum whether the queue is a quorum queue + * @param deadLetterExchange the dead letter exchange, when the destination has one + * @param deadLetterRoutingKey the dead letter routing key, when the destination has one + * @param maxLength the queue length limit, when one is set + */ +public record RabbitTopologyProfile( + String exchange, + String exchangeType, + String routingKey, + String queue, + boolean quorum, + Optional deadLetterExchange, + Optional deadLetterRoutingKey, + Optional maxLength) { + + /** RabbitMQ's argument key for a queue's type. */ + public static final String QUEUE_TYPE = "x-queue-type"; + + /** RabbitMQ's argument key for a queue's dead letter exchange. */ + public static final String DEAD_LETTER_EXCHANGE = "x-dead-letter-exchange"; + + /** RabbitMQ's argument key for a queue's dead letter routing key. */ + public static final String DEAD_LETTER_ROUTING_KEY = "x-dead-letter-routing-key"; + + /** RabbitMQ's argument key for a queue's length limit. */ + public static final String MAX_LENGTH = "x-max-length"; + + public RabbitTopologyProfile { + Objects.requireNonNull(deadLetterExchange, "deadLetterExchange must not be null"); + Objects.requireNonNull(deadLetterRoutingKey, "deadLetterRoutingKey must not be null"); + Objects.requireNonNull(maxLength, "maxLength must not be null"); + requireText(exchange, "exchange"); + requireText(exchangeType, "exchangeType"); + requireText(routingKey, "routingKey"); + requireText(queue, "queue"); + + if (deadLetterRoutingKey.isPresent() && deadLetterExchange.isEmpty()) { + throw new IllegalArgumentException( + "a dead letter routing key without a dead letter exchange routes nowhere"); + } + if (maxLength.isPresent() && maxLength.get() < 1) { + throw new IllegalArgumentException("maxLength must be positive when set"); + } + } + + /** + * Refuses a topology that cannot back a durable destination. + * + * @param durable whether the destination promises durability + * @throws MessagingConfigurationException when a durable destination lacks a quorum queue + */ + public void requireSafeFor(boolean durable) { + if (durable && !quorum) { + throw new MessagingConfigurationException( + "CLASSIC_QUEUE_ON_DURABLE_DESTINATION", + "queue %s is not a quorum queue; a classic mirrored queue can lose acknowledged messages " + .formatted(queue) + + "when a stale mirror is promoted during a partition"); + } + } + + /** + * Returns the arguments this queue must be declared with. + * + * @return the declaration arguments + */ + public Map queueArguments() { + Map arguments = new LinkedHashMap<>(); + arguments.put(QUEUE_TYPE, quorum ? "quorum" : "classic"); + deadLetterExchange.ifPresent(value -> arguments.put(DEAD_LETTER_EXCHANGE, value)); + deadLetterRoutingKey.ifPresent(value -> arguments.put(DEAD_LETTER_ROUTING_KEY, value)); + maxLength.ifPresent(value -> arguments.put(MAX_LENGTH, value)); + return Map.copyOf(arguments); + } + + /** + * Reports whether the broker will route a rejected message itself. + * + * @return true when the queue was declared with a dead letter exchange + */ + public boolean hasNativeDeadLetter() { + return deadLetterExchange.isPresent(); + } + + /** + * Returns a durable work-queue topology with a dead letter exchange. + * + * @param exchange the exchange + * @param routingKey the routing key + * @param queue the queue + * @param deadLetterExchange the dead letter exchange + * @return the topology profile + */ + public static RabbitTopologyProfile workQueue( + String exchange, String routingKey, String queue, String deadLetterExchange) { + return new RabbitTopologyProfile( + exchange, + "direct", + routingKey, + queue, + true, + Optional.of(deadLetterExchange), + Optional.of(routingKey), + Optional.empty()); + } + + private static void requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " must not be blank"); + } + } +} diff --git a/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitAdapterContractTest.java b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitAdapterContractTest.java new file mode 100644 index 00000000..60d8e245 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitAdapterContractTest.java @@ -0,0 +1,24 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.testkit.MessagingAdapterContract; +import dev.caskeleton.messaging.testkit.MessagingAdapterHarness; +import org.junit.jupiter.api.Nested; + +/** + * Runs the shared adapter contract against the RabbitMQ adapter. + * + *

Identical to the Kafka run. Two brokers with completely different machinery — offsets and + * commits versus delivery tags and confirms — answering the same seven questions the same way is + * what makes the logical destination abstraction real rather than aspirational. + */ +class RabbitAdapterContractTest { + + @Nested + class Contract extends MessagingAdapterContract { + + @Override + protected MessagingAdapterHarness harness() { + return RabbitContractHarness.create(); + } + } +} diff --git a/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitBrokerIT.java b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitBrokerIT.java new file mode 100644 index 00000000..7937f528 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitBrokerIT.java @@ -0,0 +1,232 @@ +package dev.caskeleton.messaging.rabbit; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.rabbitmq.client.AMQP; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.ConnectionFactory; +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.schema.EncodedMessage; +import dev.caskeleton.messaging.testkit.DockerAvailability; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.TimeoutException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.springframework.amqp.core.MessageProperties; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.rabbitmq.RabbitMQContainer; + +/** + * Certifies the RabbitMQ adapter against a live broker. + * + *

The test that matters is the unroutable one. AMQP delivers a {@code basic.return} + * before the confirm for a message no queue was bound to receive, and an adapter that + * completes on the confirm alone reports success for a message the broker threw away. No mock + * proves the ordering is handled correctly, because the ordering is the broker's behaviour. + */ +@Testcontainers +@EnabledIf("dockerAvailable") +class RabbitBrokerIT { + + private static final String EXCHANGE = "notification.work"; + private static final String BOUND_KEY = "email"; + private static final String UNBOUND_KEY = "nothing.is.bound.here"; + private static final String QUEUE = "notification.email.q"; + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + @Container + private static final RabbitMQContainer RABBIT = new RabbitMQContainer("rabbitmq:4.3-management"); + + private Connection connection; + private Channel channel; + private RabbitMessagingTransport transport; + + static boolean dockerAvailable() { + return DockerAvailability.isAvailable(); + } + + @BeforeEach + void declareTopology() throws IOException, TimeoutException { + ConnectionFactory factory = new ConnectionFactory(); + factory.setHost(RABBIT.getHost()); + factory.setPort(RABBIT.getAmqpPort()); + factory.setUsername(RABBIT.getAdminUsername()); + factory.setPassword(RABBIT.getAdminPassword()); + connection = factory.newConnection(); + channel = connection.createChannel(); + + channel.exchangeDeclare(EXCHANGE, "direct", true); + channel.queueDeclare(QUEUE, true, false, false, Map.of("x-queue-type", "quorum")); + channel.queueBind(QUEUE, EXCHANGE, BOUND_KEY); + // The container is shared across the class, so a message left by an earlier test would make a + // later queue-depth assertion pass or fail for the wrong reason. + channel.queuePurge(QUEUE); + channel.confirmSelect(); + + transport = + new RabbitMessagingTransport( + "rabbit-primary", + 1, + new RabbitChannelPublisher() { + @Override + public long nextPublishSequence() { + return channel.getNextPublishSeqNo(); + } + + @Override + public void publish( + long sequence, + String exchange, + String routingKey, + org.springframework.amqp.core.Message message, + boolean mandatory) { + RabbitBrokerIT.this.publish(sequence, exchange, routingKey, message, mandatory); + } + }); + + channel.addConfirmListener( + (deliveryTag, multiple) -> transport.onConfirm(deliveryTag, true, Duration.ofMillis(1)), + (deliveryTag, multiple) -> transport.onConfirm(deliveryTag, false, Duration.ofMillis(1))); + channel.addReturnListener( + returned -> + transport.onReturn( + Long.parseLong(returned.getProperties().getHeaders().get("x-seq").toString()))); + } + + @AfterEach + void close() throws IOException, TimeoutException { + if (channel != null && channel.isOpen()) { + channel.close(); + } + if (connection != null && connection.isOpen()) { + connection.close(); + } + } + + @Test + void aRoutablePublishConfirmsWithReplicationEvidence() { + PublishResult result = await(transport.publish(request(BOUND_KEY))); + + assertThat(result.completion()).isEqualTo(PublishCompletion.CONFIRMED); + assertThat(result.routingOutcome()).isEqualTo(RoutingOutcome.ROUTED); + assertThat(result.evidence().confirmationLevel()) + .isEqualTo(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK); + } + + @Test + void anUnroutablePublishIsRejectedEvenThoughTheExchangeConfirmedIt() { + PublishResult result = await(transport.publish(request(UNBOUND_KEY))); + + assertThat(result.completion()) + .as("the exchange accepted it and the broker threw it away; that is not success") + .isEqualTo(PublishCompletion.REJECTED); + assertThat(result.routingOutcome()).isEqualTo(RoutingOutcome.UNROUTABLE); + assertThat(result.evidence().confirmationLevel()).isEqualTo(ConfirmationLevel.NONE); + } + + @Test + void aRoutablePublishReachesTheQueue() throws IOException { + await(transport.publish(request(BOUND_KEY))); + + assertThat(channel.messageCount(QUEUE)).isEqualTo(1); + } + + @Test + void anUnroutablePublishReachesNoQueue() throws IOException { + await(transport.publish(request(UNBOUND_KEY))); + + assertThat(channel.messageCount(QUEUE)).isZero(); + } + + /** + * Publishes and records the sequence number in a header. + * + *

A returned message arrives without its publish sequence number, so the adapter has to carry + * one itself to correlate the return with the pending publish. + */ + private void publish( + long sequence, + String exchange, + String routingKey, + org.springframework.amqp.core.Message message, + boolean mandatory) { + try { + MessageProperties source = message.getMessageProperties(); + AMQP.BasicProperties properties = + new AMQP.BasicProperties.Builder() + .messageId(source.getMessageId()) + .contentType(source.getContentType()) + .deliveryMode(2) + .headers(withSequence(source, sequence)) + .build(); + // No waitForConfirms: the coordinator completes the publish from the confirm listener, and + // blocking here would hold the calling thread through the round trip for no benefit. + channel.basicPublish(exchange, routingKey, mandatory, properties, message.getBody()); + } catch (IOException exception) { + throw new IllegalStateException("rabbit publish failed", exception); + } + } + + private static Map withSequence(MessageProperties source, long sequence) { + java.util.Map headers = new java.util.LinkedHashMap<>(source.getHeaders()); + headers.put("x-seq", Long.toString(sequence)); + return headers; + } + + private static PublishResult await( + CompletionStage stage) { + return stage.toCompletableFuture().join().result(); + } + + private TransportPublishRequest request(String routingKey) { + return new TransportPublishRequest( + RabbitFixtureProfiles.routingTo(EXCHANGE, routingKey, QUEUE), + envelope(), + PublishOptions.defaults()); + } + + private static MessageEnvelope envelope() { + return new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + NOW, + Optional.of(NOW), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage( + "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), + ContentType.JSON, + Optional.empty())); + } +} diff --git a/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitConfirmCoordinatorTest.java b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitConfirmCoordinatorTest.java new file mode 100644 index 00000000..c719821f --- /dev/null +++ b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitConfirmCoordinatorTest.java @@ -0,0 +1,116 @@ +package dev.caskeleton.messaging.rabbit; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.destination.ConfirmationRequirement; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import java.time.Duration; +import java.util.concurrent.CompletionStage; +import org.junit.jupiter.api.Test; + +class RabbitConfirmCoordinatorTest { + + private static final ConfirmationRequirement REPLICATION = + ConfirmationRequirement.REPLICATION_OR_PERSISTENCE_ACK; + + private final RabbitConfirmCoordinator coordinator = new RabbitConfirmCoordinator(); + + @Test + void aPlainConfirmIsAConfirmedPublish() { + CompletionStage stage = + coordinator.awaiting(1L, "notification.work", "email", REPLICATION); + + coordinator.confirmed(1L, true, Duration.ofMillis(4)); + + PublishResult result = stage.toCompletableFuture().join(); + assertThat(result.completion()).isEqualTo(PublishCompletion.CONFIRMED); + assertThat(result.routingOutcome()).isEqualTo(RoutingOutcome.ROUTED); + assertThat(result.evidence().confirmationLevel()) + .isEqualTo(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK); + } + + @Test + void aReturnFollowedByAConfirmIsRejectedAsUnroutable() { + CompletionStage stage = + coordinator.awaiting(1L, "notification.work", "missing", REPLICATION); + + coordinator.returned(1L); + coordinator.confirmed(1L, true, Duration.ofMillis(4)); + + PublishResult result = stage.toCompletableFuture().join(); + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.routingOutcome()).isEqualTo(RoutingOutcome.UNROUTABLE); + assertThat(result.evidence().confirmationLevel()).isEqualTo(ConfirmationLevel.NONE); + } + + @Test + void aReturnAloneDoesNotCompleteThePublish() { + CompletionStage stage = + coordinator.awaiting(1L, "notification.work", "missing", REPLICATION); + + coordinator.returned(1L); + + assertThat(stage.toCompletableFuture()).isNotDone(); + assertThat(coordinator.pendingCount()).isEqualTo(1); + } + + @Test + void aNackIsARejectedPublish() { + CompletionStage stage = + coordinator.awaiting(2L, "notification.work", "email", REPLICATION); + + coordinator.confirmed(2L, false, Duration.ofMillis(9)); + + PublishResult result = stage.toCompletableFuture().join(); + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.failure()).isPresent(); + } + + @Test + void aMissingConfirmIsAmbiguousRatherThanFailed() { + CompletionStage stage = + coordinator.awaiting(3L, "notification.work", "email", REPLICATION); + + coordinator.timedOut(3L, Duration.ofSeconds(5)); + + PublishResult result = stage.toCompletableFuture().join(); + assertThat(result.completion()).isEqualTo(PublishCompletion.AMBIGUOUS); + assertThat(result.mayHaveBeenStored()).isTrue(); + assertThat(result.evidence().confirmationLevel()).isEqualTo(ConfirmationLevel.NONE); + } + + @Test + void aConfirmedPublishCarriesItsExchangeAndSequence() { + CompletionStage stage = + coordinator.awaiting(7L, "notification.work", "email", REPLICATION); + + coordinator.confirmed(7L, true, Duration.ofMillis(1)); + + PublishResult result = stage.toCompletableFuture().join(); + assertThat(result.position()).isPresent(); + assertThat(result.position().orElseThrow().diagnosticAttributes()) + .containsEntry("exchange", "notification.work") + .containsEntry("routingKey", "email") + .containsEntry("sequence", "7"); + } + + @Test + void resolvingAPublishRemovesItFromThePendingSet() { + coordinator.awaiting(1L, "notification.work", "email", REPLICATION); + coordinator.confirmed(1L, true, Duration.ofMillis(1)); + + assertThat(coordinator.pendingCount()).isZero(); + } + + @Test + void anUnknownSequenceIsIgnoredRatherThanFailingTheChannel() { + coordinator.confirmed(99L, true, Duration.ofMillis(1)); + coordinator.returned(99L); + coordinator.timedOut(99L, Duration.ofMillis(1)); + + assertThat(coordinator.pendingCount()).isZero(); + } +} diff --git a/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitContractHarness.java b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitContractHarness.java new file mode 100644 index 00000000..54394d56 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitContractHarness.java @@ -0,0 +1,304 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.destination.ConfirmationRequirement; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.testkit.ContractMessage; +import dev.caskeleton.messaging.testkit.FaultController; +import dev.caskeleton.messaging.testkit.HandleOutcome; +import dev.caskeleton.messaging.testkit.MessagingAdapterHarness; +import dev.caskeleton.messaging.testkit.ObservedDelivery; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Runs the shared adapter contract against the real RabbitMQ confirm model. + * + *

Publishes go through the production {@link RabbitConfirmCoordinator}, which is the piece that + * has to get AMQP's confirm and return ordering right. Driving it from the harness is the only way + * to exercise a lost confirm deterministically — a live broker will not drop one on request. + * + *

Settlement goes through the production {@link RabbitSettlementController}, so the + * one-terminal-call guard and the DLQ-before-ack ordering are the real ones. + */ +final class RabbitContractHarness implements MessagingAdapterHarness { + + private static final int MAX_PAYLOAD_BYTES = 1_048_576; + private static final String EXCHANGE = "order.events"; + private static final String ROUTING_KEY = "created"; + + private final RabbitConfirmCoordinator coordinator = new RabbitConfirmCoordinator(); + private final AtomicLong sequence = new AtomicLong(); + private final AtomicLong deliveryTag = new AtomicLong(); + private final Faults faults = new Faults(); + private final Deque pending = new ArrayDeque<>(); + private final List deadLetters = new ArrayList<>(); + private final Set unsettled = new LinkedHashSet<>(); + + private boolean shuttingDown; + + static RabbitContractHarness create() { + return new RabbitContractHarness(); + } + + @Override + public String brokerName() { + return "rabbitmq"; + } + + @Override + public FaultController faults() { + return faults; + } + + @Override + public CompletionStage publish(ContractMessage message) { + if (shuttingDown) { + return CompletableFuture.completedFuture( + rejectedLocally("SHUTTING_DOWN", "the consumer is draining and accepts no new work")); + } + if (message.envelope().payload().size() > MAX_PAYLOAD_BYTES) { + return CompletableFuture.completedFuture( + rejectedLocally( + "PAYLOAD_TOO_LARGE", "encoded payload exceeds " + MAX_PAYLOAD_BYTES + " bytes")); + } + + long next = sequence.incrementAndGet(); + CompletionStage stage = + coordinator.awaiting( + next, EXCHANGE, ROUTING_KEY, ConfirmationRequirement.REPLICATION_OR_PERSISTENCE_ACK); + + if (faults.consumeRejectPublish()) { + coordinator.confirmed(next, false, Duration.ofMillis(2)); + } else if (faults.consumeDropPublishConfirmation()) { + coordinator.timedOut(next, Duration.ofSeconds(5)); + } else { + coordinator.confirmed(next, true, Duration.ofMillis(2)); + } + + return stage.thenApply( + result -> { + if (result.completion() == PublishCompletion.CONFIRMED) { + pending.addLast( + new Pending(message.messageId(), deliveryTag.incrementAndGet(), 1, false)); + } + return result; + }); + } + + @Override + public List drain(HandleOutcome outcome) { + List batch = new ArrayList<>(pending); + pending.clear(); + + List observed = new ArrayList<>(); + for (Pending delivery : batch) { + RabbitSettlementController controller = + new RabbitSettlementController(delivery.deliveryTag(), new HarnessOperations(delivery)); + boolean settled = + switch (outcome) { + case SUCCESS -> join(controller.ack()); + case RETRY -> join(controller.retry(Duration.ofSeconds(1))); + case DEAD_LETTER -> + join( + controller.deadLetter( + FailureDescriptor.of( + FailureCategory.PERMANENT_BUSINESS, + "CONTRACT_FAILURE", + "permanent failure"))); + }; + observed.add( + new ObservedDelivery( + delivery.messageId(), delivery.attempt(), delivery.redelivered(), settled)); + } + return List.copyOf(observed); + } + + private static boolean join( + CompletionStage stage) { + return stage.toCompletableFuture().join().completion() + == dev.caskeleton.messaging.api.settlement.SettlementCompletion.SETTLED; + } + + private void redeliver(Pending delivery) { + pending.addLast( + new Pending( + delivery.messageId(), deliveryTag.incrementAndGet(), delivery.attempt() + 1, true)); + } + + @Override + public List deadLettered() { + return List.copyOf(deadLetters); + } + + @Override + public List unsettled() { + return List.copyOf(unsettled); + } + + @Override + public void beginShutdown() { + shuttingDown = true; + } + + @Override + public boolean isAcceptingWork() { + return !shuttingDown; + } + + @Override + public void close() { + pending.clear(); + } + + private static PublishResult rejectedLocally(String code, String message) { + return new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of(FailureDescriptor.of(FailureCategory.PERMANENT_BUSINESS, code, message))); + } + + /** One message in flight on the channel. */ + private record Pending(MessageId messageId, long deliveryTag, int attempt, boolean redelivered) {} + + /** The channel operations the settlement controller drives. */ + private final class HarnessOperations implements RabbitSettlementOperations { + + private final Pending delivery; + + private HarnessOperations(Pending delivery) { + this.delivery = delivery; + } + + @Override + public CompletionStage ack(long tag) { + if (faults.consumeDropSettlementConfirmation()) { + unsettled.add(delivery.messageId()); + redeliver(delivery); + return CompletableFuture.completedFuture(unknownSettlement()); + } + unsettled.remove(delivery.messageId()); + return CompletableFuture.completedFuture( + dev.caskeleton.messaging.api.settlement.SettlementResult.settled()); + } + + @Override + public CompletionStage + publishToRetryQueue(long tag, Duration delay) { + unsettled.add(delivery.messageId()); + redeliver(delivery); + return CompletableFuture.completedFuture(unknownSettlement()); + } + + /** Publishes to the dead letter destination first, and only then acknowledges the source. */ + @Override + public CompletionStage deadLetter( + long tag, FailureDescriptor failure) { + if (faults.deadLetterPublishFails()) { + unsettled.add(delivery.messageId()); + redeliver(delivery); + return CompletableFuture.completedFuture(unknownSettlement()); + } + deadLetters.add(delivery.messageId()); + unsettled.remove(delivery.messageId()); + return CompletableFuture.completedFuture( + dev.caskeleton.messaging.api.settlement.SettlementResult.settled()); + } + + @Override + public CompletionStage discard( + long tag, FailureDescriptor failure) { + unsettled.remove(delivery.messageId()); + return CompletableFuture.completedFuture( + dev.caskeleton.messaging.api.settlement.SettlementResult.settled()); + } + + private dev.caskeleton.messaging.api.settlement.SettlementResult unknownSettlement() { + return new dev.caskeleton.messaging.api.settlement.SettlementResult( + dev.caskeleton.messaging.api.settlement.SettlementCompletion.UNKNOWN, + dev.caskeleton.messaging.api.settlement.SettlementEvidence.unknown(), + Optional.of( + FailureDescriptor.of( + FailureCategory.AMBIGUOUS, + "SETTLEMENT_UNKNOWN", + "the settlement was not confirmed"))); + } + } + + /** One-shot fault flags. */ + private static final class Faults implements FaultController { + + private boolean dropPublishConfirmation; + private boolean dropSettlementConfirmation; + private boolean failDeadLetterPublish; + private boolean rejectPublish; + + @Override + public void dropPublishConfirmation() { + dropPublishConfirmation = true; + } + + @Override + public void dropSettlementConfirmation() { + dropSettlementConfirmation = true; + } + + @Override + public void failDeadLetterPublish() { + failDeadLetterPublish = true; + } + + @Override + public void rejectPublish() { + rejectPublish = true; + } + + @Override + public void reset() { + dropPublishConfirmation = false; + dropSettlementConfirmation = false; + failDeadLetterPublish = false; + rejectPublish = false; + } + + boolean consumeDropPublishConfirmation() { + boolean active = dropPublishConfirmation; + dropPublishConfirmation = false; + return active; + } + + boolean consumeDropSettlementConfirmation() { + boolean active = dropSettlementConfirmation; + dropSettlementConfirmation = false; + return active; + } + + boolean consumeRejectPublish() { + boolean active = rejectPublish; + rejectPublish = false; + return active; + } + + boolean deadLetterPublishFails() { + return failDeadLetterPublish; + } + } +} diff --git a/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitFixtureProfiles.java b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitFixtureProfiles.java new file mode 100644 index 00000000..da405309 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitFixtureProfiles.java @@ -0,0 +1,32 @@ +package dev.caskeleton.messaging.rabbit; + +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.PhysicalDestination; + +/** Builds destination profiles that vary only in their physical mapping. */ +final class RabbitFixtureProfiles { + + private RabbitFixtureProfiles() {} + + static DestinationProfile routingTo(String exchange, String routingKey, String queue) { + DestinationProfile base = RabbitProfileFixtures.workQueueDestination(); + return new DestinationProfile( + base.name(), + base.broker(), + base.kind(), + PhysicalDestination.rabbitQueue(exchange, routingKey, queue), + base.schema(), + base.deliveryGuarantee(), + base.orderingScope(), + base.externalSideEffectGuarantee(), + base.producer(), + base.consumer(), + base.retry(), + base.deadLetter(), + base.payload(), + base.tier(), + base.production(), + base.keyResolverConfigured(), + base.topologyAutoCreate()); + } +} diff --git a/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitProfileValidatorTest.java b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitProfileValidatorTest.java new file mode 100644 index 00000000..54193982 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitProfileValidatorTest.java @@ -0,0 +1,184 @@ +package dev.caskeleton.messaging.rabbit; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.delivery.ExternalSideEffectGuarantee; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.DestinationKind; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.policy.CapabilityTier; +import dev.caskeleton.messaging.policy.ConsumerPolicy; +import dev.caskeleton.messaging.policy.DeadLetterPolicy; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.PayloadPolicy; +import dev.caskeleton.messaging.policy.PhysicalDestination; +import dev.caskeleton.messaging.policy.ProducerPolicy; +import dev.caskeleton.messaging.policy.RetryPolicy; +import dev.caskeleton.messaging.policy.SchemaPolicy; +import dev.caskeleton.messaging.schema.SchemaCompatibility; +import java.time.Duration; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class RabbitProfileValidatorTest { + + private final RabbitProfileValidator validator = new RabbitProfileValidator(); + + @Test + void stableProducerRequiresConfirms() { + assertThatThrownBy(() -> validator.validate(RabbitProfileFixtures.withoutConfirms())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("publisher confirms"); + } + + @Test + void stableProducerRequiresReturnsBecauseAConfirmDoesNotProveRouting() { + assertThatThrownBy(() -> validator.validate(RabbitProfileFixtures.withoutReturns())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("publisher returns"); + } + + @Test + void stableProducerRequiresMandatoryRouting() { + assertThatThrownBy(() -> validator.validate(RabbitProfileFixtures.withoutMandatory())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("mandatory"); + } + + @Test + void consumerAutoAckIsForbidden() { + assertThatThrownBy(() -> validator.validate(RabbitProfileFixtures.withAutoAck())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("auto ack"); + } + + @Test + void productionRequiresTlsAndAuthentication() { + assertThatThrownBy(() -> validator.validate(RabbitProfileFixtures.productionWithoutTls())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("TLS"); + } + + @Test + void aDurableWorkQueueMustBeAQuorumQueue() { + assertThatThrownBy( + () -> + validator.validateDestination( + RabbitProfileFixtures.workQueueDestination(), + RabbitProfileFixtures.withoutQuorumQueues())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("quorum"); + } + + @Test + void aStableProfileAndDestinationValidate() { + assertThatCode( + () -> { + validator.validate(RabbitProfileFixtures.stable()); + validator.validateDestination( + RabbitProfileFixtures.workQueueDestination(), RabbitProfileFixtures.stable()); + }) + .doesNotThrowAnyException(); + } +} + +/** Builds RabbitMQ profiles for validator tests. */ +final class RabbitProfileFixtures { + + private RabbitProfileFixtures() {} + + static RabbitBrokerProfile stable() { + return new RabbitBrokerProfile( + "rabbit-primary", + true, + false, + List.of("localhost:5672"), + true, + true, + true, + Duration.ofSeconds(5), + false, + 16, + true, + true, + true); + } + + static RabbitBrokerProfile withoutConfirms() { + return copy(stable(), false, true, true, false, true, false, true); + } + + static RabbitBrokerProfile withoutReturns() { + return copy(stable(), true, false, true, false, true, false, true); + } + + static RabbitBrokerProfile withoutMandatory() { + return copy(stable(), true, true, false, false, true, false, true); + } + + static RabbitBrokerProfile withAutoAck() { + return copy(stable(), true, true, true, true, true, false, true); + } + + static RabbitBrokerProfile withoutQuorumQueues() { + return copy(stable(), true, true, true, false, false, false, true); + } + + static RabbitBrokerProfile productionWithoutTls() { + return copy(stable(), true, true, true, false, true, true, false); + } + + private static RabbitBrokerProfile copy( + RabbitBrokerProfile base, + boolean confirms, + boolean returns, + boolean mandatory, + boolean autoAck, + boolean quorumQueues, + boolean production, + boolean tlsEnabled) { + return new RabbitBrokerProfile( + base.broker(), + base.stable(), + production, + base.addresses(), + confirms, + returns, + mandatory, + base.confirmTimeout(), + autoAck, + base.prefetch(), + quorumQueues, + tlsEnabled, + base.authenticationEnabled()); + } + + static DestinationProfile workQueueDestination() { + return new DestinationProfile( + new DestinationName("email-work"), + "rabbit-primary", + DestinationKind.WORK_QUEUE, + PhysicalDestination.rabbitQueue("notification.work", "email", "notification.email.q"), + new SchemaPolicy( + ContentType.JSON, + SchemaCompatibility.BACKWARD, + Set.of(new MessageType("email.requested"))), + DeliveryGuarantee.AT_LEAST_ONCE, + OrderingScope.NONE, + ExternalSideEffectGuarantee.IDEMPOTENCY_REQUIRED, + ProducerPolicy.defaults(), + ConsumerPolicy.defaults("notification"), + RetryPolicy.none(), + DeadLetterPolicy.to(new DestinationName("email-work-dlq")), + PayloadPolicy.defaults(), + CapabilityTier.M1, + false, + false, + false); + } +} diff --git a/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitRuntimeTest.java b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitRuntimeTest.java new file mode 100644 index 00000000..36f9d792 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitRuntimeTest.java @@ -0,0 +1,263 @@ +package dev.caskeleton.messaging.rabbit; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.CorrelationId; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.schema.EncodedMessage; +import dev.caskeleton.messaging.transport.GracefulShutdownCoordinator; +import dev.caskeleton.messaging.transport.TransportConsumerSpec; +import dev.caskeleton.messaging.transport.TransportDelivery; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import dev.caskeleton.messaging.transport.TransportPublishResult; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicLong; +import org.junit.jupiter.api.Test; +import org.springframework.amqp.core.Message; + +class RabbitRuntimeTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + private final AtomicLong sequence = new AtomicLong(); + private final List published = new ArrayList<>(); + private final List handled = new ArrayList<>(); + + private RabbitMessagingTransport transport() { + return new RabbitMessagingTransport( + "rabbit-primary", + 1, + new RabbitChannelPublisher() { + @Override + public long nextPublishSequence() { + return sequence.incrementAndGet(); + } + + @Override + public void publish( + long sequenceNumber, + String exchange, + String routingKey, + Message message, + boolean mandatory) { + published.add(message); + } + }); + } + + @Test + void aConfirmedPublishReportsRoutedWithReplicationEvidence() { + RabbitMessagingTransport transport = transport(); + + CompletionStage stage = transport.publish(request()); + transport.onConfirm(1L, true, Duration.ofMillis(3)); + + var result = stage.toCompletableFuture().join().result(); + assertThat(result.completion()).isEqualTo(PublishCompletion.CONFIRMED); + assertThat(result.routingOutcome()).isEqualTo(RoutingOutcome.ROUTED); + } + + @Test + void aReturnedThenConfirmedPublishIsRejectedAsUnroutable() { + RabbitMessagingTransport transport = transport(); + + CompletionStage stage = transport.publish(request()); + transport.onReturn(1L); + transport.onConfirm(1L, true, Duration.ofMillis(3)); + + var result = stage.toCompletableFuture().join().result(); + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.routingOutcome()).isEqualTo(RoutingOutcome.UNROUTABLE); + } + + @Test + void aMissingConfirmIsAmbiguous() { + RabbitMessagingTransport transport = transport(); + + CompletionStage stage = transport.publish(request()); + transport.onConfirmTimeout(1L, Duration.ofSeconds(5)); + + assertThat(stage.toCompletableFuture().join().result().mayHaveBeenStored()).isTrue(); + } + + @Test + void anOversizedPayloadNeverReachesTheChannel() { + RabbitMessagingTransport transport = transport(); + + var result = + transport.publish(request(new byte[1_048_577])).toCompletableFuture().join().result(); + + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(published).isEmpty(); + } + + @Test + void aClosedTransportRefusesPublishesLocally() { + RabbitMessagingTransport transport = transport(); + transport.close(); + + var result = transport.publish(request()).toCompletableFuture().join().result(); + + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(transport.isAcceptingWork()).isFalse(); + assertThat(published).isEmpty(); + } + + @Test + void theHeaderRoundTripPreservesIdentityAndProvenance() { + RabbitHeaderMapper headers = new RabbitHeaderMapper(); + MessageEnvelope envelope = envelope(new byte[] {1, 2, 3}); + + var properties = headers.toProperties(envelope); + + assertThat(headers.value(properties, ReservedHeaders.MESSAGE_ID)) + .isEqualTo(envelope.messageId().value().toString()); + assertThat(headers.value(properties, ReservedHeaders.MESSAGE_TYPE)).isEqualTo("order.created"); + assertThat(headers.value(properties, ReservedHeaders.CORRELATION_ID)).isEqualTo("wf-1"); + assertThat(properties.getCorrelationId()).isEqualTo("wf-1"); + } + + @Test + void aDeliveryIsRebuiltFromItsHeaders() { + RabbitHeaderMapper headers = new RabbitHeaderMapper(); + MessageEnvelope original = envelope(new byte[] {1, 2, 3}); + Message message = new Message(new byte[] {1, 2, 3}, headers.toProperties(original)); + + var restored = new RabbitDeliveryMapper().toEnvelope(message); + + assertThat(restored.messageId()).isEqualTo(original.messageId()); + assertThat(restored.messageType()).isEqualTo(original.messageType()); + assertThat(restored.schemaVersion()).isEqualTo(original.schemaVersion()); + assertThat(restored.payload().bytes()).containsExactly(1, 2, 3); + } + + @Test + void theConsumerDispatchesAndSettlesThroughTheController() { + RecordingOperations operations = new RecordingOperations(); + RabbitConsumerRegistrar registrar = registrar(operations, this::acknowledge); + + boolean accepted = registrar.onMessage(deliveredMessage(7L), NOW); + + assertThat(accepted).isTrue(); + assertThat(handled).hasSize(1); + assertThat(operations.calls).containsExactly("ack:7"); + } + + @Test + void aPausedQueueRefusesDeliveriesSoTheBrokerRedeliversThem() { + RecordingOperations operations = new RecordingOperations(); + RabbitConsumerRegistrar registrar = registrar(operations, this::acknowledge); + registrar.pause("").toCompletableFuture().join(); + + boolean accepted = registrar.onMessage(deliveredMessage(7L), NOW); + + assertThat(accepted).isFalse(); + assertThat(handled).isEmpty(); + assertThat(operations.calls).isEmpty(); + } + + @Test + void drainingRefusesNewDeliveries() { + RecordingOperations operations = new RecordingOperations(); + GracefulShutdownCoordinator shutdown = new GracefulShutdownCoordinator(Duration.ofSeconds(30)); + RabbitConsumerRegistrar registrar = + new RabbitConsumerRegistrar( + new TransportConsumerSpec( + RabbitProfileFixtures.workQueueDestination(), this::acknowledge), + operations, + shutdown); + + shutdown.beginDrain(NOW); + boolean accepted = registrar.onMessage(deliveredMessage(7L), NOW); + + assertThat(accepted).isFalse(); + assertThat(operations.calls).isEmpty(); + } + + @Test + void anUndecodableMessageIsDiscardedRatherThanRequeued() { + RecordingOperations operations = new RecordingOperations(); + RabbitConsumerRegistrar registrar = registrar(operations, this::acknowledge); + + Message malformed = + new Message(new byte[] {1}, new org.springframework.amqp.core.MessageProperties()); + malformed.getMessageProperties().setDeliveryTag(9L); + + boolean accepted = registrar.onMessage(malformed, NOW); + + assertThat(accepted).isTrue(); + assertThat(handled).isEmpty(); + assertThat(operations.calls).containsExactly("discard:9"); + } + + private RabbitConsumerRegistrar registrar( + RabbitSettlementOperations operations, + java.util.function.Function> sink) { + return new RabbitConsumerRegistrar( + new TransportConsumerSpec(RabbitProfileFixtures.workQueueDestination(), sink), + operations, + new GracefulShutdownCoordinator(Duration.ofSeconds(30))); + } + + private CompletionStage acknowledge(TransportDelivery delivery) { + handled.add(delivery); + delivery.settlement().acknowledge().toCompletableFuture().join(); + return CompletableFuture.completedFuture(null); + } + + private Message deliveredMessage(long deliveryTag) { + Message message = + new Message( + "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), + new RabbitHeaderMapper().toProperties(envelope(new byte[] {1}))); + message.getMessageProperties().setDeliveryTag(deliveryTag); + message.getMessageProperties().setConsumerQueue("notification.email.q"); + return message; + } + + private TransportPublishRequest request() { + return request("{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8)); + } + + private TransportPublishRequest request(byte[] payload) { + return new TransportPublishRequest( + RabbitProfileFixtures.workQueueDestination(), envelope(payload), PublishOptions.defaults()); + } + + private MessageEnvelope envelope(byte[] payload) { + return new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + NOW, + Optional.of(NOW), + new ProducerId("order-api"), + Optional.of(new CorrelationId("wf-1")), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage(payload, ContentType.JSON, Optional.empty())); + } +} diff --git a/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitSettlementControllerTest.java b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitSettlementControllerTest.java new file mode 100644 index 00000000..f0cba51f --- /dev/null +++ b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitSettlementControllerTest.java @@ -0,0 +1,105 @@ +package dev.caskeleton.messaging.rabbit; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.settlement.SettlementResult; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import org.junit.jupiter.api.Test; + +class RabbitSettlementControllerTest { + + private static final FailureDescriptor FAILURE = + FailureDescriptor.of(FailureCategory.PERMANENT_BUSINESS, "REJECTED", "rejected"); + + @Test + void acknowledgesExactlyOnce() { + RecordingOperations operations = new RecordingOperations(); + RabbitSettlementController controller = new RabbitSettlementController(7L, operations); + + controller.ack().toCompletableFuture().join(); + + assertThat(operations.calls).containsExactly("ack:7"); + assertThat(controller.isSettled()).isTrue(); + } + + @Test + void aSecondTerminalCallFailsRatherThanRacingTheFirst() { + RecordingOperations operations = new RecordingOperations(); + RabbitSettlementController controller = new RabbitSettlementController(7L, operations); + + controller.ack().toCompletableFuture().join(); + + assertThat(controller.ack().toCompletableFuture()).isCompletedExceptionally(); + assertThat(operations.calls).containsExactly("ack:7"); + } + + @Test + void mixingTerminalOperationsIsAlsoRefused() { + RecordingOperations operations = new RecordingOperations(); + RabbitSettlementController controller = new RabbitSettlementController(7L, operations); + + controller.deadLetter(FAILURE).toCompletableFuture().join(); + + assertThat(controller.ack().toCompletableFuture()).isCompletedExceptionally(); + assertThat(operations.calls).containsExactly("deadLetter:7"); + } + + @Test + void retryGoesToTheDelayedQueueRatherThanRequeueingImmediately() { + RecordingOperations operations = new RecordingOperations(); + RabbitSettlementController controller = new RabbitSettlementController(7L, operations); + + controller.retry(Duration.ofSeconds(30)).toCompletableFuture().join(); + + assertThat(operations.calls).containsExactly("retry:7:PT30S"); + } + + @Test + void aRetryQueueCarriesTtlAndADeadLetterRouteBackToTheWorkQueue() { + RabbitRetryQueueTopology topology = + new RabbitRetryQueueTopology( + "notification.email.retry.q", "notification.work", "email", Duration.ofSeconds(30)); + + assertThat(topology.queueArguments()) + .containsEntry("x-queue-type", "quorum") + .containsEntry("x-message-ttl", 30_000L) + .containsEntry("x-dead-letter-exchange", "notification.work") + .containsEntry("x-dead-letter-routing-key", "email"); + } +} + +/** Records which channel operation the controller invoked. */ +final class RecordingOperations implements RabbitSettlementOperations { + + final List calls = new ArrayList<>(); + + @Override + public CompletionStage ack(long deliveryTag) { + calls.add("ack:" + deliveryTag); + return CompletableFuture.completedFuture(SettlementResult.settled()); + } + + @Override + public CompletionStage publishToRetryQueue(long deliveryTag, Duration delay) { + calls.add("retry:" + deliveryTag + ":" + delay); + return CompletableFuture.completedFuture(SettlementResult.settled()); + } + + @Override + public CompletionStage deadLetter(long deliveryTag, FailureDescriptor failure) { + calls.add("deadLetter:" + deliveryTag); + return CompletableFuture.completedFuture(SettlementResult.settled()); + } + + @Override + public CompletionStage discard(long deliveryTag, FailureDescriptor failure) { + calls.add("discard:" + deliveryTag); + return CompletableFuture.completedFuture(SettlementResult.settled()); + } +} diff --git a/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitTopologyAndBatchTest.java b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitTopologyAndBatchTest.java new file mode 100644 index 00000000..040a8df3 --- /dev/null +++ b/src/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitTopologyAndBatchTest.java @@ -0,0 +1,218 @@ +package dev.caskeleton.messaging.rabbit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class RabbitTopologyAndBatchTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + private static RabbitTopologyProfile quorumWithDlx() { + return RabbitTopologyProfile.workQueue( + "notification.work", "email", "notification.email.q", "notification.dlx"); + } + + private static RabbitTopologyProfile classic() { + return new RabbitTopologyProfile( + "notification.work", + "direct", + "email", + "notification.email.q", + false, + Optional.empty(), + Optional.empty(), + Optional.empty()); + } + + @Test + void aDurableDestinationRequiresAQuorumQueue() { + assertThatThrownBy(() -> classic().requireSafeFor(true)) + .as("a promoted stale mirror loses messages the broker already confirmed") + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("quorum"); + } + + @Test + void aQuorumQueueSatisfiesADurableDestination() { + assertThatCode(() -> quorumWithDlx().requireSafeFor(true)).doesNotThrowAnyException(); + } + + @Test + void theQueueArgumentsCarryTheDeclaredGuarantees() { + assertThat(quorumWithDlx().queueArguments()) + .containsEntry(RabbitTopologyProfile.QUEUE_TYPE, "quorum") + .containsEntry(RabbitTopologyProfile.DEAD_LETTER_EXCHANGE, "notification.dlx"); + } + + @Test + void aDeadLetterRoutingKeyWithoutAnExchangeIsRejected() { + assertThatThrownBy( + () -> + new RabbitTopologyProfile( + "x", + "direct", + "k", + "q", + true, + Optional.empty(), + Optional.of("k"), + Optional.empty())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("routes nowhere"); + } + + @Test + void nativeDeadLetterNeedsBothADeclarationAndAVerifiedBinding() { + assertThat(new RabbitNativeDeadLetterCapability(quorumWithDlx(), true).mayUseNativeDeadLetter()) + .isTrue(); + assertThat( + new RabbitNativeDeadLetterCapability(quorumWithDlx(), false).mayUseNativeDeadLetter()) + .as("an unbound dead letter exchange discards the message while the reject still succeeds") + .isFalse(); + } + + @Test + void aQueueWithoutADeadLetterExchangeFallsBackToThePlatformPublish() { + RabbitNativeDeadLetterCapability capability = + new RabbitNativeDeadLetterCapability(classic(), true); + + assertThat(capability.requiresPlatformPublish()).isTrue(); + assertThat(capability.reasonForPlatformPublish()).contains("without a dead letter exchange"); + } + + @Test + void theUnboundReasonExplainsTheSilentDiscard() { + assertThat( + new RabbitNativeDeadLetterCapability(quorumWithDlx(), false).reasonForPlatformPublish()) + .contains("silently") + .contains("reporting success"); + } + + @Test + void aFailureBeforeTheFrameWasWrittenIsADefinitiveRejection() { + PublishResult result = + new RabbitPublishFailureClassifier() + .classify(new IllegalStateException("channel closed"), false, Duration.ofMillis(3)); + + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.evidence().transmission()).isEqualTo(TransmissionEvidence.NOT_TRANSMITTED); + } + + @Test + void aFailureAfterTheFrameWasWrittenIsAmbiguous() { + PublishResult result = + new RabbitPublishFailureClassifier() + .classify(new IOException("connection reset"), true, Duration.ofMillis(3)); + + assertThat(result.completion()) + .as("the broker may hold the message and only the confirm was lost") + .isEqualTo(PublishCompletion.AMBIGUOUS); + assertThat(result.evidence().transmission()) + .isEqualTo(TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED); + } + + @Test + void aNackIsRejectedButStillCountsAsTransmitted() { + PublishResult result = new RabbitPublishFailureClassifier().nacked(Duration.ofMillis(3)); + + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.evidence().transmission()).isEqualTo(TransmissionEvidence.TRANSMITTED); + } + + @Test + void aBatchIsReleasedOnceItIsFull() { + RabbitBatchConsumerRegistrar registrar = + new RabbitBatchConsumerRegistrar( + RabbitProfileFixtures.workQueueDestination(), 3, Duration.ofSeconds(5)); + + assertThat(registrar.add(1, NOW)).isEmpty(); + assertThat(registrar.add(2, NOW)).isEmpty(); + assertThat(registrar.add(3, NOW)) + .hasValueSatisfying(batch -> assertThat(batch.deliveryTags()).containsExactly(1L, 2L, 3L)); + } + + @Test + void aPartialBatchIsReleasedOnceItsOldestDeliveryGoesStale() { + RabbitBatchConsumerRegistrar registrar = + new RabbitBatchConsumerRegistrar( + RabbitProfileFixtures.workQueueDestination(), 100, Duration.ofSeconds(5)); + registrar.add(1, NOW); + + assertThat(registrar.releaseIfStale(NOW.plusSeconds(1))).isEmpty(); + assertThat(registrar.releaseIfStale(NOW.plusSeconds(6))) + .as("a quiet queue would otherwise hold its last messages unacknowledged indefinitely") + .isPresent(); + } + + @Test + void anAccumulatedBatchIsNotSettlableAsAUnit() { + RabbitBatchConsumerRegistrar registrar = + new RabbitBatchConsumerRegistrar( + RabbitProfileFixtures.workQueueDestination(), 2, Duration.ofSeconds(5)); + registrar.add(1, NOW); + + assertThat(registrar.add(2, NOW)) + .hasValueSatisfying( + batch -> + assertThat(batch.metadata().settlableAsBatch()) + .as("AMQP multiple-ack settles work that is still in flight") + .isFalse()); + } + + @Test + void anAccumulatedBatchNamesExactlyOneOrderingUnit() { + RabbitBatchConsumerRegistrar registrar = + new RabbitBatchConsumerRegistrar( + RabbitProfileFixtures.workQueueDestination(), 1, Duration.ofSeconds(5)); + + assertThat(registrar.add(1, NOW)) + .hasValueSatisfying( + batch -> assertThat(batch.metadata().isSafeForOrderedDestination()).isTrue()); + } + + @Test + void aPrefetchBelowTheBatchSizeWouldDeadlockAndIsRefused() { + RabbitBatchConsumerRegistrar registrar = + new RabbitBatchConsumerRegistrar( + RabbitProfileFixtures.workQueueDestination(), 50, Duration.ofSeconds(5)); + + assertThatThrownBy(() -> registrar.requirePrefetchFor(10)) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("prefetch"); + assertThatCode(() -> registrar.requirePrefetchFor(100)).doesNotThrowAnyException(); + } + + @Test + void aZeroBatchAgeIsRefusedBecauseItStrandsQuietQueues() { + assertThatThrownBy( + () -> + new RabbitBatchConsumerRegistrar( + RabbitProfileFixtures.workQueueDestination(), 10, Duration.ZERO)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void drainingReleasesWhateverIsPendingForShutdown() { + RabbitBatchConsumerRegistrar registrar = + new RabbitBatchConsumerRegistrar( + RabbitProfileFixtures.workQueueDestination(), 100, Duration.ofSeconds(5)); + registrar.add(1, NOW); + registrar.add(2, NOW); + + assertThat(registrar.drain(NOW)) + .hasValueSatisfying(batch -> assertThat(batch.deliveryTags()).hasSize(2)); + assertThat(registrar.pending()).isZero(); + assertThat(registrar.drain(NOW)).isEmpty(); + } +} diff --git a/src/messaging/messaging-reliability-api/build.gradle b/src/messaging/messaging-reliability-api/build.gradle new file mode 100644 index 00000000..95af0ac3 --- /dev/null +++ b/src/messaging/messaging-reliability-api/build.gradle @@ -0,0 +1,5 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') +} diff --git a/src/messaging/messaging-reliability-api/gradle.lockfile b/src/messaging/messaging-reliability-api/gradle.lockfile new file mode 100644 index 00000000..599ff921 --- /dev/null +++ b/src/messaging/messaging-reliability-api/gradle.lockfile @@ -0,0 +1,83 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty=compileClasspath,runtimeClasspath diff --git a/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/ClaimCheckReference.java b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/ClaimCheckReference.java new file mode 100644 index 00000000..1985439f --- /dev/null +++ b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/ClaimCheckReference.java @@ -0,0 +1,50 @@ +package dev.caskeleton.messaging.reliability; + +import java.time.Instant; +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * Points at a payload stored outside the broker. + * + *

The digest is part of the reference, not an optional extra. A claim check splits a message + * into two systems with independent retention and replication, so a consumer that fetches the + * payload has to be able to prove it got the bytes the producer stored — otherwise a truncated or + * replaced object is indistinguishable from a valid one. + * + *

The expiry is carried for the same reason: a claim check whose payload has been reaped is a + * dead message, and detecting that at fetch time is better than a mysterious not-found. + * + * @param storageKey the object key in the payload store + * @param sizeBytes the stored payload size + * @param sha256 the lowercase hex digest of the stored payload + * @param expiresAt when the payload store may reap the object + */ +public record ClaimCheckReference( + String storageKey, long sizeBytes, String sha256, Instant expiresAt) { + + private static final Pattern SHA256 = Pattern.compile("[a-f0-9]{64}"); + + public ClaimCheckReference { + Objects.requireNonNull(expiresAt, "expiresAt must not be null"); + if (storageKey == null || storageKey.isBlank()) { + throw new IllegalArgumentException("storageKey must not be blank"); + } + if (sizeBytes < 0) { + throw new IllegalArgumentException("sizeBytes must not be negative"); + } + if (sha256 == null || !SHA256.matcher(sha256).matches()) { + throw new IllegalArgumentException("sha256 must be 64 lowercase hex characters"); + } + } + + /** + * Reports whether the payload store may already have reaped this object. + * + * @param now the current instant + * @return true once the retention window has passed + */ + public boolean isExpired(Instant now) { + return !now.isBefore(expiresAt); + } +} diff --git a/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/IdempotentMessageHandler.java b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/IdempotentMessageHandler.java new file mode 100644 index 00000000..478e6ff6 --- /dev/null +++ b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/IdempotentMessageHandler.java @@ -0,0 +1,33 @@ +package dev.caskeleton.messaging.reliability; + +import dev.caskeleton.messaging.api.delivery.HandleResult; +import dev.caskeleton.messaging.api.delivery.MessageDelivery; +import java.util.concurrent.CompletionStage; + +/** + * Runs a side effect at most once per consumer, whatever the broker redelivers. + * + *

{@code consumerName} is a parameter rather than a property of the message because the same + * message legitimately reaches several independent consumers, and each of them is entitled to apply + * its own effect once. Keying the inbox on the message id alone would let whichever consumer ran + * first suppress all the others. + * + *

A duplicate is not a failure. The handler returns success for a message it has already + * applied, so the broker settles it and stops redelivering; treating it as an error would park a + * message whose work is already done. + * + * @param the payload type + */ +public interface IdempotentMessageHandler { + + /** + * Applies an action once for a given consumer. + * + * @param consumerName the consumer identity the reservation is keyed on + * @param delivery the message being handled + * @param action the effect to apply inside the reservation's transaction + * @return a stage completing with the handler outcome + */ + CompletionStage handleOnce( + String consumerName, MessageDelivery delivery, TransactionalMessageAction action); +} diff --git a/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/InboxRecord.java b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/InboxRecord.java new file mode 100644 index 00000000..bfa16dc0 --- /dev/null +++ b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/InboxRecord.java @@ -0,0 +1,27 @@ +package dev.caskeleton.messaging.reliability; + +import dev.caskeleton.messaging.api.MessageId; +import java.time.Instant; +import java.util.Objects; + +/** + * One row of the consumer inbox. + * + *

Keyed by message id and consumer id, because two independent consumers of the same + * event must each process it once — deduplicating on the message alone would let the first consumer + * suppress the second. + * + * @param messageId the logical message identity + * @param consumerId the consumer's stable identity + * @param processedAt when the side effect committed + */ +public record InboxRecord(MessageId messageId, String consumerId, Instant processedAt) { + + public InboxRecord { + Objects.requireNonNull(messageId, "messageId must not be null"); + Objects.requireNonNull(processedAt, "processedAt must not be null"); + if (consumerId == null || consumerId.isBlank()) { + throw new IllegalArgumentException("consumerId must not be blank"); + } + } +} diff --git a/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/InboxRepository.java b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/InboxRepository.java new file mode 100644 index 00000000..7b124b1e --- /dev/null +++ b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/InboxRepository.java @@ -0,0 +1,46 @@ +package dev.caskeleton.messaging.reliability; + +import dev.caskeleton.messaging.api.MessageId; +import java.time.Instant; + +/** + * Storage for consumer-side deduplication. + * + *

{@link #reserve} must run inside the same database transaction as the handler's side effect. + * That is the entire mechanism: the uniqueness constraint on the inbox row and the business write + * commit together, so a redelivered message either finds the row already present and skips, or + * writes both. Reserving in a separate transaction reintroduces exactly the gap the Inbox exists to + * close. + */ +public interface InboxRepository { + + /** + * Reserves a message for processing inside the caller's transaction. + * + * @param messageId the logical message identity + * @param consumerId the consumer's stable identity + * @param now the current instant + * @return true when this is the first time; false when it was already processed + */ + boolean reserve(MessageId messageId, String consumerId, Instant now); + + /** + * Reports whether a message has already been processed by a consumer. + * + * @param messageId the logical message identity + * @param consumerId the consumer's stable identity + * @return true when already recorded + */ + boolean isProcessed(MessageId messageId, String consumerId); + + /** + * Deletes records older than a cutoff. + * + *

Retention must outlive the broker's maximum redelivery window, otherwise a late redelivery + * arrives after its inbox row was pruned and is processed a second time. + * + * @param processedBefore the retention cutoff + * @return how many rows were removed + */ + int purgeProcessedBefore(Instant processedBefore); +} diff --git a/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/InboxResult.java b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/InboxResult.java new file mode 100644 index 00000000..76d0befa --- /dev/null +++ b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/InboxResult.java @@ -0,0 +1,45 @@ +package dev.caskeleton.messaging.reliability; + +/** + * What the inbox decided about one delivery. + * + *

Three outcomes, not two. Collapsing {@link #ALREADY_APPLIED} and {@link #CLAIMED_ELSEWHERE} + * into a single "duplicate" would settle a message whose effect is still only half-written by + * another instance: if that instance then rolls back, the effect is lost and the broker will never + * redeliver, because this instance already acknowledged it. + */ +public enum InboxResult { + + /** No reservation existed; the effect ran inside this transaction. */ + APPLIED(true), + + /** + * A committed reservation already exists, so the effect ran before. + * + *

Settle the message: the work is done and redelivering it achieves nothing. + */ + ALREADY_APPLIED(true), + + /** + * Another instance holds an uncommitted reservation for this message. + * + *

Do not settle. The other transaction may still roll back, and this delivery is the + * only remaining copy that could re-apply the effect. + */ + CLAIMED_ELSEWHERE(false); + + private final boolean safeToSettle; + + InboxResult(boolean safeToSettle) { + this.safeToSettle = safeToSettle; + } + + /** + * Reports whether the source message may be settled on this outcome. + * + * @return true when settling cannot lose the effect + */ + public boolean isSafeToSettle() { + return safeToSettle; + } +} diff --git a/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxRecord.java b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxRecord.java new file mode 100644 index 00000000..058c4bb7 --- /dev/null +++ b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxRecord.java @@ -0,0 +1,131 @@ +package dev.caskeleton.messaging.reliability; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.time.Instant; +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * One row of the transactional outbox. + * + *

The whole point of the pattern is that this row is written in the same database + * transaction as the business change. Either both commit or neither does, which removes the window + * where a service updates its state and then dies before publishing. + * + *

What the outbox does not do is remove duplicates. A relay that cannot confirm a publish will + * retry it, and the same message may reach the broker twice. Effectively-once processing comes from + * this row carrying a stable {@code messageId} and the consumer having an Inbox — not from the + * outbox alone. + * + * @param messageId the stable logical identity, preserved across every retry + * @param destination the logical destination + * @param messageType the catalog message type + * @param schemaVersion the payload schema revision + * @param contentType the codec media type + * @param payload the encoded payload + * @param headers the reserved and application headers to publish + * @param createdAt when the business transaction wrote this row + * @param status the current lifecycle state + * @param attempts how many publish attempts have been made + * @param leaseExpiresAt when the current relay lease expires + * @param lastFailureCode the last sanitized failure code + */ +@SuppressWarnings("ArrayRecordComponent") +public record OutboxRecord( + MessageId messageId, + DestinationName destination, + MessageType messageType, + SchemaVersion schemaVersion, + ContentType contentType, + byte[] payload, + Map headers, + Instant createdAt, + OutboxStatus status, + int attempts, + Optional leaseExpiresAt, + Optional lastFailureCode) { + + public OutboxRecord { + Objects.requireNonNull(messageId, "messageId must not be null"); + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(messageType, "messageType must not be null"); + Objects.requireNonNull(schemaVersion, "schemaVersion must not be null"); + Objects.requireNonNull(contentType, "contentType must not be null"); + Objects.requireNonNull(payload, "payload must not be null"); + Objects.requireNonNull(headers, "headers must not be null"); + Objects.requireNonNull(createdAt, "createdAt must not be null"); + Objects.requireNonNull(status, "status must not be null"); + Objects.requireNonNull(leaseExpiresAt, "leaseExpiresAt must not be null"); + Objects.requireNonNull(lastFailureCode, "lastFailureCode must not be null"); + if (attempts < 0) { + throw new IllegalArgumentException("attempts must not be negative"); + } + payload = payload.clone(); + headers = Map.copyOf(headers); + } + + @Override + public byte[] payload() { + return payload.clone(); + } + + /** + * Returns a copy in a new lifecycle state. + * + *

The message id is never a parameter, so no state transition can change it. + * + * @param newStatus the new status + * @param newAttempts the updated attempt count + * @param failureCode the last sanitized failure code + * @return the updated record + */ + public OutboxRecord withStatus( + OutboxStatus newStatus, int newAttempts, Optional failureCode) { + return new OutboxRecord( + messageId, + destination, + messageType, + schemaVersion, + contentType, + payload, + headers, + createdAt, + newStatus, + newAttempts, + leaseExpiresAt, + failureCode); + } + + @Override + public boolean equals(Object other) { + return other instanceof OutboxRecord record + && messageId.equals(record.messageId) + && status == record.status + && attempts == record.attempts + && Arrays.equals(payload, record.payload); + } + + @Override + public int hashCode() { + return Objects.hash(messageId, status, attempts, Arrays.hashCode(payload)); + } + + @Override + public String toString() { + return "OutboxRecord[messageId=" + + messageId.value() + + ", destination=" + + destination.value() + + ", status=" + + status + + ", attempts=" + + attempts + + "]"; + } +} diff --git a/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxRepository.java b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxRepository.java new file mode 100644 index 00000000..fa94f243 --- /dev/null +++ b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxRepository.java @@ -0,0 +1,88 @@ +package dev.caskeleton.messaging.reliability; + +import dev.caskeleton.messaging.api.MessageId; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; + +/** + * Storage for the transactional outbox. + * + *

{@link #append} must be callable inside the caller's business transaction; every other method + * runs in the relay's own transaction. + */ +public interface OutboxRepository { + + /** + * Appends a record inside the caller's transaction. + * + * @param record the record to write + */ + void append(OutboxRecord record); + + /** + * Leases a batch of publishable records. + * + *

Leasing rather than simply selecting is what makes multiple relay instances safe: a record + * claimed by one relay is invisible to the others until its lease expires, so the same message is + * not published concurrently by two processes. + * + * @param batchSize how many records to claim + * @param leaseDuration how long the claim holds + * @param now the current instant + * @return the leased records, oldest first + */ + List leaseBatch(int batchSize, Duration leaseDuration, Instant now); + + /** + * Marks a record as confirmed by the broker. + * + * @param messageId the record identity + * @param now the current instant + */ + void markPublished(MessageId messageId, Instant now); + + /** + * Marks a record whose publish outcome is unknown. + * + *

The record stays eligible for retry under the same identity. + * + * @param messageId the record identity + * @param failureCode the sanitized failure code + * @param now the current instant + */ + void markAmbiguous(MessageId messageId, String failureCode, Instant now); + + /** + * Marks a record the broker definitively rejected. + * + * @param messageId the record identity + * @param failureCode the sanitized failure code + * @param now the current instant + */ + void markFailed(MessageId messageId, String failureCode, Instant now); + + /** + * Releases a lease without changing the record's outcome. + * + * @param messageId the record identity + */ + void releaseLease(MessageId messageId); + + /** + * Finds one record. + * + * @param messageId the record identity + * @return the record when present + */ + Optional find(MessageId messageId); + + /** + * Deletes records published before a cutoff. + * + * @param publishedBefore the retention cutoff + * @return how many rows were removed + */ + int purgePublishedBefore(Instant publishedBefore); +} diff --git a/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxStatus.java b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxStatus.java new file mode 100644 index 00000000..2fe183f8 --- /dev/null +++ b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxStatus.java @@ -0,0 +1,27 @@ +package dev.caskeleton.messaging.reliability; + +/** + * Where an outbox record is in its lifecycle. + * + *

{@link #AMBIGUOUS} is a distinct state rather than a flavour of failure. A record whose + * publish timed out may already be on the broker; retrying it is correct, but only under the same + * logical message id, and an operator looking at the table needs to be able to tell those rows + * apart from ones that definitely never landed. + */ +public enum OutboxStatus { + + /** Written by the business transaction, not yet picked up by the relay. */ + PENDING, + + /** Leased by a relay instance and being published. */ + IN_FLIGHT, + + /** Confirmed by the broker. */ + PUBLISHED, + + /** The publish outcome was unknown; the broker may hold it. */ + AMBIGUOUS, + + /** Definitively rejected; will not be retried without intervention. */ + FAILED +} diff --git a/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/ReliableMessagePublisher.java b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/ReliableMessagePublisher.java new file mode 100644 index 00000000..9eac6c70 --- /dev/null +++ b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/ReliableMessagePublisher.java @@ -0,0 +1,27 @@ +package dev.caskeleton.messaging.reliability; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.destination.MessageDestination; + +/** + * Enrols a message in the caller's database transaction instead of publishing it. + * + *

The return type is {@code void}, and that is the contract. There is no publish outcome to + * report yet: the row is written inside the caller's transaction, so if the transaction rolls back + * the message never existed, and if it commits the relay will publish it later. Handing back a + * {@code PublishResult} here would be a lie about work that has not happened. + * + *

This is the answer to the dual-write problem. Writing to the database and publishing to the + * broker in the same method cannot be made atomic; writing both to the database can. + */ +public interface ReliableMessagePublisher { + + /** + * Stages a message for publication when the current transaction commits. + * + * @param the payload type + * @param destination the logical destination + * @param message the envelope to publish, carrying the identity the relay will preserve + */ + void addToOutbox(MessageDestination destination, MessageEnvelope message); +} diff --git a/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/TransactionalMessageAction.java b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/TransactionalMessageAction.java new file mode 100644 index 00000000..5b5b1218 --- /dev/null +++ b/src/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/TransactionalMessageAction.java @@ -0,0 +1,29 @@ +package dev.caskeleton.messaging.reliability; + +import dev.caskeleton.messaging.api.delivery.MessageDelivery; + +/** + * The business work that runs inside the same transaction as the inbox reservation. + * + *

Sharing one transaction is the entire mechanism. If the effect committed separately from the + * "I have handled this message" marker, a crash between the two would either replay the effect or + * suppress a message that was never handled — and which of those you get would depend on the order + * the two commits happened to be written in. + * + *

Implementations must not settle the message, publish, or start their own transaction. The + * runtime owns the transaction boundary precisely so that the action cannot accidentally commit + * half of it. + * + * @param the payload type + */ +@FunctionalInterface +public interface TransactionalMessageAction { + + /** + * Applies the side effect for one delivery. + * + * @param delivery the message being handled + * @throws Exception when the effect fails, rolling back both it and the inbox reservation + */ + void apply(MessageDelivery delivery) throws Exception; +} diff --git a/src/messaging/messaging-schema-api/build.gradle b/src/messaging/messaging-schema-api/build.gradle new file mode 100644 index 00000000..95af0ac3 --- /dev/null +++ b/src/messaging/messaging-schema-api/build.gradle @@ -0,0 +1,5 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') +} diff --git a/src/messaging/messaging-schema-api/gradle.lockfile b/src/messaging/messaging-schema-api/gradle.lockfile new file mode 100644 index 00000000..599ff921 --- /dev/null +++ b/src/messaging/messaging-schema-api/gradle.lockfile @@ -0,0 +1,83 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty=compileClasspath,runtimeClasspath diff --git a/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/EncodedMessage.java b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/EncodedMessage.java new file mode 100644 index 00000000..51f5e422 --- /dev/null +++ b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/EncodedMessage.java @@ -0,0 +1,66 @@ +package dev.caskeleton.messaging.schema; + +import dev.caskeleton.messaging.api.ContentType; +import java.util.Arrays; +import java.util.Objects; +import java.util.Optional; + +/** + * An encoded payload together with the facts needed to decode it again. + * + *

The byte array is defensively copied on both construction and access. These bytes travel + * through retry, DLQ, and redrive paths where a shared mutable array would let one stage corrupt + * another's copy of the same logical message. + * + * @param bytes the encoded payload + * @param contentType the codec media type + * @param schemaReference the schema the payload was written against + */ +@SuppressWarnings("ArrayRecordComponent") +public record EncodedMessage( + byte[] bytes, ContentType contentType, Optional schemaReference) { + + public EncodedMessage { + Objects.requireNonNull(bytes, "encoded bytes must not be null"); + Objects.requireNonNull(contentType, "contentType must not be null"); + Objects.requireNonNull(schemaReference, "schemaReference must not be null"); + bytes = bytes.clone(); + } + + /** + * Returns a copy of the encoded bytes. + * + * @return the payload bytes + */ + @Override + public byte[] bytes() { + return bytes.clone(); + } + + /** + * Returns the encoded size without copying. + * + * @return the payload length in bytes + */ + public int size() { + return bytes.length; + } + + @Override + public boolean equals(Object other) { + return other instanceof EncodedMessage message + && Arrays.equals(bytes, message.bytes) + && contentType.equals(message.contentType) + && schemaReference.equals(message.schemaReference); + } + + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(bytes), contentType, schemaReference); + } + + @Override + public String toString() { + return "EncodedMessage[contentType=" + contentType.value() + ", size=" + bytes.length + "]"; + } +} diff --git a/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/MessageCodec.java b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/MessageCodec.java new file mode 100644 index 00000000..b51f1573 --- /dev/null +++ b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/MessageCodec.java @@ -0,0 +1,59 @@ +package dev.caskeleton.messaging.schema; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; + +/** + * Encodes and decodes payloads for a single wire format. + * + *

Implementations operate against a closed message-type registry. Accepting an unregistered type + * would let a producer introduce a wire contract nothing has reviewed, which is the same class of + * problem that makes Java serialization unsupported here. + */ +public interface MessageCodec { + + /** + * Returns the media type this codec produces. + * + * @return the content type + */ + ContentType contentType(); + + /** + * Encodes a payload. + * + * @param type the catalog message type + * @param version the schema revision + * @param payload the payload to encode + * @return the encoded message + */ + EncodedMessage encode(MessageType type, SchemaVersion version, Object payload); + + /** + * Decodes a payload from raw bytes. + * + * @param the payload type + * @param type the catalog message type + * @param version the schema revision + * @param encoded the encoded bytes + * @param payloadType the expected payload class + * @return the decoded payload + */ + T decode(MessageType type, SchemaVersion version, byte[] encoded, Class payloadType); + + /** + * Decodes a payload from an encoded message. + * + * @param the payload type + * @param type the catalog message type + * @param version the schema revision + * @param encoded the encoded message + * @param payloadType the expected payload class + * @return the decoded payload + */ + default T decode( + MessageType type, SchemaVersion version, EncodedMessage encoded, Class payloadType) { + return decode(type, version, encoded.bytes(), payloadType); + } +} diff --git a/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/MessageCodecRegistry.java b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/MessageCodecRegistry.java new file mode 100644 index 00000000..a237c325 --- /dev/null +++ b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/MessageCodecRegistry.java @@ -0,0 +1,26 @@ +package dev.caskeleton.messaging.schema; + +import dev.caskeleton.messaging.api.ContentType; +import java.util.Optional; + +/** Resolves the codec for a content type. */ +public interface MessageCodecRegistry { + + /** + * Returns the codec for a content type. + * + * @param contentType the media type + * @return the codec when registered + */ + Optional find(ContentType contentType); + + /** + * Returns the Stable default codec. + * + *

The raw bytes codec is never eligible: selecting it by default would silently disable schema + * validation for every destination that forgot to declare one. + * + * @return the default codec + */ + MessageCodec defaultCodec(); +} diff --git a/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/RawBytesMessageCodec.java b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/RawBytesMessageCodec.java new file mode 100644 index 00000000..30079ec5 --- /dev/null +++ b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/RawBytesMessageCodec.java @@ -0,0 +1,76 @@ +package dev.caskeleton.messaging.schema; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.error.MessageSerializationException; +import dev.caskeleton.messaging.api.error.MessageTooLargeException; +import java.util.Objects; +import java.util.Optional; + +/** + * The M2 escape hatch for payloads that carry no schema. + * + *

It still enforces the byte limit, and it is deliberately excluded from default codec + * selection: schema-free publishing has to be an explicit, auditable choice per destination, never + * something a destination falls back to because its codec was misconfigured. + */ +public final class RawBytesMessageCodec implements MessageCodec { + + /** The default encoded byte limit shared with the Stable codecs. */ + public static final int DEFAULT_MAX_BYTES = 1_048_576; + + private final int maxBytes; + + /** Creates a codec with the default one mebibyte limit. */ + public RawBytesMessageCodec() { + this(DEFAULT_MAX_BYTES); + } + + /** + * Creates a codec with an explicit byte limit. + * + * @param maxBytes the maximum encoded size + */ + public RawBytesMessageCodec(int maxBytes) { + if (maxBytes < 1) { + throw new IllegalArgumentException("maxBytes must be positive"); + } + this.maxBytes = maxBytes; + } + + @Override + public ContentType contentType() { + return ContentType.OCTET_STREAM; + } + + @Override + public EncodedMessage encode(MessageType type, SchemaVersion version, Object payload) { + Objects.requireNonNull(type, "messageType must not be null"); + Objects.requireNonNull(version, "schemaVersion must not be null"); + if (!(payload instanceof byte[] bytes)) { + throw new MessageSerializationException( + "RAW_BYTES_PAYLOAD_REQUIRED", "raw bytes codec accepts only byte[] payloads"); + } + if (bytes.length > maxBytes) { + throw new MessageTooLargeException( + "RAW_BYTES_TOO_LARGE", "encoded payload exceeds " + maxBytes + " bytes"); + } + return new EncodedMessage(bytes, ContentType.OCTET_STREAM, Optional.empty()); + } + + @Override + public T decode( + MessageType type, SchemaVersion version, byte[] encoded, Class payloadType) { + Objects.requireNonNull(encoded, "encoded bytes must not be null"); + if (!payloadType.equals(byte[].class)) { + throw new MessageSerializationException( + "RAW_BYTES_TARGET_REQUIRED", "raw bytes codec decodes only to byte[]"); + } + if (encoded.length > maxBytes) { + throw new MessageTooLargeException( + "RAW_BYTES_TOO_LARGE", "encoded payload exceeds " + maxBytes + " bytes"); + } + return payloadType.cast(encoded.clone()); + } +} diff --git a/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaCompatibility.java b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaCompatibility.java new file mode 100644 index 00000000..6aea0406 --- /dev/null +++ b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaCompatibility.java @@ -0,0 +1,32 @@ +package dev.caskeleton.messaging.schema; + +/** + * The compatibility mode a message type's schema evolution must satisfy. + * + *

Transitive modes check every historical version, not just the immediate predecessor. That + * matters for integration events, where a consumer may be several releases behind and a chain of + * individually-compatible changes can still be collectively breaking. + */ +public enum SchemaCompatibility { + + /** A new schema can read data written by the previous schema. */ + BACKWARD, + + /** A new schema can read data written by every previous schema. */ + BACKWARD_TRANSITIVE, + + /** The previous schema can read data written by the new schema. */ + FORWARD, + + /** Every previous schema can read data written by the new schema. */ + FORWARD_TRANSITIVE, + + /** Both directions hold against the previous schema. */ + FULL, + + /** Both directions hold against every previous schema. */ + FULL_TRANSITIVE, + + /** No compatibility is enforced; permitted only for M2 raw bytes. */ + NONE_EXPERIMENTAL +} diff --git a/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaCompatibilityValidator.java b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaCompatibilityValidator.java new file mode 100644 index 00000000..3fa0c5ed --- /dev/null +++ b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaCompatibilityValidator.java @@ -0,0 +1,113 @@ +package dev.caskeleton.messaging.schema; + +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.error.MessageSchemaIncompatibleException; +import java.util.List; +import java.util.Objects; + +/** + * The format-independent half of schema evolution: which versions have to be compared, and whether + * a mode is even allowed on a production destination. + * + *

Split from the per-format gates on purpose. Whether v3 must be checked against v1 as well as + * v2 is a property of the compatibility mode, not of Avro or Protobuf, and duplicating that + * reasoning in each codec is how the two formats drift apart. + * + *

{@link SchemaCompatibility#NONE_EXPERIMENTAL} is refused for production destinations. A mode + * that checks nothing is useful while a message type is being designed and actively dangerous once + * a retained log exists, because the log outlives every consumer that could still read it. + */ +public final class SchemaCompatibilityValidator { + + private final SchemaRegistry registry; + + /** + * Creates a validator over a registry. + * + * @param registry the schema source + */ + public SchemaCompatibilityValidator(SchemaRegistry registry) { + this.registry = Objects.requireNonNull(registry, "registry must not be null"); + } + + /** + * Returns the versions a candidate must be checked against, newest first. + * + *

Transitive modes return the whole history; pairwise modes return only the immediate + * predecessor; {@link SchemaCompatibility#NONE_EXPERIMENTAL} returns nothing. + * + * @param subject the registry subject + * @return the versions to compare against + */ + public List versionsToCheck(String subject) { + Objects.requireNonNull(subject, "subject must not be null"); + SchemaCompatibility mode = registry.compatibilityOf(subject); + if (mode == SchemaCompatibility.NONE_EXPERIMENTAL) { + return List.of(); + } + List history = registry.history(subject).reversed(); + if (history.isEmpty()) { + return List.of(); + } + return isTransitive(mode) ? history : List.of(history.get(0)); + } + + /** + * Refuses a compatibility mode that must not reach a production destination. + * + * @param subject the registry subject + * @param destination the logical destination name, used in the diagnostic + * @throws MessageSchemaIncompatibleException when the subject is unchecked + */ + public void requireProductionMode(String subject, String destination) { + Objects.requireNonNull(subject, "subject must not be null"); + Objects.requireNonNull(destination, "destination must not be null"); + if (registry.compatibilityOf(subject) == SchemaCompatibility.NONE_EXPERIMENTAL) { + throw new MessageSchemaIncompatibleException( + "UNCHECKED_SCHEMA_ON_PRODUCTION_DESTINATION", + "subject %s is NONE_EXPERIMENTAL and cannot back the production destination %s" + .formatted(subject, destination)); + } + } + + /** + * Reports whether readers must be able to read data written by older writers. + * + * @param mode the compatibility mode + * @return true when the backward direction is enforced + */ + public static boolean checksBackward(SchemaCompatibility mode) { + Objects.requireNonNull(mode, "mode must not be null"); + return mode == SchemaCompatibility.BACKWARD + || mode == SchemaCompatibility.BACKWARD_TRANSITIVE + || mode == SchemaCompatibility.FULL + || mode == SchemaCompatibility.FULL_TRANSITIVE; + } + + /** + * Reports whether older readers must be able to read data written by the candidate. + * + * @param mode the compatibility mode + * @return true when the forward direction is enforced + */ + public static boolean checksForward(SchemaCompatibility mode) { + Objects.requireNonNull(mode, "mode must not be null"); + return mode == SchemaCompatibility.FORWARD + || mode == SchemaCompatibility.FORWARD_TRANSITIVE + || mode == SchemaCompatibility.FULL + || mode == SchemaCompatibility.FULL_TRANSITIVE; + } + + /** + * Reports whether a mode compares against the whole history. + * + * @param mode the compatibility mode + * @return true for the transitive modes + */ + public static boolean isTransitive(SchemaCompatibility mode) { + Objects.requireNonNull(mode, "mode must not be null"); + return mode == SchemaCompatibility.BACKWARD_TRANSITIVE + || mode == SchemaCompatibility.FORWARD_TRANSITIVE + || mode == SchemaCompatibility.FULL_TRANSITIVE; + } +} diff --git a/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaReference.java b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaReference.java new file mode 100644 index 00000000..427c434f --- /dev/null +++ b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaReference.java @@ -0,0 +1,39 @@ +package dev.caskeleton.messaging.schema; + +import dev.caskeleton.messaging.api.SchemaVersion; +import java.net.URI; +import java.util.Objects; +import java.util.Optional; + +/** + * Points at the schema an encoded payload was written against. + * + *

Carried alongside the bytes rather than inferred from the message type, because a consumer + * that guesses the schema from the type is exactly the consumer that breaks when a producer rolls + * forward. + * + * @param subject the registry subject or schema name + * @param version the schema revision + * @param schemaUri the resolvable schema location, when one exists + */ +public record SchemaReference(String subject, SchemaVersion version, Optional schemaUri) { + + public SchemaReference { + Objects.requireNonNull(version, "schema version must not be null"); + Objects.requireNonNull(schemaUri, "schemaUri must not be null"); + if (subject == null || subject.isBlank()) { + throw new IllegalArgumentException("schema subject must not be blank"); + } + } + + /** + * Builds a reference with no resolvable location. + * + * @param subject the registry subject + * @param version the schema revision + * @return a schema reference + */ + public static SchemaReference of(String subject, SchemaVersion version) { + return new SchemaReference(subject, version, Optional.empty()); + } +} diff --git a/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaRegistry.java b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaRegistry.java new file mode 100644 index 00000000..49f39b3b --- /dev/null +++ b/src/messaging/messaging-schema-api/src/main/java/dev/caskeleton/messaging/schema/SchemaRegistry.java @@ -0,0 +1,56 @@ +package dev.caskeleton.messaging.schema; + +import dev.caskeleton.messaging.api.SchemaVersion; +import java.util.List; +import java.util.Optional; + +/** + * Resolves schemas by subject and version, and records the compatibility mode each subject is held + * to. + * + *

Deliberately a port. A hosted registry, a classpath directory of schema files, and a static + * in-process map are all legitimate sources depending on the deployment, and the platform's + * compatibility rules must hold identically across all three. Binding to a vendor client here would + * make the rules untestable without that vendor running. + * + *

{@link #history} returns oldest first. Transitive compatibility checks read the whole list, so + * an ordering mistake here silently converts a transitive check into a pairwise one. + */ +public interface SchemaRegistry { + + /** + * Resolves one schema definition. + * + * @param subject the registry subject + * @param version the schema revision + * @return the schema text, or empty when the subject or version is unknown + */ + Optional lookup(String subject, SchemaVersion version); + + /** + * Returns every registered version of a subject, oldest first. + * + * @param subject the registry subject + * @return the version history, empty when the subject is unknown + */ + List history(String subject); + + /** + * Returns the compatibility mode enforced for a subject. + * + * @param subject the registry subject + * @return the subject's compatibility mode + */ + SchemaCompatibility compatibilityOf(String subject); + + /** + * Returns the newest registered version of a subject. + * + * @param subject the registry subject + * @return the latest version, or empty when the subject is unknown + */ + default Optional latest(String subject) { + List versions = history(subject); + return versions.isEmpty() ? Optional.empty() : Optional.of(versions.get(versions.size() - 1)); + } +} diff --git a/src/messaging/messaging-schema-api/src/test/java/dev/caskeleton/messaging/schema/RawBytesMessageCodecTest.java b/src/messaging/messaging-schema-api/src/test/java/dev/caskeleton/messaging/schema/RawBytesMessageCodecTest.java new file mode 100644 index 00000000..69531deb --- /dev/null +++ b/src/messaging/messaging-schema-api/src/test/java/dev/caskeleton/messaging/schema/RawBytesMessageCodecTest.java @@ -0,0 +1,67 @@ +package dev.caskeleton.messaging.schema; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.error.MessageSerializationException; +import dev.caskeleton.messaging.api.error.MessageTooLargeException; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class RawBytesMessageCodecTest { + + private static final MessageType OPAQUE = new MessageType("opaque.blob"); + private static final SchemaVersion V1 = new SchemaVersion(1); + + private final RawBytesMessageCodec codec = new RawBytesMessageCodec(); + + @Test + void roundTripsOpaqueBytes() { + byte[] payload = {1, 2, 3, 4}; + + EncodedMessage encoded = codec.encode(OPAQUE, V1, payload); + + assertThat(codec.decode(OPAQUE, V1, encoded.bytes(), byte[].class)).containsExactly(1, 2, 3, 4); + } + + @Test + void advertisesOpaqueContentTypeAndNoSchemaReference() { + EncodedMessage encoded = codec.encode(OPAQUE, V1, new byte[] {9}); + + assertThat(codec.contentType()).isEqualTo(ContentType.OCTET_STREAM); + assertThat(encoded.schemaReference()).isEmpty(); + } + + @Test + void refusesNonByteArrayPayloads() { + assertThatThrownBy(() -> codec.encode(OPAQUE, V1, "text")) + .isInstanceOf(MessageSerializationException.class); + } + + @Test + void refusesDecodingToAnythingButByteArray() { + assertThatThrownBy(() -> codec.decode(OPAQUE, V1, new byte[] {1}, String.class)) + .isInstanceOf(MessageSerializationException.class); + } + + @Test + void enforcesTheByteLimit() { + RawBytesMessageCodec small = new RawBytesMessageCodec(4); + + assertThatThrownBy(() -> small.encode(OPAQUE, V1, new byte[5])) + .isInstanceOf(MessageTooLargeException.class); + } + + @Test + void encodedMessageDefensivelyCopiesItsBytes() { + byte[] source = {1, 2, 3}; + EncodedMessage encoded = new EncodedMessage(source, ContentType.OCTET_STREAM, Optional.empty()); + + source[0] = 99; + + assertThat(encoded.bytes()).containsExactly(1, 2, 3); + } +} diff --git a/src/messaging/messaging-schema-api/src/test/java/dev/caskeleton/messaging/schema/SchemaCompatibilityValidatorTest.java b/src/messaging/messaging-schema-api/src/test/java/dev/caskeleton/messaging/schema/SchemaCompatibilityValidatorTest.java new file mode 100644 index 00000000..7976c059 --- /dev/null +++ b/src/messaging/messaging-schema-api/src/test/java/dev/caskeleton/messaging/schema/SchemaCompatibilityValidatorTest.java @@ -0,0 +1,109 @@ +package dev.caskeleton.messaging.schema; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.error.MessageSchemaIncompatibleException; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class SchemaCompatibilityValidatorTest { + + private static final SchemaVersion V1 = new SchemaVersion(1); + private static final SchemaVersion V2 = new SchemaVersion(2); + private static final SchemaVersion V3 = new SchemaVersion(3); + + /** A registry whose history is deliberately oldest-first, as the port documents. */ + private record FixedRegistry(List versions, SchemaCompatibility mode) + implements SchemaRegistry { + + @Override + public Optional lookup(String subject, SchemaVersion version) { + return versions.contains(version) ? Optional.of("{}") : Optional.empty(); + } + + @Override + public List history(String subject) { + return versions; + } + + @Override + public SchemaCompatibility compatibilityOf(String subject) { + return mode; + } + } + + private static SchemaCompatibilityValidator validator(SchemaCompatibility mode) { + return new SchemaCompatibilityValidator(new FixedRegistry(List.of(V1, V2, V3), mode)); + } + + @Test + void aPairwiseModeChecksOnlyTheImmediatePredecessor() { + assertThat(validator(SchemaCompatibility.BACKWARD).versionsToCheck("order.created")) + .containsExactly(V3); + } + + @Test + void aTransitiveModeChecksTheWholeHistoryNewestFirst() { + assertThat(validator(SchemaCompatibility.BACKWARD_TRANSITIVE).versionsToCheck("order.created")) + .as( + "a chain of individually compatible changes can still break a consumer several releases behind") + .containsExactly(V3, V2, V1); + } + + @Test + void anUncheckedModeComparesAgainstNothing() { + assertThat(validator(SchemaCompatibility.NONE_EXPERIMENTAL).versionsToCheck("order.created")) + .isEmpty(); + } + + @Test + void anEmptyHistoryHasNothingToCompareAgainst() { + SchemaCompatibilityValidator validator = + new SchemaCompatibilityValidator( + new FixedRegistry(List.of(), SchemaCompatibility.FULL_TRANSITIVE)); + + assertThat(validator.versionsToCheck("order.created")).isEmpty(); + } + + @Test + void anUncheckedSubjectCannotBackAProductionDestination() { + assertThatThrownBy( + () -> + validator(SchemaCompatibility.NONE_EXPERIMENTAL) + .requireProductionMode("order.created", "orders.v1")) + .isInstanceOf(MessageSchemaIncompatibleException.class) + .hasMessageContaining("orders.v1"); + } + + @Test + void aCheckedSubjectMayBackAProductionDestination() { + assertThatCode( + () -> + validator(SchemaCompatibility.FULL) + .requireProductionMode("order.created", "orders.v1")) + .doesNotThrowAnyException(); + } + + @Test + void fullModeEnforcesBothDirections() { + assertThat(SchemaCompatibilityValidator.checksBackward(SchemaCompatibility.FULL)).isTrue(); + assertThat(SchemaCompatibilityValidator.checksForward(SchemaCompatibility.FULL)).isTrue(); + } + + @Test + void eachOneDirectionalModeEnforcesOnlyItsOwnDirection() { + assertThat(SchemaCompatibilityValidator.checksForward(SchemaCompatibility.BACKWARD)).isFalse(); + assertThat(SchemaCompatibilityValidator.checksBackward(SchemaCompatibility.FORWARD)).isFalse(); + } + + @Test + void theLatestVersionIsTheNewestNotTheFirstListed() { + SchemaRegistry registry = new FixedRegistry(List.of(V1, V2, V3), SchemaCompatibility.FULL); + + assertThat(registry.latest("order.created")).hasValue(V3); + } +} diff --git a/src/messaging/messaging-schema-avro/build.gradle b/src/messaging/messaging-schema-avro/build.gradle new file mode 100644 index 00000000..22844df5 --- /dev/null +++ b/src/messaging/messaging-schema-avro/build.gradle @@ -0,0 +1,8 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-schema-api') + + implementation 'org.apache.avro:avro:1.12.0' +} diff --git a/src/messaging/messaging-schema-avro/gradle.lockfile b/src/messaging/messaging-schema-avro/gradle.lockfile new file mode 100644 index 00000000..46c2ee7a --- /dev/null +++ b/src/messaging/messaging-schema-avro/gradle.lockfile @@ -0,0 +1,91 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.fasterxml.jackson.core:jackson-annotations:2.20=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-core:2.20.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.20.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson:jackson-bom:2.20.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-codec:commons-codec:1.19.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.16.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.avro:avro:1.12.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-compress:1.26.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-lang3:3.20.0=checkstyle,compileClasspath,runtimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=compileClasspath,runtimeClasspath,spotbugs,spotbugsSlf4j,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty= diff --git a/src/messaging/messaging-schema-avro/src/main/java/dev/caskeleton/messaging/schema/avro/AvroCompatibilityGate.java b/src/messaging/messaging-schema-avro/src/main/java/dev/caskeleton/messaging/schema/avro/AvroCompatibilityGate.java new file mode 100644 index 00000000..853c33db --- /dev/null +++ b/src/messaging/messaging-schema-avro/src/main/java/dev/caskeleton/messaging/schema/avro/AvroCompatibilityGate.java @@ -0,0 +1,73 @@ +package dev.caskeleton.messaging.schema.avro; + +import dev.caskeleton.messaging.api.error.MessageSchemaIncompatibleException; +import dev.caskeleton.messaging.schema.SchemaCompatibility; +import java.util.List; +import java.util.Objects; +import org.apache.avro.Schema; +import org.apache.avro.SchemaCompatibility.SchemaPairCompatibility; + +/** + * Refuses an Avro schema change that would break the destination's declared compatibility mode. + * + *

Run in CI rather than at runtime. By the time a producer has published one incompatible + * record, the damage is durable: the record sits in a retained log that every current and future + * consumer must be able to read. The gate therefore checks the whole declared history for the + * transitive modes, not just the immediate predecessor, because a chain of individually compatible + * changes can still be collectively breaking for a consumer several releases behind. + */ +public final class AvroCompatibilityGate { + + /** + * Checks a candidate schema against the registered history. + * + * @param candidate the new schema + * @param history the previously registered schemas, newest first + * @param mode the destination's compatibility mode + * @throws MessageSchemaIncompatibleException when the change is not permitted + */ + public void check(Schema candidate, List history, SchemaCompatibility mode) { + Objects.requireNonNull(candidate, "candidate must not be null"); + Objects.requireNonNull(history, "history must not be null"); + Objects.requireNonNull(mode, "mode must not be null"); + + if (mode == SchemaCompatibility.NONE_EXPERIMENTAL || history.isEmpty()) { + return; + } + List checked = isTransitive(mode) ? history : history.subList(0, 1); + + for (Schema previous : checked) { + if (readsBackward(mode)) { + requireCompatible(candidate, previous, "backward"); + } + if (readsForward(mode)) { + requireCompatible(previous, candidate, "forward"); + } + } + } + + private static boolean isTransitive(SchemaCompatibility mode) { + return mode == SchemaCompatibility.BACKWARD_TRANSITIVE + || mode == SchemaCompatibility.FORWARD_TRANSITIVE + || mode == SchemaCompatibility.FULL_TRANSITIVE; + } + + private static boolean readsBackward(SchemaCompatibility mode) { + return mode != SchemaCompatibility.FORWARD && mode != SchemaCompatibility.FORWARD_TRANSITIVE; + } + + private static boolean readsForward(SchemaCompatibility mode) { + return mode != SchemaCompatibility.BACKWARD && mode != SchemaCompatibility.BACKWARD_TRANSITIVE; + } + + private static void requireCompatible(Schema reader, Schema writer, String direction) { + SchemaPairCompatibility result = + org.apache.avro.SchemaCompatibility.checkReaderWriterCompatibility(reader, writer); + if (result.getType() + != org.apache.avro.SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE) { + throw new MessageSchemaIncompatibleException( + "AVRO_" + direction.toUpperCase(java.util.Locale.ROOT) + "_INCOMPATIBLE", + "the candidate Avro schema is not " + direction + " compatible"); + } + } +} diff --git a/src/messaging/messaging-schema-avro/src/main/java/dev/caskeleton/messaging/schema/avro/AvroMessageCodec.java b/src/messaging/messaging-schema-avro/src/main/java/dev/caskeleton/messaging/schema/avro/AvroMessageCodec.java new file mode 100644 index 00000000..3f2d6106 --- /dev/null +++ b/src/messaging/messaging-schema-avro/src/main/java/dev/caskeleton/messaging/schema/avro/AvroMessageCodec.java @@ -0,0 +1,200 @@ +package dev.caskeleton.messaging.schema.avro; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.error.MessageSerializationException; +import dev.caskeleton.messaging.api.error.MessageTooLargeException; +import dev.caskeleton.messaging.api.error.MessageValidationException; +import dev.caskeleton.messaging.schema.EncodedMessage; +import dev.caskeleton.messaging.schema.MessageCodec; +import dev.caskeleton.messaging.schema.SchemaReference; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; + +/** + * The optional Avro codec, bound to a closed registry of writer schemas. + * + *

Decoding uses an explicit writer schema and reader schema pair. Avro binary carries no schema + * of its own, so decoding with the wrong schema does not fail — it produces plausible garbage. The + * registry is what makes the writer schema knowable, and passing both schemas to the reader is what + * makes evolution work: Avro resolves added, removed, and defaulted fields only when it can see + * both sides. + * + *

Single-object encoding without a header is used deliberately. The framing that would carry a + * schema fingerprint belongs to the transport headers, where the platform already carries schema + * identity for every format, rather than being duplicated inside the Avro payload for this one + * format. + */ +public final class AvroMessageCodec implements MessageCodec { + + private static final int DEFAULT_MAX_BYTES = 1_048_576; + + private final Map> schemas; + private final int maxBytes; + + /** + * Creates a codec over an explicit schema registry. + * + * @param schemas each registered message type's schema, by version + */ + public AvroMessageCodec(Map> schemas) { + this(schemas, DEFAULT_MAX_BYTES); + } + + /** + * Creates a codec over an explicit schema registry and byte limit. + * + * @param schemas each registered message type's schema, by version + * @param maxBytes the maximum encoded size + */ + public AvroMessageCodec(Map> schemas, int maxBytes) { + Objects.requireNonNull(schemas, "schemas must not be null"); + if (maxBytes < 1) { + throw new IllegalArgumentException("maxBytes must be positive"); + } + this.schemas = Map.copyOf(schemas); + this.maxBytes = maxBytes; + } + + @Override + public ContentType contentType() { + return ContentType.AVRO; + } + + @Override + public EncodedMessage encode(MessageType type, SchemaVersion version, Object payload) { + Objects.requireNonNull(payload, "payload must not be null"); + Schema schema = schemaFor(type, version); + + if (!(payload instanceof GenericRecord record)) { + throw new MessageValidationException( + "AVRO_PAYLOAD_NOT_A_RECORD", + "the Avro codec encodes GenericRecord payloads, not " + + payload.getClass().getSimpleName()); + } + if (!schema.equals(record.getSchema())) { + throw new MessageValidationException( + "AVRO_SCHEMA_MISMATCH", + "the payload's schema does not match the registered schema for %s v%d" + .formatted(type.value(), version.value())); + } + + ByteArrayOutputStream sink = new ByteArrayOutputStream(); + BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(sink, null); + DatumWriter writer = new GenericDatumWriter<>(schema); + try { + writer.write(record, encoder); + encoder.flush(); + } catch (IOException | RuntimeException failure) { + throw new MessageSerializationException( + "AVRO_ENCODE_FAILED", "the payload could not be encoded as Avro", failure); + } + + byte[] bytes = sink.toByteArray(); + if (bytes.length > maxBytes) { + throw new MessageTooLargeException( + "AVRO_PAYLOAD_TOO_LARGE", + "the encoded payload of %d bytes exceeds the %d byte limit" + .formatted(bytes.length, maxBytes)); + } + return new EncodedMessage( + bytes, ContentType.AVRO, Optional.of(SchemaReference.of(type.value(), version))); + } + + @Override + public T decode( + MessageType type, SchemaVersion version, byte[] encoded, Class payloadType) { + Objects.requireNonNull(encoded, "encoded must not be null"); + Objects.requireNonNull(payloadType, "payloadType must not be null"); + if (encoded.length > maxBytes) { + throw new MessageTooLargeException( + "AVRO_PAYLOAD_TOO_LARGE", + "the encoded payload of %d bytes exceeds the %d byte limit" + .formatted(encoded.length, maxBytes)); + } + Schema writerSchema = schemaFor(type, version); + + if (!payloadType.isAssignableFrom(GenericRecord.class) + && !GenericRecord.class.isAssignableFrom(payloadType)) { + throw new MessageValidationException( + "AVRO_PAYLOAD_NOT_A_RECORD", + "the Avro codec decodes into GenericRecord, not " + payloadType.getSimpleName()); + } + + DatumReader reader = new GenericDatumReader<>(writerSchema, writerSchema); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(encoded, null); + try { + return payloadType.cast(reader.read(null, decoder)); + } catch (IOException | RuntimeException failure) { + throw new MessageSerializationException( + "AVRO_DECODE_FAILED", + "the payload could not be decoded against the registered schema", + failure); + } + } + + /** + * Decodes using a reader schema that differs from the writer schema. + * + *

This is where Avro evolution actually happens: the reader schema is the consumer's view and + * the writer schema is what the producer wrote. Both are required — resolving with only one of + * them silently drops added fields or fails on removed ones. + * + * @param writerVersion the version the producer wrote + * @param readerVersion the version the consumer expects + * @param type the catalog message type + * @param encoded the encoded bytes + * @return the resolved record + */ + public GenericRecord decodeEvolved( + MessageType type, SchemaVersion writerVersion, SchemaVersion readerVersion, byte[] encoded) { + Objects.requireNonNull(encoded, "encoded must not be null"); + Schema writerSchema = schemaFor(type, writerVersion); + Schema readerSchema = schemaFor(type, readerVersion); + + DatumReader reader = new GenericDatumReader<>(writerSchema, readerSchema); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(encoded, null); + try { + return reader.read(null, decoder); + } catch (IOException | RuntimeException failure) { + throw new MessageSerializationException( + "AVRO_EVOLUTION_FAILED", + "v%d bytes could not be resolved against the v%d reader schema" + .formatted(writerVersion.value(), readerVersion.value()), + failure); + } + } + + private Schema schemaFor(MessageType type, SchemaVersion version) { + Objects.requireNonNull(type, "type must not be null"); + Objects.requireNonNull(version, "version must not be null"); + Map versions = schemas.get(type); + if (versions == null) { + throw new MessageValidationException( + "AVRO_TYPE_NOT_REGISTERED", + "message type %s is not in the Avro registry".formatted(type.value())); + } + Schema schema = versions.get(version); + if (schema == null) { + throw new MessageValidationException( + "AVRO_VERSION_NOT_REGISTERED", + "schema version %d of %s is not in the Avro registry" + .formatted(version.value(), type.value())); + } + return schema; + } +} diff --git a/src/messaging/messaging-schema-avro/src/test/java/dev/caskeleton/messaging/schema/avro/AvroCompatibilityTest.java b/src/messaging/messaging-schema-avro/src/test/java/dev/caskeleton/messaging/schema/avro/AvroCompatibilityTest.java new file mode 100644 index 00000000..344990dc --- /dev/null +++ b/src/messaging/messaging-schema-avro/src/test/java/dev/caskeleton/messaging/schema/avro/AvroCompatibilityTest.java @@ -0,0 +1,178 @@ +package dev.caskeleton.messaging.schema.avro; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.error.MessageSchemaIncompatibleException; +import dev.caskeleton.messaging.api.error.MessageValidationException; +import dev.caskeleton.messaging.schema.EncodedMessage; +import dev.caskeleton.messaging.schema.SchemaCompatibility; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.junit.jupiter.api.Test; + +class AvroCompatibilityTest { + + private static final MessageType ORDER_CREATED = new MessageType("order.created"); + private static final SchemaVersion V1 = new SchemaVersion(1); + private static final SchemaVersion V2 = new SchemaVersion(2); + + private static final Schema V1_SCHEMA = loadV1(); + + /** + * v2 adds a defaulted field, which is the only shape of addition that stays readable both ways. + */ + private static final Schema V2_SCHEMA = + new Schema.Parser() + .parse( + """ + { + "type": "record", + "name": "OrderCreated", + "namespace": "dev.caskeleton.messaging.sample", + "fields": [ + { "name": "orderId", "type": "string" }, + { "name": "customerId", "type": "string" }, + { "name": "totalMinorUnits", "type": "long" }, + { "name": "currency", "type": "string" }, + { "name": "channel", "type": "string", "default": "WEB" } + ] + } + """); + + /** v2-breaking adds a field with no default, which an old writer can never supply. */ + private static final Schema V2_WITHOUT_DEFAULT = + new Schema.Parser() + .parse( + """ + { + "type": "record", + "name": "OrderCreated", + "namespace": "dev.caskeleton.messaging.sample", + "fields": [ + { "name": "orderId", "type": "string" }, + { "name": "customerId", "type": "string" }, + { "name": "totalMinorUnits", "type": "long" }, + { "name": "currency", "type": "string" }, + { "name": "channel", "type": "string" } + ] + } + """); + + private static Schema loadV1() { + try (InputStream stream = + AvroCompatibilityTest.class.getResourceAsStream("/schemas/order.created/v1.avsc")) { + if (stream == null) { + throw new IllegalStateException("the v1 schema fixture is missing from the classpath"); + } + return new Schema.Parser().parse(new String(stream.readAllBytes(), StandardCharsets.UTF_8)); + } catch (IOException failure) { + throw new IllegalStateException("could not read the v1 schema fixture", failure); + } + } + + private static AvroMessageCodec codec() { + return new AvroMessageCodec(Map.of(ORDER_CREATED, Map.of(V1, V1_SCHEMA, V2, V2_SCHEMA))); + } + + private static GenericRecord v1Record() { + GenericRecord record = new GenericData.Record(V1_SCHEMA); + record.put("orderId", "o-1"); + record.put("customerId", "c-1"); + record.put("totalMinorUnits", 12_500L); + record.put("currency", "KRW"); + return record; + } + + @Test + void aRoundTripPreservesEveryField() { + AvroMessageCodec codec = codec(); + + EncodedMessage encoded = codec.encode(ORDER_CREATED, V1, v1Record()); + GenericRecord decoded = codec.decode(ORDER_CREATED, V1, encoded.bytes(), GenericRecord.class); + + assertThat(decoded.get("orderId")).hasToString("o-1"); + assertThat(decoded.get("totalMinorUnits")).isEqualTo(12_500L); + assertThat(encoded.contentType()).isEqualTo(ContentType.AVRO); + } + + @Test + void theEncodedMessageCarriesItsSchemaReference() { + EncodedMessage encoded = codec().encode(ORDER_CREATED, V1, v1Record()); + + assertThat(encoded.schemaReference()) + .as("Avro binary carries no schema of its own, so identity has to travel beside it") + .hasValueSatisfying( + reference -> { + assertThat(reference.subject()).isEqualTo("order.created"); + assertThat(reference.version()).isEqualTo(V1); + }); + } + + @Test + void aV1WriterIsReadableByAV2ReaderThroughTheDefault() { + AvroMessageCodec codec = codec(); + EncodedMessage encoded = codec.encode(ORDER_CREATED, V1, v1Record()); + + GenericRecord evolved = codec.decodeEvolved(ORDER_CREATED, V1, V2, encoded.bytes()); + + assertThat(evolved.get("channel")) + .as("the reader schema's default is what makes the added field readable") + .hasToString("WEB"); + } + + @Test + void addingADefaultedFieldIsBackwardCompatible() { + assertThatCode( + () -> + new AvroCompatibilityGate() + .check(V2_SCHEMA, List.of(V1_SCHEMA), SchemaCompatibility.BACKWARD)) + .doesNotThrowAnyException(); + } + + @Test + void addingAFieldWithoutADefaultIsRejected() { + assertThatThrownBy( + () -> + new AvroCompatibilityGate() + .check(V2_WITHOUT_DEFAULT, List.of(V1_SCHEMA), SchemaCompatibility.BACKWARD)) + .isInstanceOf(MessageSchemaIncompatibleException.class); + } + + @Test + void aPayloadWhoseSchemaDiffersFromTheRegisteredOneIsRejectedBeforeEncoding() { + GenericRecord wrongShape = new GenericData.Record(V2_SCHEMA); + wrongShape.put("orderId", "o-1"); + wrongShape.put("customerId", "c-1"); + wrongShape.put("totalMinorUnits", 1L); + wrongShape.put("currency", "KRW"); + wrongShape.put("channel", "APP"); + + assertThatThrownBy(() -> codec().encode(ORDER_CREATED, V1, wrongShape)) + .as("encoding v2 data under the v1 version would produce bytes nothing can decode") + .isInstanceOf(MessageValidationException.class); + } + + @Test + void anUnregisteredVersionIsRejectedRatherThanGuessed() { + assertThatThrownBy(() -> codec().encode(ORDER_CREATED, new SchemaVersion(9), v1Record())) + .isInstanceOf(MessageValidationException.class) + .hasMessageContaining("not in the Avro registry"); + } + + @Test + void anUnregisteredTypeIsRejected() { + assertThatThrownBy(() -> codec().encode(new MessageType("unknown.event"), V1, v1Record())) + .isInstanceOf(MessageValidationException.class); + } +} diff --git a/src/messaging/messaging-schema-avro/src/test/resources/schemas/order.created/v1.avsc b/src/messaging/messaging-schema-avro/src/test/resources/schemas/order.created/v1.avsc new file mode 100644 index 00000000..2d12c31e --- /dev/null +++ b/src/messaging/messaging-schema-avro/src/test/resources/schemas/order.created/v1.avsc @@ -0,0 +1,11 @@ +{ + "type": "record", + "name": "OrderCreated", + "namespace": "dev.caskeleton.messaging.sample", + "fields": [ + { "name": "orderId", "type": "string" }, + { "name": "customerId", "type": "string" }, + { "name": "totalMinorUnits", "type": "long" }, + { "name": "currency", "type": "string" } + ] +} diff --git a/src/messaging/messaging-schema-json/build.gradle b/src/messaging/messaging-schema-json/build.gradle new file mode 100644 index 00000000..982f2efb --- /dev/null +++ b/src/messaging/messaging-schema-json/build.gradle @@ -0,0 +1,8 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-schema-api') + + implementation 'tools.jackson.core:jackson-databind' +} diff --git a/src/messaging/messaging-schema-json/gradle.lockfile b/src/messaging/messaging-schema-json/gradle.lockfile new file mode 100644 index 00000000..f0c349e9 --- /dev/null +++ b/src/messaging/messaging-schema-json/gradle.lockfile @@ -0,0 +1,87 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.fasterxml.jackson.core:jackson-annotations:2.20=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +tools.jackson.core:jackson-core:3.0.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +tools.jackson.core:jackson-databind:3.0.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +tools.jackson:jackson-bom:3.0.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +empty= diff --git a/src/messaging/messaging-schema-json/src/main/java/dev/caskeleton/messaging/schema/json/JacksonMessageCodec.java b/src/messaging/messaging-schema-json/src/main/java/dev/caskeleton/messaging/schema/json/JacksonMessageCodec.java new file mode 100644 index 00000000..f6294b6d --- /dev/null +++ b/src/messaging/messaging-schema-json/src/main/java/dev/caskeleton/messaging/schema/json/JacksonMessageCodec.java @@ -0,0 +1,176 @@ +package dev.caskeleton.messaging.schema.json; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.error.MessageSerializationException; +import dev.caskeleton.messaging.api.error.MessageTooLargeException; +import dev.caskeleton.messaging.api.error.MessageValidationException; +import dev.caskeleton.messaging.schema.EncodedMessage; +import dev.caskeleton.messaging.schema.MessageCodec; +import dev.caskeleton.messaging.schema.SchemaReference; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import tools.jackson.core.JacksonException; +import tools.jackson.core.StreamReadConstraints; +import tools.jackson.core.StreamReadFeature; +import tools.jackson.core.json.JsonFactory; +import tools.jackson.databind.DeserializationFeature; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; + +/** + * The Stable JSON codec. + * + *

Three things make this safe to expose as the default. The message-type registry is closed, so + * a payload class only becomes reachable when someone registered it. The parser is constrained on + * depth, document length, and duplicate keys, so a hostile document cannot exhaust the consumer + * before the handler ever runs. And the encoded size is checked against the destination limit here + * rather than at the broker, so an oversized payload fails locally with {@code NOT_TRANSMITTED} + * evidence instead of ambiguously mid-flight. + * + *

Polymorphic default typing is never enabled. It is the mechanism behind most JSON + * deserialization gadget chains, and no legitimate message contract needs it. + */ +public final class JacksonMessageCodec implements MessageCodec { + + /** The portability limit shared with the other Stable codecs. */ + public static final int DEFAULT_MAX_BYTES = 1_048_576; + + /** The parser nesting depth ceiling. */ + public static final int MAX_NESTING_DEPTH = 100; + + private static final int MAX_STRING_CHARACTERS = 5_000_000; + private static final int MAX_NUMBER_DIGITS = 1_000; + + private final ObjectMapper mapper; + private final Map> registry; + private final int maxBytes; + + private JacksonMessageCodec(Map> registry, int maxBytes) { + this.registry = Map.copyOf(registry); + this.maxBytes = maxBytes; + this.mapper = strictMapper(maxBytes); + } + + /** + * Creates a codec over an explicit message-type registry. + * + * @param registry the closed map of message type to payload class + * @return a codec with the default one mebibyte limit + */ + public static JacksonMessageCodec of(Map> registry) { + return new JacksonMessageCodec(registry, DEFAULT_MAX_BYTES); + } + + /** + * Creates a codec over an explicit registry and byte limit. + * + * @param registry the closed map of message type to payload class + * @param maxBytes the maximum encoded size + * @return a codec + */ + public static JacksonMessageCodec of(Map> registry, int maxBytes) { + return new JacksonMessageCodec(registry, maxBytes); + } + + /** + * Creates a single-entry codec for focused tests. + * + * @param type the catalog message type + * @param payloadType the payload class + * @return a codec registering exactly one type + */ + public static JacksonMessageCodec testingDefault(MessageType type, Class payloadType) { + return new JacksonMessageCodec(Map.of(type, payloadType), DEFAULT_MAX_BYTES); + } + + @Override + public ContentType contentType() { + return ContentType.JSON; + } + + @Override + public EncodedMessage encode(MessageType type, SchemaVersion version, Object payload) { + Class registered = requireRegistered(type); + Objects.requireNonNull(version, "schemaVersion must not be null"); + Objects.requireNonNull(payload, "payload must not be null"); + if (!registered.isInstance(payload)) { + throw new MessageValidationException( + "PAYLOAD_TYPE_MISMATCH", + "payload does not match the registered type for " + type.value()); + } + + byte[] bytes; + try { + bytes = mapper.writeValueAsBytes(payload); + } catch (JacksonException exception) { + throw new MessageSerializationException( + "JSON_ENCODE_FAILED", "payload could not be encoded as JSON", exception); + } + if (bytes.length > maxBytes) { + throw new MessageTooLargeException( + "PAYLOAD_TOO_LARGE", + "encoded payload is " + bytes.length + " bytes, limit is " + maxBytes); + } + return new EncodedMessage( + bytes, ContentType.JSON, Optional.of(SchemaReference.of(type.value(), version))); + } + + @Override + public T decode( + MessageType type, SchemaVersion version, byte[] encoded, Class payloadType) { + Class registered = requireRegistered(type); + Objects.requireNonNull(version, "schemaVersion must not be null"); + Objects.requireNonNull(encoded, "encoded bytes must not be null"); + Objects.requireNonNull(payloadType, "payloadType must not be null"); + if (!registered.equals(payloadType)) { + throw new MessageValidationException( + "PAYLOAD_TYPE_MISMATCH", + "requested type does not match the registered type for " + type.value()); + } + if (encoded.length > maxBytes) { + throw new MessageTooLargeException( + "PAYLOAD_TOO_LARGE", + "encoded payload is " + encoded.length + " bytes, limit is " + maxBytes); + } + + try { + return mapper.readValue(encoded, payloadType); + } catch (JacksonException exception) { + throw new MessageSerializationException( + "JSON_DECODE_FAILED", "payload could not be decoded as JSON", exception); + } + } + + private Class requireRegistered(MessageType type) { + Objects.requireNonNull(type, "messageType must not be null"); + Class registered = registry.get(type); + if (registered == null) { + throw new MessageValidationException( + "UNKNOWN_MESSAGE_TYPE", "message type is not registered: " + type.value()); + } + return registered; + } + + private static ObjectMapper strictMapper(int maxBytes) { + JsonFactory factory = + JsonFactory.builder() + .streamReadConstraints( + StreamReadConstraints.builder() + .maxNestingDepth(MAX_NESTING_DEPTH) + .maxDocumentLength(maxBytes) + .maxNumberLength(MAX_NUMBER_DIGITS) + .maxStringLength(MAX_STRING_CHARACTERS) + .maxNameLength(MAX_STRING_CHARACTERS) + .build()) + .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) + .build(); + return JsonMapper.builder(factory) + .enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY) + .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .enable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .build(); + } +} diff --git a/src/messaging/messaging-schema-json/src/test/java/dev/caskeleton/messaging/schema/json/JacksonMessageCodecTest.java b/src/messaging/messaging-schema-json/src/test/java/dev/caskeleton/messaging/schema/json/JacksonMessageCodecTest.java new file mode 100644 index 00000000..d9de94e9 --- /dev/null +++ b/src/messaging/messaging-schema-json/src/test/java/dev/caskeleton/messaging/schema/json/JacksonMessageCodecTest.java @@ -0,0 +1,124 @@ +package dev.caskeleton.messaging.schema.json; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.error.MessageSerializationException; +import dev.caskeleton.messaging.api.error.MessageTooLargeException; +import dev.caskeleton.messaging.api.error.MessageValidationException; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Test; + +class JacksonMessageCodecTest { + + private static final MessageType ORDER_CREATED = new MessageType("order.created"); + private static final SchemaVersion V1 = new SchemaVersion(1); + + record OrderCreated(String orderId, long amount) {} + + @Test + void roundTripsRegisteredType() { + JacksonMessageCodec codec = + JacksonMessageCodec.testingDefault(ORDER_CREATED, OrderCreated.class); + + byte[] encoded = codec.encode(ORDER_CREATED, V1, new OrderCreated("o-1", 1000)).bytes(); + + assertThat(codec.decode(ORDER_CREATED, V1, encoded, OrderCreated.class)) + .isEqualTo(new OrderCreated("o-1", 1000)); + } + + @Test + void rejectsPayloadOverOneMibibyte() { + JacksonMessageCodec codec = + JacksonMessageCodec.testingDefault(new MessageType("text.large"), String.class); + String value = "a".repeat(1_048_577); + + assertThatThrownBy(() -> codec.encode(new MessageType("text.large"), V1, value)) + .isInstanceOf(MessageTooLargeException.class) + .hasMessageContaining("1048576"); + } + + @Test + void rejectsUnregisteredMessageType() { + JacksonMessageCodec codec = + JacksonMessageCodec.testingDefault(ORDER_CREATED, OrderCreated.class); + + assertThatThrownBy( + () -> codec.encode(new MessageType("order.cancelled"), V1, new OrderCreated("o", 1))) + .isInstanceOf(MessageValidationException.class) + .hasMessageContaining("not registered"); + } + + @Test + void rejectsPayloadThatDoesNotMatchTheRegisteredType() { + JacksonMessageCodec codec = + JacksonMessageCodec.testingDefault(ORDER_CREATED, OrderCreated.class); + + assertThatThrownBy(() -> codec.encode(ORDER_CREATED, V1, "not an order")) + .isInstanceOf(MessageValidationException.class); + } + + @Test + void rejectsTrailingDataAfterTheDocument() { + JacksonMessageCodec codec = + JacksonMessageCodec.testingDefault(ORDER_CREATED, OrderCreated.class); + byte[] trailing = + "{\"orderId\":\"o-1\",\"amount\":1} {\"orderId\":\"o-2\",\"amount\":2}" + .getBytes(StandardCharsets.UTF_8); + + assertThatThrownBy(() -> codec.decode(ORDER_CREATED, V1, trailing, OrderCreated.class)) + .isInstanceOf(MessageSerializationException.class); + } + + @Test + void rejectsUnknownProperties() { + JacksonMessageCodec codec = + JacksonMessageCodec.testingDefault(ORDER_CREATED, OrderCreated.class); + byte[] extra = + "{\"orderId\":\"o-1\",\"amount\":1,\"injected\":true}".getBytes(StandardCharsets.UTF_8); + + assertThatThrownBy(() -> codec.decode(ORDER_CREATED, V1, extra, OrderCreated.class)) + .isInstanceOf(MessageSerializationException.class); + } + + @Test + void rejectsDuplicateKeys() { + JacksonMessageCodec codec = + JacksonMessageCodec.testingDefault(ORDER_CREATED, OrderCreated.class); + byte[] duplicate = + "{\"orderId\":\"o-1\",\"orderId\":\"o-2\",\"amount\":1}".getBytes(StandardCharsets.UTF_8); + + assertThatThrownBy(() -> codec.decode(ORDER_CREATED, V1, duplicate, OrderCreated.class)) + .isInstanceOf(MessageSerializationException.class); + } + + @Test + void rejectsNestingBeyondTheDepthCeiling() { + JacksonMessageCodec codec = + JacksonMessageCodec.testingDefault(new MessageType("deep.doc"), Object.class); + String deep = "[".repeat(200) + "]".repeat(200); + + assertThatThrownBy( + () -> + codec.decode( + new MessageType("deep.doc"), + V1, + deep.getBytes(StandardCharsets.UTF_8), + Object.class)) + .isInstanceOf(MessageSerializationException.class); + } + + @Test + void encodedMessageCarriesTheSchemaReference() { + JacksonMessageCodec codec = + JacksonMessageCodec.testingDefault(ORDER_CREATED, OrderCreated.class); + + var encoded = codec.encode(ORDER_CREATED, V1, new OrderCreated("o-1", 1000)); + + assertThat(encoded.schemaReference()).isPresent(); + assertThat(encoded.schemaReference().orElseThrow().subject()).isEqualTo("order.created"); + assertThat(encoded.contentType().value()).isEqualTo("application/json"); + } +} diff --git a/src/messaging/messaging-schema-json/src/test/java/dev/caskeleton/messaging/schema/json/PlatformOverheadPerformanceTest.java b/src/messaging/messaging-schema-json/src/test/java/dev/caskeleton/messaging/schema/json/PlatformOverheadPerformanceTest.java new file mode 100644 index 00000000..11b074d1 --- /dev/null +++ b/src/messaging/messaging-schema-json/src/test/java/dev/caskeleton/messaging/schema/json/PlatformOverheadPerformanceTest.java @@ -0,0 +1,135 @@ +package dev.caskeleton.messaging.schema.json; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +/** + * Bounds the platform's own per-message cost. + * + *

This measures what the platform adds — identity, validation, encoding — and nothing else. + * There is no broker in the loop, deliberately: broker throughput is a property of the deployment + * and varies by an order of magnitude between a laptop and a cluster, so asserting on it produces a + * test that fails for reasons nobody can act on. + * + *

The budgets are generous on purpose. The regression worth catching here is structural — an + * accidental per-message reflection call, a defensive copy that became a deep copy, a validator + * that started compiling a regex per invocation — and those cost orders of magnitude, not + * percentages. A tight budget would instead catch a busy CI agent. + */ +class PlatformOverheadPerformanceTest { + + private static final MessageType ORDER_CREATED = new MessageType("order.created"); + private static final SchemaVersion V1 = new SchemaVersion(1); + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + private static final int WARMUP = 5_000; + private static final int MEASURED = 50_000; + + /** Generous enough that only a structural regression trips it. */ + private static final Duration ENVELOPE_BUDGET_PER_MESSAGE = Duration.ofNanos(20_000); + + private static final Duration ENCODE_BUDGET_PER_MESSAGE = Duration.ofNanos(50_000); + + record OrderCreated(String orderId, long amount) {} + + @Test + void envelopeConstructionStaysWellUnderItsBudget() { + for (int i = 0; i < WARMUP; i++) { + envelope(new byte[64]); + } + + long startedAt = System.nanoTime(); + for (int i = 0; i < MEASURED; i++) { + envelope(new byte[64]); + } + Duration perMessage = Duration.ofNanos((System.nanoTime() - startedAt) / MEASURED); + + assertThat(perMessage) + .as("envelope construction per message: %s", perMessage) + .isLessThan(ENVELOPE_BUDGET_PER_MESSAGE); + } + + @Test + void jsonEncodingStaysWellUnderItsBudget() { + JacksonMessageCodec codec = + JacksonMessageCodec.testingDefault(ORDER_CREATED, OrderCreated.class); + OrderCreated payload = new OrderCreated("o-1", 1000); + + for (int i = 0; i < WARMUP; i++) { + codec.encode(ORDER_CREATED, V1, payload); + } + + long startedAt = System.nanoTime(); + for (int i = 0; i < MEASURED; i++) { + codec.encode(ORDER_CREATED, V1, payload); + } + Duration perMessage = Duration.ofNanos((System.nanoTime() - startedAt) / MEASURED); + + assertThat(perMessage) + .as("JSON encode per message: %s", perMessage) + .isLessThan(ENCODE_BUDGET_PER_MESSAGE); + } + + @Test + void aRoundTripDoesNotAllocateAGrowingRetainedSet() { + JacksonMessageCodec codec = + JacksonMessageCodec.testingDefault(ORDER_CREATED, OrderCreated.class); + OrderCreated payload = new OrderCreated("o-1", 1000); + + for (int i = 0; i < WARMUP; i++) { + byte[] encoded = codec.encode(ORDER_CREATED, V1, payload).bytes(); + codec.decode(ORDER_CREATED, V1, encoded, OrderCreated.class); + } + + Runtime runtime = Runtime.getRuntime(); + System.gc(); + long before = runtime.totalMemory() - runtime.freeMemory(); + + for (int i = 0; i < MEASURED; i++) { + byte[] encoded = codec.encode(ORDER_CREATED, V1, payload).bytes(); + codec.decode(ORDER_CREATED, V1, encoded, OrderCreated.class); + } + + System.gc(); + long after = runtime.totalMemory() - runtime.freeMemory(); + long retainedPerMessage = Math.max(0, after - before) / MEASURED; + + assertThat(retainedPerMessage) + .as( + "the codec must not retain per-message state; retained %d bytes/message", + retainedPerMessage) + .isLessThan(64); + } + + private static MessageEnvelope envelope(byte[] payload) { + return new MessageEnvelope<>( + MessageId.newId(), + ORDER_CREATED, + V1, + NOW, + Optional.of(NOW), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.of("acct-1"), + Optional.of("acct-1"), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage(payload, ContentType.JSON, Optional.empty())); + } +} diff --git a/src/messaging/messaging-schema-protobuf/build.gradle b/src/messaging/messaging-schema-protobuf/build.gradle new file mode 100644 index 00000000..d4854f62 --- /dev/null +++ b/src/messaging/messaging-schema-protobuf/build.gradle @@ -0,0 +1,8 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-schema-api') + + implementation 'com.google.protobuf:protobuf-java:4.29.3' +} diff --git a/src/messaging/messaging-schema-protobuf/gradle.lockfile b/src/messaging/messaging-schema-protobuf/gradle.lockfile new file mode 100644 index 00000000..a926f11a --- /dev/null +++ b/src/messaging/messaging-schema-protobuf/gradle.lockfile @@ -0,0 +1,84 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.29.3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty= diff --git a/src/messaging/messaging-schema-protobuf/src/main/java/dev/caskeleton/messaging/schema/protobuf/ProtobufMessageCodec.java b/src/messaging/messaging-schema-protobuf/src/main/java/dev/caskeleton/messaging/schema/protobuf/ProtobufMessageCodec.java new file mode 100644 index 00000000..894351f6 --- /dev/null +++ b/src/messaging/messaging-schema-protobuf/src/main/java/dev/caskeleton/messaging/schema/protobuf/ProtobufMessageCodec.java @@ -0,0 +1,124 @@ +package dev.caskeleton.messaging.schema.protobuf; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Message; +import com.google.protobuf.Parser; +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.error.MessageSerializationException; +import dev.caskeleton.messaging.api.error.MessageTooLargeException; +import dev.caskeleton.messaging.api.error.MessageValidationException; +import dev.caskeleton.messaging.schema.EncodedMessage; +import dev.caskeleton.messaging.schema.MessageCodec; +import dev.caskeleton.messaging.schema.SchemaReference; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * The optional Protobuf codec. + * + *

Bound to a closed registry of generated parsers. Protobuf's own wire format will happily + * decode almost any bytes into almost any message, so without the registry a type confusion is + * silent — the consumer gets a populated object built from the wrong schema rather than an error. + * + *

Unknown fields are preserved by the generated types, which is what makes forward compatibility + * work: an old consumer round-tripping a message written by a newer producer does not silently drop + * the fields it does not understand. + */ +public final class ProtobufMessageCodec implements MessageCodec { + + private static final int DEFAULT_MAX_BYTES = 1_048_576; + + private final Map> parsers; + private final Map> types; + private final int maxBytes; + + /** + * Creates a codec over an explicit registry. + * + * @param parsers the parser for each registered message type + * @param types the generated class for each registered message type + */ + public ProtobufMessageCodec( + Map> parsers, + Map> types) { + this(parsers, types, DEFAULT_MAX_BYTES); + } + + /** + * Creates a codec over an explicit registry and byte limit. + * + * @param parsers the parser for each registered message type + * @param types the generated class for each registered message type + * @param maxBytes the maximum encoded size + */ + public ProtobufMessageCodec( + Map> parsers, + Map> types, + int maxBytes) { + this.parsers = Map.copyOf(Objects.requireNonNull(parsers, "parsers must not be null")); + this.types = Map.copyOf(Objects.requireNonNull(types, "types must not be null")); + if (maxBytes < 1) { + throw new IllegalArgumentException("maxBytes must be positive"); + } + this.maxBytes = maxBytes; + } + + @Override + public ContentType contentType() { + return ContentType.PROTOBUF; + } + + @Override + public EncodedMessage encode(MessageType type, SchemaVersion version, Object payload) { + Class registered = requireRegistered(type); + if (!(payload instanceof Message message) || !registered.isInstance(payload)) { + throw new MessageValidationException( + "PAYLOAD_TYPE_MISMATCH", + "payload does not match the registered protobuf type for " + type.value()); + } + byte[] bytes = message.toByteArray(); + if (bytes.length > maxBytes) { + throw new MessageTooLargeException( + "PAYLOAD_TOO_LARGE", + "encoded payload is " + bytes.length + " bytes, limit is " + maxBytes); + } + return new EncodedMessage( + bytes, ContentType.PROTOBUF, Optional.of(SchemaReference.of(type.value(), version))); + } + + @Override + public T decode( + MessageType type, SchemaVersion version, byte[] encoded, Class payloadType) { + Class registered = requireRegistered(type); + Objects.requireNonNull(encoded, "encoded bytes must not be null"); + if (!registered.equals(payloadType)) { + throw new MessageValidationException( + "PAYLOAD_TYPE_MISMATCH", + "requested type does not match the registered protobuf type for " + type.value()); + } + if (encoded.length > maxBytes) { + throw new MessageTooLargeException( + "PAYLOAD_TOO_LARGE", + "encoded payload is " + encoded.length + " bytes, limit is " + maxBytes); + } + try { + return payloadType.cast(parsers.get(type).parseFrom(encoded)); + } catch (InvalidProtocolBufferException exception) { + throw new MessageSerializationException( + "PROTOBUF_DECODE_FAILED", "payload could not be decoded as protobuf", exception); + } + } + + private Class requireRegistered(MessageType type) { + Objects.requireNonNull(type, "messageType must not be null"); + Class registered = types.get(type); + if (registered == null || !parsers.containsKey(type)) { + throw new MessageValidationException( + "UNKNOWN_MESSAGE_TYPE", "message type is not registered: " + type.value()); + } + return registered; + } +} diff --git a/src/messaging/messaging-schema-protobuf/src/test/java/dev/caskeleton/messaging/schema/protobuf/ProtobufCompatibilityTest.java b/src/messaging/messaging-schema-protobuf/src/test/java/dev/caskeleton/messaging/schema/protobuf/ProtobufCompatibilityTest.java new file mode 100644 index 00000000..6ba1ca00 --- /dev/null +++ b/src/messaging/messaging-schema-protobuf/src/test/java/dev/caskeleton/messaging/schema/protobuf/ProtobufCompatibilityTest.java @@ -0,0 +1,186 @@ +package dev.caskeleton.messaging.schema.protobuf; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.google.protobuf.DescriptorProtos.DescriptorProto; +import com.google.protobuf.DescriptorProtos.FieldDescriptorProto; +import com.google.protobuf.DescriptorProtos.FileDescriptorProto; +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.Descriptors.DescriptorValidationException; +import com.google.protobuf.Descriptors.FileDescriptor; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.Message; +import com.google.protobuf.Parser; +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.error.MessageValidationException; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Proves the tag-number rules the {@code order_created_v1.proto} fixture documents. + * + *

Descriptors are built at runtime rather than generated by protoc. The properties under test — + * that a reader keyed on tag numbers survives a rename, that an added field decodes as its default, + * and that reusing a tag corrupts the read — are properties of the wire format, so proving them + * without a code-generation step keeps the test honest and the build free of a protoc toolchain. + */ +class ProtobufCompatibilityTest { + + private static final MessageType ORDER_CREATED = new MessageType("order.created"); + private static final SchemaVersion V1 = new SchemaVersion(1); + private static final SchemaVersion V2 = new SchemaVersion(2); + + private static final Descriptor V1_DESCRIPTOR = + descriptor( + "OrderCreatedV1", + field("order_id", 1, FieldDescriptorProto.Type.TYPE_STRING), + field("customer_id", 2, FieldDescriptorProto.Type.TYPE_STRING), + field("total_minor_units", 3, FieldDescriptorProto.Type.TYPE_INT64), + field("currency", 4, FieldDescriptorProto.Type.TYPE_STRING)); + + /** v2 renames tag 4 and appends tag 5: both are wire-compatible changes. */ + private static final Descriptor V2_DESCRIPTOR = + descriptor( + "OrderCreatedV2", + field("order_id", 1, FieldDescriptorProto.Type.TYPE_STRING), + field("customer_id", 2, FieldDescriptorProto.Type.TYPE_STRING), + field("total_minor_units", 3, FieldDescriptorProto.Type.TYPE_INT64), + field("currency_code", 4, FieldDescriptorProto.Type.TYPE_STRING), + field("channel", 5, FieldDescriptorProto.Type.TYPE_STRING)); + + /** The forbidden change: tag 4 is reused for a different type. */ + private static final Descriptor TAG_REUSE_DESCRIPTOR = + descriptor( + "OrderCreatedTagReuse", + field("order_id", 1, FieldDescriptorProto.Type.TYPE_STRING), + field("customer_id", 2, FieldDescriptorProto.Type.TYPE_STRING), + field("total_minor_units", 3, FieldDescriptorProto.Type.TYPE_INT64), + field("discount_minor_units", 4, FieldDescriptorProto.Type.TYPE_INT64)); + + private static FieldDescriptorProto field( + String name, int number, FieldDescriptorProto.Type type) { + return FieldDescriptorProto.newBuilder() + .setName(name) + .setNumber(number) + .setType(type) + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .build(); + } + + private static Descriptor descriptor(String name, FieldDescriptorProto... fields) { + DescriptorProto.Builder message = DescriptorProto.newBuilder().setName(name); + for (FieldDescriptorProto field : fields) { + message.addField(field); + } + FileDescriptorProto file = + FileDescriptorProto.newBuilder() + .setName(name + ".proto") + .setSyntax("proto3") + .setPackage("dev.caskeleton.messaging.sample") + .addMessageType(message) + .build(); + try { + return FileDescriptor.buildFrom(file, new FileDescriptor[0]).getMessageTypes().get(0); + } catch (DescriptorValidationException failure) { + throw new IllegalStateException("could not build the test descriptor", failure); + } + } + + private static ProtobufMessageCodec codec(Descriptor descriptor) { + DynamicMessage prototype = DynamicMessage.getDefaultInstance(descriptor); + Parser parser = prototype.getParserForType(); + return new ProtobufMessageCodec( + Map.of(ORDER_CREATED, parser), Map.of(ORDER_CREATED, DynamicMessage.class)); + } + + private static DynamicMessage v1Message() { + return DynamicMessage.newBuilder(V1_DESCRIPTOR) + .setField(V1_DESCRIPTOR.findFieldByNumber(1), "o-1") + .setField(V1_DESCRIPTOR.findFieldByNumber(2), "c-1") + .setField(V1_DESCRIPTOR.findFieldByNumber(3), 12_500L) + .setField(V1_DESCRIPTOR.findFieldByNumber(4), "KRW") + .build(); + } + + @Test + void aRoundTripPreservesEveryField() { + ProtobufMessageCodec codec = codec(V1_DESCRIPTOR); + + EncodedMessage encoded = codec.encode(ORDER_CREATED, V1, v1Message()); + DynamicMessage decoded = codec.decode(ORDER_CREATED, V1, encoded.bytes(), DynamicMessage.class); + + assertThat(decoded.getField(V1_DESCRIPTOR.findFieldByNumber(1))).isEqualTo("o-1"); + assertThat(decoded.getField(V1_DESCRIPTOR.findFieldByNumber(3))).isEqualTo(12_500L); + assertThat(encoded.contentType()).isEqualTo(ContentType.PROTOBUF); + } + + @Test + void renamingAFieldKeepsItsValueBecauseTheTagNumberIsTheContract() throws Exception { + byte[] wire = codec(V1_DESCRIPTOR).encode(ORDER_CREATED, V1, v1Message()).bytes(); + + DynamicMessage asV2 = DynamicMessage.parseFrom(V2_DESCRIPTOR, wire); + + assertThat(asV2.getField(V2_DESCRIPTOR.findFieldByName("currency_code"))) + .as("the name changed, the tag did not, so the value survives") + .isEqualTo("KRW"); + } + + @Test + void anAddedFieldDecodesAsItsDefaultForAnOldWriter() throws Exception { + byte[] wire = codec(V1_DESCRIPTOR).encode(ORDER_CREATED, V1, v1Message()).bytes(); + + DynamicMessage asV2 = DynamicMessage.parseFrom(V2_DESCRIPTOR, wire); + + assertThat(asV2.getField(V2_DESCRIPTOR.findFieldByName("channel"))) + .as("proto3 scalars are optional on the wire, so an absent field is the default") + .isEqualTo(""); + } + + @Test + void aNewWriterIsStillReadableByAnOldReader() throws Exception { + DynamicMessage v2 = + DynamicMessage.newBuilder(V2_DESCRIPTOR) + .setField(V2_DESCRIPTOR.findFieldByNumber(1), "o-2") + .setField(V2_DESCRIPTOR.findFieldByNumber(4), "USD") + .setField(V2_DESCRIPTOR.findFieldByNumber(5), "APP") + .build(); + + DynamicMessage asV1 = DynamicMessage.parseFrom(V1_DESCRIPTOR, v2.toByteArray()); + + assertThat(asV1.getField(V1_DESCRIPTOR.findFieldByNumber(4))).isEqualTo("USD"); + assertThat(asV1.getUnknownFields().hasField(5)) + .as("the unrecognised field is retained, not dropped, so a round trip does not lose it") + .isTrue(); + } + + @Test + void reusingATagNumberCorruptsTheReadWhichIsWhyTagsAreNeverRecycled() throws Exception { + byte[] wire = codec(V1_DESCRIPTOR).encode(ORDER_CREATED, V1, v1Message()).bytes(); + + DynamicMessage reused = DynamicMessage.parseFrom(TAG_REUSE_DESCRIPTOR, wire); + + assertThat(reused.getField(TAG_REUSE_DESCRIPTOR.findFieldByName("discount_minor_units"))) + .as("the string bytes on tag 4 are not a valid int64, so the value is lost entirely") + .isEqualTo(0L); + } + + @Test + void anUnregisteredTypeIsRejectedRatherThanGuessed() { + ProtobufMessageCodec codec = codec(V1_DESCRIPTOR); + + assertThatThrownBy(() -> codec.encode(new MessageType("unknown.event"), V1, v1Message())) + .isInstanceOf(MessageValidationException.class); + } + + @Test + void theEncodedMessageCarriesItsSchemaReference() { + EncodedMessage encoded = codec(V1_DESCRIPTOR).encode(ORDER_CREATED, V2, v1Message()); + + assertThat(encoded.schemaReference()) + .hasValueSatisfying(reference -> assertThat(reference.version()).isEqualTo(V2)); + } +} diff --git a/src/messaging/messaging-schema-protobuf/src/test/proto/order_created_v1.proto b/src/messaging/messaging-schema-protobuf/src/test/proto/order_created_v1.proto new file mode 100644 index 00000000..d765c5a0 --- /dev/null +++ b/src/messaging/messaging-schema-protobuf/src/test/proto/order_created_v1.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package dev.caskeleton.messaging.sample; + +option java_package = "dev.caskeleton.messaging.sample.proto"; +option java_outer_classname = "OrderCreatedV1Proto"; +option java_multiple_files = true; + +// The v1 wire contract for order.created. +// +// Field numbers are the contract, not the field names: renaming `currency` keeps every existing +// consumer working, while reusing tag 4 for something else silently corrupts them. Tags are never +// reused, and removed fields are reserved so that a later edit cannot take the number back. +message OrderCreated { + string order_id = 1; + string customer_id = 2; + int64 total_minor_units = 3; + string currency = 4; + + // v2 adds `channel = 5`. proto3 scalars are always optional on the wire, so an old writer that + // never sets it decodes as the default rather than failing, which is what makes the addition + // backward compatible without any per-field ceremony. +} diff --git a/src/messaging/messaging-security/build.gradle b/src/messaging/messaging-security/build.gradle new file mode 100644 index 00000000..95af0ac3 --- /dev/null +++ b/src/messaging/messaging-security/build.gradle @@ -0,0 +1,5 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') +} diff --git a/src/messaging/messaging-security/gradle.lockfile b/src/messaging/messaging-security/gradle.lockfile new file mode 100644 index 00000000..599ff921 --- /dev/null +++ b/src/messaging/messaging-security/gradle.lockfile @@ -0,0 +1,83 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty=compileClasspath,runtimeClasspath diff --git a/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerAclManifest.java b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerAclManifest.java new file mode 100644 index 00000000..87f84796 --- /dev/null +++ b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerAclManifest.java @@ -0,0 +1,141 @@ +package dev.caskeleton.messaging.security; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * The exact broker permissions a deployment declares it needs. + * + *

Written down so the grant can be reviewed and diffed rather than discovered from a broker + * dump. The manifest is what the platform checks itself against at startup: a runtime that holds + * more than it declares is a finding, because the extra permission is the one nobody reasoned + * about. + * + *

Destructive permissions are named separately from ordinary ones. {@code DELETE_TOPIC} and + * {@code PURGE} are not "write, but more"; they destroy data an application can never restore, so + * an application runtime declaring one is rejected outright. + * + * @param principal the broker principal the grants belong to + * @param grants the declared permissions + */ +public record BrokerAclManifest(String principal, List grants) { + + /** One declared permission over one resource pattern. */ + public record Grant(Operation operation, ResourceType resourceType, String pattern) { + + public Grant { + Objects.requireNonNull(operation, "operation must not be null"); + Objects.requireNonNull(resourceType, "resourceType must not be null"); + if (pattern == null || pattern.isBlank()) { + throw new IllegalArgumentException("pattern must not be blank"); + } + } + } + + /** The broker resources a grant can cover. */ + public enum ResourceType { + /** A topic, exchange, or stream. */ + DESTINATION, + /** A consumer group or subscription. */ + GROUP, + /** The broker or cluster itself. */ + CLUSTER + } + + /** The operations a grant can permit. */ + public enum Operation { + /** Publish to a destination. */ + WRITE(false), + /** Consume from a destination. */ + READ(false), + /** Read metadata. */ + DESCRIBE(false), + /** Create a destination or group. */ + CREATE(false), + /** Change a destination's configuration. */ + ALTER(true), + /** Remove a destination or group. */ + DELETE(true), + /** Discard the messages in a destination. */ + PURGE(true); + + private final boolean destructive; + + Operation(boolean destructive) { + this.destructive = destructive; + } + + /** + * Reports whether this operation can destroy data. + * + * @return true for the destructive operations + */ + public boolean isDestructive() { + return destructive; + } + } + + public BrokerAclManifest { + Objects.requireNonNull(grants, "grants must not be null"); + if (principal == null || principal.isBlank()) { + throw new IllegalArgumentException("principal must not be blank"); + } + grants = List.copyOf(grants); + } + + /** + * Returns the declared destructive grants. + * + * @return the destructive grants + */ + public List destructiveGrants() { + return grants.stream().filter(grant -> grant.operation().isDestructive()).toList(); + } + + /** + * Refuses a manifest that gives an application runtime destructive power. + * + * @throws MessagingConfigurationException when a destructive grant is declared + */ + public void requireApplicationRuntime() { + List destructive = destructiveGrants(); + if (!destructive.isEmpty()) { + throw new MessagingConfigurationException( + "APPLICATION_HOLDS_DESTRUCTIVE_GRANT", + "principal %s declares %s, which an application runtime must never hold" + .formatted(principal, destructive.stream().map(Grant::operation).toList())); + } + } + + /** + * Reports the permissions the runtime holds but never declared. + * + *

Excess is the finding, not the shortfall: a missing grant fails loudly on first use, while + * an undeclared extra one sits unnoticed until it is abused. + * + * @param observed the grants the broker actually reports for this principal + * @return the observed grants that this manifest does not declare + */ + public Set undeclared(Set observed) { + Objects.requireNonNull(observed, "observed must not be null"); + Set extra = new LinkedHashSet<>(observed); + extra.removeAll(Set.copyOf(grants)); + return Set.copyOf(extra); + } + + /** + * Reports the declared permissions the broker does not actually grant. + * + * @param observed the grants the broker actually reports for this principal + * @return the declared grants that are missing + */ + public Set missing(Set observed) { + Objects.requireNonNull(observed, "observed must not be null"); + Set absent = new LinkedHashSet<>(grants); + absent.removeAll(observed); + return Set.copyOf(absent); + } +} diff --git a/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerCredentialProfile.java b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerCredentialProfile.java new file mode 100644 index 00000000..6bd1f5b9 --- /dev/null +++ b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerCredentialProfile.java @@ -0,0 +1,78 @@ +package dev.caskeleton.messaging.security; + +/** + * A closed set of broker authentication mechanisms, referenced by credential id. + * + *

No variant carries a secret. The platform stores an identifier and resolves the material + * through a {@link CredentialProvider} at connect time, so a rotation is a provider concern and a + * heap dump or configuration print never yields a usable credential. + */ +public sealed interface BrokerCredentialProfile + permits BrokerCredentialProfile.SaslScram, + BrokerCredentialProfile.OAuth2, + BrokerCredentialProfile.MutualTls, + BrokerCredentialProfile.UsernamePassword, + BrokerCredentialProfile.Nkey { + + /** + * Returns the credential identifier this profile resolves. + * + * @return the credential id + */ + String credentialId(); + + /** + * SASL/SCRAM authentication. + * + * @param credentialId the credential id + */ + record SaslScram(String credentialId) implements BrokerCredentialProfile { + public SaslScram { + CredentialIds.require(credentialId); + } + } + + /** + * OAuth2 bearer authentication. + * + * @param credentialId the credential id + */ + record OAuth2(String credentialId) implements BrokerCredentialProfile { + public OAuth2 { + CredentialIds.require(credentialId); + } + } + + /** + * Mutual TLS authentication. + * + * @param credentialId the credential id + */ + record MutualTls(String credentialId) implements BrokerCredentialProfile { + public MutualTls { + CredentialIds.require(credentialId); + } + } + + /** + * Plain username and password authentication. + * + * @param credentialId the credential id + */ + record UsernamePassword(String credentialId) implements BrokerCredentialProfile { + public UsernamePassword { + CredentialIds.require(credentialId); + } + } + + /** + * NATS NKey authentication. + * + * @param credentialId the credential id + */ + record Nkey(String credentialId) implements BrokerCredentialProfile { + public Nkey { + CredentialIds.require(credentialId); + } + } +} diff --git a/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerSecurityProfile.java b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerSecurityProfile.java new file mode 100644 index 00000000..ab59d9a2 --- /dev/null +++ b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerSecurityProfile.java @@ -0,0 +1,41 @@ +package dev.caskeleton.messaging.security; + +import java.util.Objects; +import java.util.Optional; + +/** + * The security posture for one broker connection. + * + *

Producer, consumer, and admin credentials are separate fields rather than one connection + * credential. That separation is what makes "an application cannot purge a topic" enforceable: the + * runtime never holds admin material, so a compromised handler has nothing to escalate with. + * + * @param broker the broker name + * @param production whether this profile is used in production + * @param tlsEnabled whether transport encryption is on + * @param hostnameVerification whether the server certificate hostname is verified + * @param producer the producer credential + * @param consumer the consumer credential + * @param admin the admin credential, absent in an application runtime + * @param access the destination access policy + */ +public record BrokerSecurityProfile( + String broker, + boolean production, + boolean tlsEnabled, + boolean hostnameVerification, + BrokerCredentialProfile producer, + BrokerCredentialProfile consumer, + Optional admin, + DestinationAccessPolicy access) { + + public BrokerSecurityProfile { + Objects.requireNonNull(producer, "producer credential must not be null"); + Objects.requireNonNull(consumer, "consumer credential must not be null"); + Objects.requireNonNull(admin, "admin credential must not be null"); + Objects.requireNonNull(access, "access policy must not be null"); + if (broker == null || broker.isBlank()) { + throw new IllegalArgumentException("broker must not be blank"); + } + } +} diff --git a/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerTlsPolicy.java b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerTlsPolicy.java new file mode 100644 index 00000000..a271a6f2 --- /dev/null +++ b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/BrokerTlsPolicy.java @@ -0,0 +1,98 @@ +package dev.caskeleton.messaging.security; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * The transport-encryption rules a broker connection must satisfy before it is opened. + * + *

Checked at startup, not per connection. A TLS misconfiguration that surfaces on the first + * publish has already shipped; one that refuses to start is caught in the deployment that + * introduced it. + * + *

Disabling hostname verification is treated as a separate, worse failure than disabling TLS. + * Plaintext is at least obviously insecure, whereas TLS without hostname verification looks + * encrypted in every dashboard while accepting any certificate a man in the middle presents. + */ +public final class BrokerTlsPolicy { + + /** Protocol versions with known practical attacks. */ + private static final Set FORBIDDEN_PROTOCOLS = + Set.of("SSLv2", "SSLv3", "TLSv1", "TLSv1.1"); + + /** The minimum acceptable protocol version. */ + public static final String MINIMUM_PROTOCOL = "TLSv1.2"; + + private final boolean allowPlaintextOutsideProduction; + + /** Creates a policy that permits plaintext only outside production. */ + public BrokerTlsPolicy() { + this(true); + } + + /** + * Creates a policy. + * + * @param allowPlaintextOutsideProduction whether non-production profiles may run without TLS + */ + public BrokerTlsPolicy(boolean allowPlaintextOutsideProduction) { + this.allowPlaintextOutsideProduction = allowPlaintextOutsideProduction; + } + + /** + * Validates one broker's transport security. + * + * @param profile the broker security profile + * @param enabledProtocols the protocol versions the connection would negotiate + * @throws MessagingConfigurationException when the posture is not acceptable + */ + public void validate(BrokerSecurityProfile profile, List enabledProtocols) { + Objects.requireNonNull(profile, "profile must not be null"); + Objects.requireNonNull(enabledProtocols, "enabledProtocols must not be null"); + + if (!profile.tlsEnabled()) { + if (profile.production() || !allowPlaintextOutsideProduction) { + throw new MessagingConfigurationException( + "TLS_REQUIRED", + "broker %s runs in production and must not use a plaintext connection" + .formatted(profile.broker())); + } + return; + } + + if (!profile.hostnameVerification()) { + throw new MessagingConfigurationException( + "HOSTNAME_VERIFICATION_REQUIRED", + "broker %s enables TLS without hostname verification, which accepts any certificate" + .formatted(profile.broker())); + } + + List forbidden = + enabledProtocols.stream().filter(FORBIDDEN_PROTOCOLS::contains).toList(); + if (!forbidden.isEmpty()) { + throw new MessagingConfigurationException( + "TLS_PROTOCOL_TOO_OLD", + "broker %s enables %s; the minimum is %s" + .formatted(profile.broker(), forbidden, MINIMUM_PROTOCOL)); + } + if (enabledProtocols.isEmpty()) { + throw new MessagingConfigurationException( + "TLS_PROTOCOL_UNSPECIFIED", + "broker %s enables TLS without naming a protocol version, so the negotiated version " + .formatted(profile.broker()) + + "depends on the JVM default rather than on this policy"); + } + } + + /** + * Reports whether a protocol version is acceptable. + * + * @param protocol the protocol version + * @return true when the version may be negotiated + */ + public static boolean isAcceptable(String protocol) { + return protocol != null && !FORBIDDEN_PROTOCOLS.contains(protocol); + } +} diff --git a/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialIds.java b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialIds.java new file mode 100644 index 00000000..38faea78 --- /dev/null +++ b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialIds.java @@ -0,0 +1,37 @@ +package dev.caskeleton.messaging.security; + +import java.util.Locale; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * Validation shared by every credential profile variant. + * + *

The bounded slug pattern is not cosmetic. Credential ids reach log lines and metric tags, so + * an unbounded id is a cardinality problem, and an id that looks like a secret is a leak. The + * heuristic check rejects the most common accident: pasting the secret itself where the reference + * belongs. + */ +final class CredentialIds { + + private static final Pattern VALID = Pattern.compile("[a-z0-9][a-z0-9._-]{1,63}"); + + private static final Set SECRET_LOOKING_PREFIXES = + Set.of("bearer ", "basic ", "sk-", "-----begin", "eyj"); + + private CredentialIds() {} + + static void require(String credentialId) { + if (credentialId == null || !VALID.matcher(credentialId).matches()) { + throw new IllegalArgumentException( + "credentialId must match [a-z0-9][a-z0-9._-]{1,63}: " + credentialId); + } + String lowered = credentialId.toLowerCase(Locale.ROOT); + for (String prefix : SECRET_LOOKING_PREFIXES) { + if (lowered.startsWith(prefix)) { + throw new IllegalArgumentException( + "credentialId looks like a secret value rather than a reference"); + } + } + } +} diff --git a/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialProvider.java b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialProvider.java new file mode 100644 index 00000000..60836f62 --- /dev/null +++ b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialProvider.java @@ -0,0 +1,29 @@ +package dev.caskeleton.messaging.security; + +import java.time.Instant; +import java.util.Optional; + +/** + * Resolves credential material from an identifier. + * + *

Implementations are expected to fetch late and cache briefly. Returning an expiry lets the + * runtime rotate a generation before the broker starts refusing connections, rather than after. + */ +public interface CredentialProvider { + + /** + * Resolves the secret for a credential id. + * + * @param credentialId the identifier + * @return the resolved material + */ + char[] resolve(String credentialId); + + /** + * Returns when the credential stops being valid, when that is known. + * + * @param credentialId the identifier + * @return the expiry instant when known + */ + Optional expiresAt(String credentialId); +} diff --git a/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialRotationPlan.java b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialRotationPlan.java new file mode 100644 index 00000000..f6e9e698 --- /dev/null +++ b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialRotationPlan.java @@ -0,0 +1,54 @@ +package dev.caskeleton.messaging.security; + +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * When a credential must be replaced, and how much slack remains. + * + *

Rotation is scheduled ahead of expiry rather than triggered by an authentication failure. By + * the time the broker starts refusing connections, in-flight publishes are already failing and the + * consumer has already stopped: rotating early turns an outage into a generation swap nobody + * notices. + * + * @param credentialId the credential to rotate + * @param expiresAt when the credential stops being valid, when known + * @param rotateBefore how far ahead of expiry to rotate + */ +public record CredentialRotationPlan( + String credentialId, Optional expiresAt, Duration rotateBefore) { + + public CredentialRotationPlan { + Objects.requireNonNull(expiresAt, "expiresAt must not be null"); + Objects.requireNonNull(rotateBefore, "rotateBefore must not be null"); + if (credentialId == null || credentialId.isBlank()) { + throw new IllegalArgumentException("credentialId must not be blank"); + } + if (rotateBefore.isNegative()) { + throw new IllegalArgumentException("rotateBefore must not be negative"); + } + } + + /** + * Reports whether the credential should be rotated now. + * + * @param now the current instant + * @return true once the rotation window has opened + */ + public boolean isDue(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + return expiresAt.map(expiry -> !now.isBefore(expiry.minus(rotateBefore))).orElse(false); + } + + /** + * Reports whether the credential has already expired. + * + * @param now the current instant + * @return true once expired + */ + public boolean isExpired(Instant now) { + return expiresAt.map(expiry -> !now.isBefore(expiry)).orElse(false); + } +} diff --git a/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialRuntime.java b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialRuntime.java new file mode 100644 index 00000000..aefc5bc2 --- /dev/null +++ b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialRuntime.java @@ -0,0 +1,143 @@ +package dev.caskeleton.messaging.security; + +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Objects; +import java.util.Optional; + +/** + * One credential's live state: when it was resolved, when it expires, and whether it is due for + * rotation. + * + *

Holds the material in a {@code char[]} that {@link #clear()} overwrites. A {@code String} + * cannot be erased — it stays in the constant pool and in every heap dump taken until the next GC + * decides otherwise — so the type of the field is itself part of the control. + * + *

Rotation is driven from the expiry, ahead of it. Waiting for the broker to start refusing + * connections turns a scheduled, invisible rotation into an outage. + */ +public final class CredentialRuntime { + + /** How far ahead of expiry a credential is considered due for rotation. */ + public static final Duration DEFAULT_ROTATION_LEAD = Duration.ofMinutes(30); + + private final String credentialId; + private final Instant resolvedAt; + private final Optional expiresAt; + private final Duration rotationLead; + private char[] material; + + /** + * Creates a runtime entry. + * + * @param credentialId the credential reference + * @param material the resolved secret, taken by reference and cleared by {@link #clear()} + * @param resolvedAt when the material was resolved + * @param expiresAt when it stops being valid, when known + * @param rotationLead how far ahead of expiry rotation should begin + */ + public CredentialRuntime( + String credentialId, + char[] material, + Instant resolvedAt, + Optional expiresAt, + Duration rotationLead) { + CredentialIds.require(credentialId); + Objects.requireNonNull(material, "material must not be null"); + Objects.requireNonNull(resolvedAt, "resolvedAt must not be null"); + Objects.requireNonNull(expiresAt, "expiresAt must not be null"); + Objects.requireNonNull(rotationLead, "rotationLead must not be null"); + if (rotationLead.isNegative()) { + throw new IllegalArgumentException("rotationLead must not be negative"); + } + if (material.length == 0) { + throw new IllegalArgumentException("credential material must not be empty"); + } + this.credentialId = credentialId; + this.material = material; + this.resolvedAt = resolvedAt; + this.expiresAt = expiresAt; + this.rotationLead = rotationLead; + } + + /** + * Returns the credential reference. + * + * @return the credential id + */ + public String credentialId() { + return credentialId; + } + + /** + * Returns a copy of the material. + * + *

A copy, so a caller that clears its own array cannot blind every other holder. + * + * @return the secret material + * @throws IllegalStateException once {@link #clear()} has run + */ + public char[] material() { + if (material.length == 0) { + throw new IllegalStateException("credential " + credentialId + " has already been cleared"); + } + return material.clone(); + } + + /** + * Returns when the material was resolved. + * + * @return the resolution instant + */ + public Instant resolvedAt() { + return resolvedAt; + } + + /** + * Returns the expiry when it is known. + * + * @return the expiry instant + */ + public Optional expiresAt() { + return expiresAt; + } + + /** + * Reports whether the credential has expired. + * + * @param now the current instant + * @return true when the expiry is known and has passed + */ + public boolean isExpired(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + return expiresAt.map(expiry -> !now.isBefore(expiry)).orElse(false); + } + + /** + * Reports whether rotation should begin. + * + * @param now the current instant + * @return true when the credential is inside its rotation lead or already expired + */ + public boolean isDueForRotation(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + return expiresAt.map(expiry -> !now.isBefore(expiry.minus(rotationLead))).orElse(false); + } + + /** Overwrites the material in place. */ + public void clear() { + Arrays.fill(material, '\0'); + material = new char[0]; + } + + /** + * Returns a representation that names the credential without revealing it. + * + * @return a safe description + */ + @Override + public String toString() { + return "CredentialRuntime[credentialId=%s, expiresAt=%s]".formatted(credentialId, expiresAt); + } +} diff --git a/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialRuntimeRegistry.java b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialRuntimeRegistry.java new file mode 100644 index 00000000..9f5718c1 --- /dev/null +++ b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/CredentialRuntimeRegistry.java @@ -0,0 +1,122 @@ +package dev.caskeleton.messaging.security; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Resolves credentials late, caches them briefly, and clears the old material on rotation. + * + *

Late resolution is what makes rotation possible at all. A credential read once at startup and + * held for the process lifetime cannot be rotated without a restart, which is why rotation gets + * skipped in practice. + * + *

The previous generation is cleared as soon as the new one is installed. Keeping it "just in + * case" is how a revoked credential stays live in memory long after the store says it is gone. + */ +public final class CredentialRuntimeRegistry { + + private final CredentialProvider provider; + private final Duration rotationLead; + private final Map resolved = new ConcurrentHashMap<>(); + + /** + * Creates a registry with the default rotation lead. + * + * @param provider the credential source + */ + public CredentialRuntimeRegistry(CredentialProvider provider) { + this(provider, CredentialRuntime.DEFAULT_ROTATION_LEAD); + } + + /** + * Creates a registry. + * + * @param provider the credential source + * @param rotationLead how far ahead of expiry rotation begins + */ + public CredentialRuntimeRegistry(CredentialProvider provider, Duration rotationLead) { + this.provider = Objects.requireNonNull(provider, "provider must not be null"); + this.rotationLead = Objects.requireNonNull(rotationLead, "rotationLead must not be null"); + } + + /** + * Returns the live credential, resolving or rotating it as needed. + * + * @param credentialId the credential reference + * @param now the current instant + * @return the live credential + */ + public CredentialRuntime resolve(String credentialId, Instant now) { + CredentialIds.require(credentialId); + Objects.requireNonNull(now, "now must not be null"); + + CredentialRuntime current = resolved.get(credentialId); + if (current != null && !current.isDueForRotation(now)) { + return current; + } + CredentialRuntime replacement = fetch(credentialId, now); + resolved.put(credentialId, replacement); + if (current != null) { + // Clear only after the replacement is installed, so a concurrent reader never observes a + // window with no usable credential. + current.clear(); + } + return replacement; + } + + /** + * Returns the credentials that are inside their rotation lead. + * + * @param now the current instant + * @return the credential ids due for rotation + */ + public List dueForRotation(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + return resolved.entrySet().stream() + .filter(entry -> entry.getValue().isDueForRotation(now)) + .map(Map.Entry::getKey) + .sorted() + .toList(); + } + + /** + * Returns the credentials that have already expired. + * + * @param now the current instant + * @return the expired credential ids + */ + public List expired(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + return resolved.entrySet().stream() + .filter(entry -> entry.getValue().isExpired(now)) + .map(Map.Entry::getKey) + .sorted() + .toList(); + } + + /** + * Returns how many credentials are currently held. + * + * @return the cached credential count + */ + public int size() { + return resolved.size(); + } + + /** Clears every held credential, for shutdown. */ + public void clearAll() { + resolved.values().forEach(CredentialRuntime::clear); + resolved.clear(); + } + + private CredentialRuntime fetch(String credentialId, Instant now) { + char[] material = provider.resolve(credentialId); + Optional expiresAt = provider.expiresAt(credentialId); + return new CredentialRuntime(credentialId, material, now, expiresAt, rotationLead); + } +} diff --git a/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/DestinationAccessPolicy.java b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/DestinationAccessPolicy.java new file mode 100644 index 00000000..2616e7e8 --- /dev/null +++ b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/DestinationAccessPolicy.java @@ -0,0 +1,70 @@ +package dev.caskeleton.messaging.security; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.util.Objects; +import java.util.Set; + +/** + * Which destinations each role may touch. + * + *

The platform checks this before the broker does. Relying only on broker ACLs means an + * accidental publish surfaces as a generic authorization error at runtime, in the adapter, with no + * record of which application module attempted it. + * + * @param publishable destinations the producer credential may publish to + * @param consumable destinations the consumer credential may consume + * @param administrable destinations the admin credential may operate on + */ +public record DestinationAccessPolicy( + Set publishable, + Set consumable, + Set administrable) { + + public DestinationAccessPolicy { + Objects.requireNonNull(publishable, "publishable must not be null"); + Objects.requireNonNull(consumable, "consumable must not be null"); + Objects.requireNonNull(administrable, "administrable must not be null"); + publishable = Set.copyOf(publishable); + consumable = Set.copyOf(consumable); + administrable = Set.copyOf(administrable); + } + + /** + * Returns a policy that permits nothing. + * + * @return the empty policy + */ + public static DestinationAccessPolicy denyAll() { + return new DestinationAccessPolicy(Set.of(), Set.of(), Set.of()); + } + + /** + * Reports whether publishing is permitted. + * + * @param destination the logical destination + * @return true when permitted + */ + public boolean mayPublish(DestinationName destination) { + return publishable.contains(destination); + } + + /** + * Reports whether consuming is permitted. + * + * @param destination the logical destination + * @return true when permitted + */ + public boolean mayConsume(DestinationName destination) { + return consumable.contains(destination); + } + + /** + * Reports whether administration is permitted. + * + * @param destination the logical destination + * @return true when permitted + */ + public boolean mayAdminister(DestinationName destination) { + return administrable.contains(destination); + } +} diff --git a/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/DestinationAccessValidator.java b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/DestinationAccessValidator.java new file mode 100644 index 00000000..e8a5b04b --- /dev/null +++ b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/DestinationAccessValidator.java @@ -0,0 +1,65 @@ +package dev.caskeleton.messaging.security; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessageAuthorizationException; +import java.util.Objects; + +/** + * Checks a destination operation against the platform's own access policy. + * + *

Runs before the broker's ACL, and the failure it raises names the logical destination and the + * role. A broker ACL denial arrives as a connection-level error with no application context, which + * makes "which module tried to publish where" an investigation rather than a log line. + */ +public final class DestinationAccessValidator { + + private final DestinationAccessPolicy policy; + + /** + * Creates a validator over an access policy. + * + * @param policy the destination access policy + */ + public DestinationAccessValidator(DestinationAccessPolicy policy) { + this.policy = Objects.requireNonNull(policy, "policy must not be null"); + } + + /** + * Requires publish permission. + * + * @param destination the logical destination + */ + public void requirePublish(DestinationName destination) { + if (!policy.mayPublish(destination)) { + throw new MessageAuthorizationException( + "DESTINATION_PUBLISH_DENIED", + "the producer credential may not publish to " + destination.value()); + } + } + + /** + * Requires consume permission. + * + * @param destination the logical destination + */ + public void requireConsume(DestinationName destination) { + if (!policy.mayConsume(destination)) { + throw new MessageAuthorizationException( + "DESTINATION_CONSUME_DENIED", + "the consumer credential may not consume " + destination.value()); + } + } + + /** + * Requires administer permission. + * + * @param destination the logical destination + */ + public void requireAdminister(DestinationName destination) { + if (!policy.mayAdminister(destination)) { + throw new MessageAuthorizationException( + "DESTINATION_ADMIN_DENIED", + "the admin credential may not administer " + destination.value()); + } + } +} diff --git a/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/MessageSecurityValidator.java b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/MessageSecurityValidator.java new file mode 100644 index 00000000..552ca973 --- /dev/null +++ b/src/messaging/messaging-security/src/main/java/dev/caskeleton/messaging/security/MessageSecurityValidator.java @@ -0,0 +1,50 @@ +package dev.caskeleton.messaging.security; + +import java.util.LinkedHashSet; +import java.util.Objects; +import java.util.Set; + +/** + * Startup validation of a broker's security posture. + * + *

These checks are boot failures rather than warnings. An unencrypted production broker + * connection or a shared producer/admin credential is not a degraded mode the platform can run in + * safely; both are the kind of misconfiguration that stays invisible until it is exploited. + */ +public final class MessageSecurityValidator { + + /** + * Validates one broker security profile. + * + * @param profile the profile to validate + * @throws IllegalArgumentException when the posture is unsafe + */ + public void validate(BrokerSecurityProfile profile) { + Objects.requireNonNull(profile, "profile must not be null"); + + if (profile.production() && !profile.tlsEnabled()) { + throw new IllegalArgumentException( + "a production broker connection requires TLS: " + profile.broker()); + } + if (profile.production() && !profile.hostnameVerification()) { + throw new IllegalArgumentException( + "a production broker connection requires TLS hostname verification: " + profile.broker()); + } + + Set credentialIds = new LinkedHashSet<>(); + credentialIds.add(profile.producer().credentialId()); + if (!credentialIds.add(profile.consumer().credentialId())) { + throw new IllegalArgumentException( + "producer and consumer must use separate credentials: " + profile.broker()); + } + if (profile.admin().isPresent() + && !credentialIds.add(profile.admin().orElseThrow().credentialId())) { + throw new IllegalArgumentException( + "the admin credential must be separate from producer and consumer: " + profile.broker()); + } + if (profile.production() && profile.admin().isPresent()) { + throw new IllegalArgumentException( + "an application runtime must not hold an admin credential: " + profile.broker()); + } + } +} diff --git a/src/messaging/messaging-security/src/test/java/dev/caskeleton/messaging/security/CredentialRuntimeRegistryTest.java b/src/messaging/messaging-security/src/test/java/dev/caskeleton/messaging/security/CredentialRuntimeRegistryTest.java new file mode 100644 index 00000000..4fe4914b --- /dev/null +++ b/src/messaging/messaging-security/src/test/java/dev/caskeleton/messaging/security/CredentialRuntimeRegistryTest.java @@ -0,0 +1,218 @@ +package dev.caskeleton.messaging.security; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; + +class CredentialRuntimeRegistryTest { + + private static final String CREDENTIAL_ID = "kafka.producer"; + private static final Instant NOW = Instant.parse("2026-08-10T09:00:00Z"); + + /** Hands out a fresh generation on each resolve so rotation is observable. */ + private static final class CountingProvider implements CredentialProvider { + + private final AtomicInteger generation = new AtomicInteger(); + private final Optional expiry; + + private CountingProvider(Optional expiry) { + this.expiry = expiry; + } + + @Override + public char[] resolve(String credentialId) { + return ("generation-" + generation.incrementAndGet()).toCharArray(); + } + + @Override + public Optional expiresAt(String credentialId) { + return expiry; + } + + int resolutions() { + return generation.get(); + } + } + + @Test + void aCredentialIsResolvedOnceAndReusedUntilItIsDue() { + CountingProvider provider = new CountingProvider(Optional.of(NOW.plus(Duration.ofHours(4)))); + CredentialRuntimeRegistry registry = new CredentialRuntimeRegistry(provider); + + registry.resolve(CREDENTIAL_ID, NOW); + registry.resolve(CREDENTIAL_ID, NOW.plus(Duration.ofMinutes(5))); + + assertThat(provider.resolutions()).isEqualTo(1); + } + + @Test + void rotationHappensBeforeExpiryNotAfterIt() { + Instant expiry = NOW.plus(Duration.ofMinutes(20)); + CountingProvider provider = new CountingProvider(Optional.of(expiry)); + CredentialRuntimeRegistry registry = new CredentialRuntimeRegistry(provider); + + registry.resolve(CREDENTIAL_ID, NOW); + registry.resolve(CREDENTIAL_ID, NOW.plus(Duration.ofMinutes(1))); + + assertThat(provider.resolutions()) + .as("waiting for the broker to refuse the connection turns rotation into an outage") + .isEqualTo(2); + } + + @Test + void theSupersededGenerationIsWipedNotJustDropped() { + Instant expiry = NOW.plus(Duration.ofMinutes(20)); + CredentialRuntimeRegistry registry = + new CredentialRuntimeRegistry(new CountingProvider(Optional.of(expiry))); + + CredentialRuntime first = registry.resolve(CREDENTIAL_ID, NOW); + registry.resolve(CREDENTIAL_ID, NOW.plus(Duration.ofMinutes(1))); + + assertThatThrownBy(first::material) + .as("a revoked credential must not stay readable in memory") + .isInstanceOf(IllegalStateException.class); + } + + @Test + void aCredentialWithNoKnownExpiryIsNeverRotatedOnATimer() { + CountingProvider provider = new CountingProvider(Optional.empty()); + CredentialRuntimeRegistry registry = new CredentialRuntimeRegistry(provider); + + registry.resolve(CREDENTIAL_ID, NOW); + registry.resolve(CREDENTIAL_ID, NOW.plus(Duration.ofDays(30))); + + assertThat(provider.resolutions()).isEqualTo(1); + } + + @Test + void materialIsHandedOutAsACopySoOneHolderCannotBlindTheOthers() { + CredentialRuntimeRegistry registry = + new CredentialRuntimeRegistry(new CountingProvider(Optional.empty())); + CredentialRuntime credential = registry.resolve(CREDENTIAL_ID, NOW); + + char[] borrowed = credential.material(); + java.util.Arrays.fill(borrowed, 'x'); + + assertThat(credential.material()).containsExactly("generation-1".toCharArray()); + } + + @Test + void theDescriptionNamesTheCredentialWithoutRevealingIt() { + CredentialRuntimeRegistry registry = + new CredentialRuntimeRegistry(new CountingProvider(Optional.empty())); + + String described = registry.resolve(CREDENTIAL_ID, NOW).toString(); + + assertThat(described).contains(CREDENTIAL_ID).doesNotContain("generation-1"); + } + + @Test + void expiredCredentialsAreReportedForAlerting() { + Instant expiry = NOW.plus(Duration.ofMinutes(5)); + CredentialRuntimeRegistry registry = + new CredentialRuntimeRegistry( + new CountingProvider(Optional.of(expiry)), Duration.ofMinutes(1)); + registry.resolve(CREDENTIAL_ID, NOW); + + assertThat(registry.expired(NOW.plus(Duration.ofMinutes(10)))).containsExactly(CREDENTIAL_ID); + } + + @Test + void clearingTheRegistryWipesEverythingItHeld() { + CredentialRuntimeRegistry registry = + new CredentialRuntimeRegistry(new CountingProvider(Optional.empty())); + CredentialRuntime credential = registry.resolve(CREDENTIAL_ID, NOW); + + registry.clearAll(); + + assertThat(registry.size()).isZero(); + assertThatThrownBy(credential::material).isInstanceOf(IllegalStateException.class); + } + + @Test + void aProductionBrokerCannotRunWithoutTls() { + assertThatThrownBy( + () -> new BrokerTlsPolicy().validate(profile(true, false, true), List.of("TLSv1.3"))) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("plaintext"); + } + + @Test + void tlsWithoutHostnameVerificationIsRefusedEvenThoughItLooksEncrypted() { + assertThatThrownBy( + () -> new BrokerTlsPolicy().validate(profile(true, true, false), List.of("TLSv1.3"))) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("hostname verification"); + } + + @Test + void anObsoleteProtocolVersionIsRefused() { + assertThatThrownBy( + () -> + new BrokerTlsPolicy() + .validate(profile(true, true, true), List.of("TLSv1.3", "TLSv1.1"))) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("TLSv1.1"); + } + + @Test + void anApplicationRuntimeMayNotHoldADestructiveGrant() { + BrokerAclManifest manifest = + new BrokerAclManifest( + "orders-api", + List.of( + new BrokerAclManifest.Grant( + BrokerAclManifest.Operation.WRITE, + BrokerAclManifest.ResourceType.DESTINATION, + "orders.*"), + new BrokerAclManifest.Grant( + BrokerAclManifest.Operation.DELETE, + BrokerAclManifest.ResourceType.DESTINATION, + "orders.*"))); + + assertThatThrownBy(manifest::requireApplicationRuntime) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("DELETE"); + } + + @Test + void aGrantTheBrokerHoldsButNobodyDeclaredIsTheFinding() { + BrokerAclManifest.Grant declared = + new BrokerAclManifest.Grant( + BrokerAclManifest.Operation.WRITE, + BrokerAclManifest.ResourceType.DESTINATION, + "orders.*"); + BrokerAclManifest.Grant extra = + new BrokerAclManifest.Grant( + BrokerAclManifest.Operation.DELETE, + BrokerAclManifest.ResourceType.DESTINATION, + "orders.*"); + BrokerAclManifest manifest = new BrokerAclManifest("orders-api", List.of(declared)); + + assertThat(manifest.undeclared(Set.of(declared, extra))) + .as("a missing grant fails loudly on first use; an extra one waits to be abused") + .containsExactly(extra); + } + + private static BrokerSecurityProfile profile( + boolean production, boolean tls, boolean hostnameVerification) { + BrokerCredentialProfile credential = new BrokerCredentialProfile.SaslScram("kafka.producer"); + return new BrokerSecurityProfile( + "kafka", + production, + tls, + hostnameVerification, + credential, + credential, + Optional.empty(), + DestinationAccessPolicy.denyAll()); + } +} diff --git a/src/messaging/messaging-security/src/test/java/dev/caskeleton/messaging/security/MessageSecurityValidatorTest.java b/src/messaging/messaging-security/src/test/java/dev/caskeleton/messaging/security/MessageSecurityValidatorTest.java new file mode 100644 index 00000000..060fe93a --- /dev/null +++ b/src/messaging/messaging-security/src/test/java/dev/caskeleton/messaging/security/MessageSecurityValidatorTest.java @@ -0,0 +1,148 @@ +package dev.caskeleton.messaging.security; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.util.Optional; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class MessageSecurityValidatorTest { + + private final MessageSecurityValidator validator = new MessageSecurityValidator(); + + @Test + void productionBrokerWithoutTlsIsRejected() { + BrokerSecurityProfile profile = BrokerSecurityProfileFixtures.productionWithoutTls(); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("TLS"); + } + + @Test + void productionBrokerWithoutHostnameVerificationIsRejected() { + BrokerSecurityProfile profile = BrokerSecurityProfileFixtures.productionWithoutHostnameCheck(); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("hostname"); + } + + @Test + void producerAndConsumerMustUseSeparateCredentials() { + BrokerSecurityProfile profile = BrokerSecurityProfileFixtures.sharedProducerConsumer(); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("separate credentials"); + } + + @Test + void anApplicationRuntimeMustNotHoldAnAdminCredential() { + BrokerSecurityProfile profile = BrokerSecurityProfileFixtures.productionWithAdmin(); + + assertThatThrownBy(() -> validator.validate(profile)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("admin credential"); + } + + @Test + void aCredentialIdThatLooksLikeASecretIsRejected() { + assertThatThrownBy(() -> new BrokerCredentialProfile.OAuth2("bearer abcdef123456")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void accessPolicySeparatesPublishConsumeAndAdminister() { + DestinationName orders = new DestinationName("order-events"); + DestinationAccessPolicy policy = + new DestinationAccessPolicy(Set.of(orders), Set.of(), Set.of()); + + assertThatCode( + () -> { + if (!policy.mayPublish(orders) + || policy.mayConsume(orders) + || policy.mayAdminister(orders)) { + throw new IllegalStateException("access policy roles are not separated"); + } + }) + .doesNotThrowAnyException(); + } + + @Test + void aCoherentProductionProfileValidates() { + assertThatCode(() -> validator.validate(BrokerSecurityProfileFixtures.production())) + .doesNotThrowAnyException(); + } +} + +/** Builds broker security profiles for validator tests. */ +final class BrokerSecurityProfileFixtures { + + private BrokerSecurityProfileFixtures() {} + + static BrokerSecurityProfile production() { + return new BrokerSecurityProfile( + "kafka-primary", + true, + true, + true, + new BrokerCredentialProfile.SaslScram("kafka-producer"), + new BrokerCredentialProfile.SaslScram("kafka-consumer"), + Optional.empty(), + DestinationAccessPolicy.denyAll()); + } + + static BrokerSecurityProfile productionWithoutTls() { + BrokerSecurityProfile base = production(); + return new BrokerSecurityProfile( + base.broker(), + true, + false, + true, + base.producer(), + base.consumer(), + base.admin(), + base.access()); + } + + static BrokerSecurityProfile productionWithoutHostnameCheck() { + BrokerSecurityProfile base = production(); + return new BrokerSecurityProfile( + base.broker(), + true, + true, + false, + base.producer(), + base.consumer(), + base.admin(), + base.access()); + } + + static BrokerSecurityProfile sharedProducerConsumer() { + return new BrokerSecurityProfile( + "kafka-primary", + false, + true, + true, + new BrokerCredentialProfile.SaslScram("kafka-app"), + new BrokerCredentialProfile.SaslScram("kafka-app"), + Optional.empty(), + DestinationAccessPolicy.denyAll()); + } + + static BrokerSecurityProfile productionWithAdmin() { + BrokerSecurityProfile base = production(); + return new BrokerSecurityProfile( + base.broker(), + true, + true, + true, + base.producer(), + base.consumer(), + Optional.of(new BrokerCredentialProfile.SaslScram("kafka-admin")), + base.access()); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/build.gradle b/src/messaging/messaging-spring-boot-starter/build.gradle new file mode 100644 index 00000000..cda84618 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/build.gradle @@ -0,0 +1,31 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-schema-api') + api project(':messaging:messaging-schema-json') + api project(':messaging:messaging-cloudevents') + api project(':messaging:messaging-policy') + api project(':messaging:messaging-transport-spi') + api project(':messaging:messaging-observability') + api project(':messaging:messaging-security') + api project(':messaging:messaging-kafka') + api project(':messaging:messaging-rabbit') + api project(':messaging:messaging-reliability-api') + api project(':messaging:messaging-outbox-jpa') + api project(':messaging:messaging-inbox-jpa') + api project(':messaging:messaging-claim-check') + api project(':messaging:messaging-admin-api') + api project(':messaging:messaging-admin-runtime') + + implementation 'org.springframework.boot:spring-boot-autoconfigure' + implementation 'org.springframework.boot:spring-boot-actuator' + annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' + + // The Reactor facade lives here, not in core-api: the core contract stays CompletionStage so + // that a service which does not use Reactor never inherits it. + implementation 'io.projectreactor:reactor-core' + testImplementation 'io.projectreactor:reactor-test' + testImplementation 'org.springframework.boot:spring-boot-test' + testImplementation 'org.springframework:spring-test' +} diff --git a/src/messaging/messaging-spring-boot-starter/gradle.lockfile b/src/messaging/messaging-spring-boot-starter/gradle.lockfile new file mode 100644 index 00000000..7e993f38 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/gradle.lockfile @@ -0,0 +1,131 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.fasterxml.jackson.core:jackson-annotations:2.20=runtimeClasspath,testRuntimeClasspath +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.luben:zstd-jni:1.5.6-10=runtimeClasspath,testRuntimeClasspath +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +com.rabbitmq:amqp-client:5.27.1=runtimeClasspath,testRuntimeClasspath +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +commons-logging:commons-logging:1.3.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +info.picocli:picocli:4.7.7=checkstyle +io.cloudevents:cloudevents-api:4.0.1=runtimeClasspath,testRuntimeClasspath +io.cloudevents:cloudevents-core:4.0.1=runtimeClasspath,testRuntimeClasspath +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +io.micrometer:micrometer-commons:1.16.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.micrometer:micrometer-core:1.16.0=runtimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-observation:1.16.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-buffer:4.2.17.Final=runtimeClasspath,testRuntimeClasspath +io.netty:netty-codec-base:4.2.17.Final=runtimeClasspath,testRuntimeClasspath +io.netty:netty-codec-compression:4.2.17.Final=runtimeClasspath,testRuntimeClasspath +io.netty:netty-codec-marshalling:4.2.17.Final=runtimeClasspath,testRuntimeClasspath +io.netty:netty-codec-protobuf:4.2.17.Final=runtimeClasspath,testRuntimeClasspath +io.netty:netty-codec:4.2.17.Final=runtimeClasspath,testRuntimeClasspath +io.netty:netty-common:4.2.17.Final=runtimeClasspath,testRuntimeClasspath +io.netty:netty-handler:4.2.17.Final=runtimeClasspath,testRuntimeClasspath +io.netty:netty-resolver:4.2.17.Final=runtimeClasspath,testRuntimeClasspath +io.netty:netty-transport-native-unix-common:4.2.17.Final=runtimeClasspath,testRuntimeClasspath +io.netty:netty-transport:4.2.17.Final=runtimeClasspath,testRuntimeClasspath +io.projectreactor:reactor-core:3.8.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.projectreactor:reactor-test:3.8.0=testCompileClasspath,testRuntimeClasspath +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.kafka:kafka-clients:4.1.1=runtimeClasspath,testRuntimeClasspath +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.hdrhistogram:HdrHistogram:2.2.2=runtimeClasspath,testRuntimeClasspath +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,compileClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.latencyutils:LatencyUtils:2.0.3=runtimeClasspath,testRuntimeClasspath +org.lz4:lz4-java:1.8.0=runtimeClasspath,testRuntimeClasspath +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reactivestreams:reactive-streams:1.0.4=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=runtimeClasspath,spotbugs,spotbugsSlf4j,testRuntimeClasspath +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.springframework.amqp:spring-amqp:4.0.0=runtimeClasspath,testRuntimeClasspath +org.springframework.amqp:spring-rabbit:4.0.0=runtimeClasspath,testRuntimeClasspath +org.springframework.boot:spring-boot-actuator:4.0.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework.boot:spring-boot-autoconfigure:4.0.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework.boot:spring-boot-configuration-processor:4.0.0=annotationProcessor +org.springframework.boot:spring-boot-test:4.0.0=testCompileClasspath,testRuntimeClasspath +org.springframework.boot:spring-boot:4.0.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework.kafka:spring-kafka:4.0.0=runtimeClasspath,testRuntimeClasspath +org.springframework:spring-aop:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-beans:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-context:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-core:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-expression:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-jdbc:7.0.1=runtimeClasspath,testRuntimeClasspath +org.springframework:spring-messaging:7.0.1=runtimeClasspath,testRuntimeClasspath +org.springframework:spring-test:7.0.1=testCompileClasspath,testRuntimeClasspath +org.springframework:spring-tx:7.0.1=runtimeClasspath,testRuntimeClasspath +org.xerial.snappy:snappy-java:1.1.10.7=runtimeClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +tools.jackson.core:jackson-core:3.0.2=runtimeClasspath,testRuntimeClasspath +tools.jackson.core:jackson-databind:3.0.2=runtimeClasspath,testRuntimeClasspath +tools.jackson:jackson-bom:3.0.2=runtimeClasspath,testRuntimeClasspath +empty= diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DefaultBatchMessagePublisher.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DefaultBatchMessagePublisher.java new file mode 100644 index 00000000..d7ab4206 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DefaultBatchMessagePublisher.java @@ -0,0 +1,118 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.error.MessageTooLargeException; +import dev.caskeleton.messaging.api.publish.BatchMessagePublisher; +import dev.caskeleton.messaging.api.publish.BatchPublishItemResult; +import dev.caskeleton.messaging.api.publish.BatchPublishOptions; +import dev.caskeleton.messaging.api.publish.BatchPublishResult; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishRequest; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.Supplier; + +/** + * Fans a batch out over the single-message publisher and keeps every entry's outcome. + * + *

Entries are submitted concurrently and awaited together, because a batch's value is the + * overlap: submitting serially would make a 500-entry batch take 500 round trips, which is slower + * than the loop the caller was trying to avoid. + * + *

A failed entry never fails the batch. Each entry's exception is converted into that entry's + * result, so an oversized message at index 7 does not hide the outcomes of the other 499 — the + * caller needs to know exactly which entries to resubmit, and a thrown exception tells it nothing. + * + *

{@code stopOnFirstRejection} short-circuits the submission, not the awaiting. Entries + * already in flight when the first rejection arrives cannot be recalled, so they are still awaited + * and reported; pretending otherwise would leave the caller unsure whether they landed. + */ +public final class DefaultBatchMessagePublisher implements BatchMessagePublisher { + + private final MessagePublisher publisher; + private final Supplier clock; + + /** + * Creates a batch publisher over the single-message publisher. + * + * @param publisher the platform publisher + * @param clock supplies the current instant + */ + public DefaultBatchMessagePublisher(MessagePublisher publisher, Supplier clock) { + this.publisher = Objects.requireNonNull(publisher, "publisher must not be null"); + this.clock = Objects.requireNonNull(clock, "clock must not be null"); + } + + @Override + public CompletionStage publish( + List> requests, BatchPublishOptions options) { + Objects.requireNonNull(requests, "requests must not be null"); + Objects.requireNonNull(options, "options must not be null"); + + if (requests.size() > options.maxBatchSize()) { + throw new MessageTooLargeException( + "BATCH_COUNT_EXCEEDED", + "batch of %d entries exceeds the %d entry limit" + .formatted(requests.size(), options.maxBatchSize())); + } + + Instant startedAt = clock.get(); + List> pending = new ArrayList<>(requests.size()); + boolean rejectionSeen = false; + + for (int index = 0; index < requests.size(); index++) { + if (rejectionSeen && options.stopOnFirstRejection()) { + break; + } + pending.add(submit(requests.get(index), index)); + if (options.stopOnFirstRejection()) { + // Only observable for an entry that failed synchronously, which is the local-validation + // case; a broker rejection resolves later and is picked up on the next iteration. + CompletableFuture latest = pending.get(pending.size() - 1); + rejectionSeen = + latest.isDone() && latest.join().result().completion() == PublishCompletion.REJECTED; + } + } + + return CompletableFuture.allOf(pending.toArray(CompletableFuture[]::new)) + .thenApply( + ignored -> + new BatchPublishResult( + pending.stream().map(CompletableFuture::join).toList(), + Duration.between(startedAt, clock.get()))); + } + + private CompletableFuture submit( + PublishRequest request, int index) { + @SuppressWarnings("unchecked") + PublishRequest typed = (PublishRequest) request; + MessageDestination destination = typed.destination(); + MessageEnvelope message = typed.message(); + + try { + return publisher + .publish(destination, message, typed.options()) + .toCompletableFuture() + .handle( + (result, failure) -> + new BatchPublishItemResult( + index, + message.messageId(), + failure == null ? result : PublishResults.from(failure))) + .toCompletableFuture(); + } catch (RuntimeException synchronousFailure) { + // A publisher that validates eagerly throws instead of returning a failed stage. Converting + // it here keeps the "one result per index" contract that the caller resubmits from. + return CompletableFuture.completedFuture( + new BatchPublishItemResult( + index, message.messageId(), PublishResults.from(synchronousFailure))); + } + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DefaultBlockingMessagePublisher.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DefaultBlockingMessagePublisher.java new file mode 100644 index 00000000..9931b917 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DefaultBlockingMessagePublisher.java @@ -0,0 +1,65 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.error.MessagePublishTimeoutException; +import dev.caskeleton.messaging.api.publish.BlockingMessagePublisher; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Converts the asynchronous publish lifecycle into a blocking call. + * + *

Three details make this safe rather than a thread-pool trap. The wait is bounded by the call's + * own deadline. A timeout is reported as {@link MessagePublishTimeoutException}, which the error + * model classifies as ambiguous — the broker may still hold the message, and the caller must not + * treat it as a failed send. And an interrupt restores the thread's interrupt flag instead of + * swallowing it, so a shutdown that interrupts a blocked publisher still unwinds. + */ +public final class DefaultBlockingMessagePublisher implements BlockingMessagePublisher { + + private final MessagePublisher delegate; + + /** + * Creates a blocking facade. + * + * @param delegate the asynchronous publisher + */ + public DefaultBlockingMessagePublisher(MessagePublisher delegate) { + this.delegate = Objects.requireNonNull(delegate, "delegate must not be null"); + } + + @Override + public PublishResult publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options) { + Objects.requireNonNull(options, "options must not be null"); + CompletableFuture future = + delegate.publish(destination, message, options).toCompletableFuture(); + try { + return future.get(options.timeout().toMillis(), TimeUnit.MILLISECONDS); + } catch (TimeoutException exception) { + future.cancel(true); + throw new MessagePublishTimeoutException( + "PUBLISH_TIMEOUT", + "no publish outcome within " + options.timeout() + "; the broker may hold the message", + exception); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + future.cancel(true); + throw new MessagePublishTimeoutException( + "PUBLISH_INTERRUPTED", "the publishing thread was interrupted", exception); + } catch (ExecutionException exception) { + Throwable cause = exception.getCause(); + if (cause instanceof RuntimeException runtime) { + throw runtime; + } + throw new IllegalStateException("publish failed", cause); + } + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DefaultReactiveMessagePublisher.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DefaultReactiveMessagePublisher.java new file mode 100644 index 00000000..a17c0e37 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DefaultReactiveMessagePublisher.java @@ -0,0 +1,39 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import java.util.Objects; +import reactor.core.publisher.Mono; + +/** + * Converts the asynchronous publish lifecycle into a cold, cancellable {@code Mono}. + * + *

{@code Mono.fromCompletionStage} with a supplier is used deliberately: the eager overload + * would start the publish when the {@code Mono} is assembled rather than when it is subscribed, + * which makes a request never sent by an unsubscribed pipeline indistinguishable from one that was. + */ +public final class DefaultReactiveMessagePublisher implements ReactiveMessagePublisher { + + private final MessagePublisher delegate; + + /** + * Creates a Reactor facade. + * + * @param delegate the asynchronous publisher + */ + public DefaultReactiveMessagePublisher(MessagePublisher delegate) { + this.delegate = Objects.requireNonNull(delegate, "delegate must not be null"); + } + + @Override + public Mono publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options) { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(message, "message must not be null"); + Objects.requireNonNull(options, "options must not be null"); + return Mono.fromCompletionStage(() -> delegate.publish(destination, message, options)); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/KafkaMessagingAutoConfiguration.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/KafkaMessagingAutoConfiguration.java new file mode 100644 index 00000000..1c75d69f --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/KafkaMessagingAutoConfiguration.java @@ -0,0 +1,76 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.kafka.KafkaProfileValidator; +import dev.caskeleton.messaging.kafka.KafkaPublishFailureClassifier; +import dev.caskeleton.messaging.kafka.KafkaSecurityConfigurer; +import dev.caskeleton.messaging.kafka.KafkaTransactionProfileValidator; +import dev.caskeleton.messaging.security.BrokerTlsPolicy; +import dev.caskeleton.messaging.security.CredentialRuntimeRegistry; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; + +/** + * Wires the Kafka adapter's validators and security configurer. + * + *

Conditional on the Kafka client being present, so a service that depends on the starter for + * RabbitMQ alone does not fail to start because a Kafka class is missing. That is the reason the + * per-broker configurations are separate classes rather than sections of one: a single class + * referencing both clients would require both on the classpath. + * + *

No producer or consumer bean is created here. Those need a broker profile the application has + * not declared yet at this point, and a default-constructed producer would connect to {@code + * localhost:9092} — which in a test environment usually succeeds against something. + */ +@AutoConfiguration(after = MessagingCoreAutoConfiguration.class) +@ConditionalOnClass(name = "org.apache.kafka.clients.producer.Producer") +public class KafkaMessagingAutoConfiguration { + + /** + * Returns the Kafka destination profile validator. + * + * @return the validator + */ + @Bean + @ConditionalOnMissingBean + public KafkaProfileValidator kafkaProfileValidator() { + return new KafkaProfileValidator(); + } + + /** + * Returns the Kafka transaction profile validator. + * + * @return the validator + */ + @Bean + @ConditionalOnMissingBean + public KafkaTransactionProfileValidator kafkaTransactionProfileValidator() { + return new KafkaTransactionProfileValidator(); + } + + /** + * Returns the producer failure classifier. + * + * @return the classifier + */ + @Bean + @ConditionalOnMissingBean + public KafkaPublishFailureClassifier kafkaPublishFailureClassifier() { + return new KafkaPublishFailureClassifier(); + } + + /** + * Returns the Kafka client security configurer. + * + * @param credentials the credential runtime registry + * @param tlsPolicy the transport security policy + * @return the configurer + */ + @Bean + @ConditionalOnMissingBean + public KafkaSecurityConfigurer kafkaSecurityConfigurer( + CredentialRuntimeRegistry credentials, BrokerTlsPolicy tlsPolicy) { + return new KafkaSecurityConfigurer(credentials, tlsPolicy); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingAdminAutoConfiguration.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingAdminAutoConfiguration.java new file mode 100644 index 00000000..295918c6 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingAdminAutoConfiguration.java @@ -0,0 +1,65 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.admin.DestructiveOperationGuard; +import dev.caskeleton.messaging.admin.runtime.AdminOperationIdempotencyStore; +import dev.caskeleton.messaging.admin.runtime.BrokerTopologyInspector; +import dev.caskeleton.messaging.admin.runtime.CompositeTopologyValidator; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; + +/** + * Wires the admin plane, and only when a deployment has explicitly asked for it. + * + *

Off unless {@code backend.messaging.admin.enabled=true}. An application that acquires the + * admin plane by adding a starter to its classpath is exactly the situation the plane's guards + * exist to prevent — the guards would still refuse an unapproved operation, but the beans would be + * reachable from any code in the process. + * + *

{@link dev.caskeleton.messaging.admin.runtime.DestructiveMessagingAdmin} is deliberately + * absent from this class. No bean for it is ever auto-configured: an operator tool that needs purge + * or delete registers one itself, with an admin credential this runtime does not hold. + */ +@AutoConfiguration(after = MessagingCoreAutoConfiguration.class) +@ConditionalOnProperty(prefix = "backend.messaging.admin", name = "enabled", havingValue = "true") +public class MessagingAdminAutoConfiguration { + + /** + * Returns the guard that authorises non-destructive admin operations. + * + * @return the guard + */ + @Bean + @ConditionalOnMissingBean + public DestructiveOperationGuard destructiveOperationGuard() { + // false: an application runtime never holds an admin credential, so the guard refuses the + // operations that would need one. An operator tool overrides this bean with true. + return new DestructiveOperationGuard(false); + } + + /** + * Returns the store that stops one approval being executed twice. + * + * @return the idempotency store + */ + @Bean + @ConditionalOnMissingBean + public AdminOperationIdempotencyStore adminOperationIdempotencyStore() { + return new AdminOperationIdempotencyStore(); + } + + /** + * Returns the topology validator, when the application supplied a broker inspector. + * + * @param inspector reads the broker's current topology + * @return the composite validator + */ + @Bean + @ConditionalOnBean(BrokerTopologyInspector.class) + @ConditionalOnMissingBean + public CompositeTopologyValidator compositeTopologyValidator(BrokerTopologyInspector inspector) { + return new CompositeTopologyValidator(inspector); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingCoreAutoConfiguration.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingCoreAutoConfiguration.java new file mode 100644 index 00000000..04fcaa6b --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingCoreAutoConfiguration.java @@ -0,0 +1,247 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.api.publish.BatchMessagePublisher; +import dev.caskeleton.messaging.api.publish.BlockingMessagePublisher; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.observation.CardinalityGuard; +import dev.caskeleton.messaging.observation.MessagingRedactor; +import dev.caskeleton.messaging.policy.BackoffCalculator; +import dev.caskeleton.messaging.policy.DeadLetterOrchestrator; +import dev.caskeleton.messaging.policy.DefaultRetryDecisionEngine; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.DestinationProfileValidator; +import dev.caskeleton.messaging.policy.RetryDecisionEngine; +import dev.caskeleton.messaging.security.BrokerTlsPolicy; +import dev.caskeleton.messaging.security.CredentialProvider; +import dev.caskeleton.messaging.security.CredentialRuntimeRegistry; +import dev.caskeleton.messaging.security.MessageSecurityValidator; +import dev.caskeleton.messaging.transport.BackpressureController; +import dev.caskeleton.messaging.transport.DefaultMessagingRuntimeRegistry; +import dev.caskeleton.messaging.transport.GracefulShutdownCoordinator; +import dev.caskeleton.messaging.transport.MessagingRuntimeRegistry; +import java.time.Instant; +import java.util.List; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; + +/** + * Wires the platform's policy, transport, and observability primitives. + * + *

Destination profiles are validated at context refresh through a bean that fails to construct + * when the registry is contradictory. Startup failure is the point: a profile that promises ordered + * delivery while configuring a reordering retry does not fail on the happy path, it fails months + * later on the first retry, looking like a data bug rather than a configuration one. + * + *

The M2 and M3 surfaces are deliberately not auto-configured. Manual settlement, native + * capabilities, and the Admin Plane are opt-in beans with their own credentials, so a service + * cannot acquire them by adding the starter to its classpath. + */ +@AutoConfiguration +@EnableConfigurationProperties(MessagingProperties.class) +public class MessagingCoreAutoConfiguration { + + /** + * Validates every registered destination profile at startup. + * + *

Returns a marker rather than a service because its only job is to fail the context when the + * registry is inconsistent. + * + * @param profiles the registered profiles + * @return the validated registry + */ + @Bean + @ConditionalOnMissingBean + public ValidatedDestinationRegistry validatedDestinationRegistry( + ObjectProvider profiles) { + List registered = profiles.orderedStream().toList(); + new DestinationProfileValidator().validateAll(registered); + return new ValidatedDestinationRegistry(registered); + } + + /** + * Returns the destination profile validator. + * + * @return the validator + */ + @Bean + @ConditionalOnMissingBean + public DestinationProfileValidator destinationProfileValidator() { + return new DestinationProfileValidator(); + } + + /** + * Returns the broker security validator. + * + * @return the validator + */ + @Bean + @ConditionalOnMissingBean + public MessageSecurityValidator messageSecurityValidator() { + return new MessageSecurityValidator(); + } + + /** + * Returns the retry decision engine. + * + * @return the engine + */ + @Bean + @ConditionalOnMissingBean + public RetryDecisionEngine retryDecisionEngine() { + return new DefaultRetryDecisionEngine(new BackoffCalculator()); + } + + /** + * Returns the dead letter orchestrator. + * + * @param publisher the platform publisher + * @return the orchestrator + */ + @Bean + @ConditionalOnMissingBean + public DeadLetterOrchestrator deadLetterOrchestrator(MessagePublisher publisher) { + return new DeadLetterOrchestrator(publisher); + } + + /** + * Returns the runtime registry used for credential and topology rotation. + * + * @param properties the bound configuration + * @return the registry + */ + @Bean + @ConditionalOnMissingBean + public MessagingRuntimeRegistry messagingRuntimeRegistry(MessagingProperties properties) { + return new DefaultMessagingRuntimeRegistry(properties.getShutdown().getDrainDeadline()); + } + + /** + * Returns the process-wide backpressure controller. + * + * @param properties the bound configuration + * @return the controller + */ + @Bean + @ConditionalOnMissingBean + public BackpressureController backpressureController(MessagingProperties properties) { + return new BackpressureController( + properties.getBackpressure().getGlobalLimit(), + properties.getBackpressure().getPerDestinationLimit()); + } + + /** + * Returns the graceful shutdown coordinator. + * + * @param properties the bound configuration + * @return the coordinator + */ + @Bean + @ConditionalOnMissingBean + public GracefulShutdownCoordinator gracefulShutdownCoordinator(MessagingProperties properties) { + return new GracefulShutdownCoordinator(properties.getShutdown().getDrainDeadline()); + } + + /** + * Returns the diagnostic redactor. + * + * @return the redactor + */ + @Bean + @ConditionalOnMissingBean + public MessagingRedactor messagingRedactor() { + return new MessagingRedactor(); + } + + /** + * Returns the metric cardinality guard. + * + * @return the guard + */ + @Bean + @ConditionalOnMissingBean + public CardinalityGuard cardinalityGuard() { + return new CardinalityGuard(); + } + + /** + * Returns the blocking publish facade. + * + * @param publisher the asynchronous publisher + * @return the blocking facade + */ + @Bean + @ConditionalOnMissingBean + public BlockingMessagePublisher blockingMessagePublisher(MessagePublisher publisher) { + return new DefaultBlockingMessagePublisher(publisher); + } + + /** + * Returns the Reactor publish facade. + * + * @param publisher the asynchronous publisher + * @return the Reactor facade + */ + @Bean + @ConditionalOnMissingBean + public ReactiveMessagePublisher reactiveMessagePublisher(MessagePublisher publisher) { + return new DefaultReactiveMessagePublisher(publisher); + } + + /** + * Returns the M2 batch publish facade. + * + * @param publisher the asynchronous publisher + * @return the batch publisher + */ + @Bean + @ConditionalOnMissingBean + public BatchMessagePublisher batchMessagePublisher(MessagePublisher publisher) { + return new DefaultBatchMessagePublisher(publisher, Instant::now); + } + + /** + * Returns the credential runtime registry, when the application supplied a credential provider. + * + *

Conditional on the provider because there is no safe default: a registry with no source + * would resolve nothing, and the adapters would connect with whatever their client's defaults + * are. + * + * @param provider the application's credential source + * @return the credential runtime registry + */ + @Bean + @ConditionalOnBean(CredentialProvider.class) + @ConditionalOnMissingBean + public CredentialRuntimeRegistry credentialRuntimeRegistry(CredentialProvider provider) { + return new CredentialRuntimeRegistry(provider); + } + + /** + * Returns the transport security policy. + * + * @return the TLS policy + */ + @Bean + @ConditionalOnMissingBean + public BrokerTlsPolicy brokerTlsPolicy() { + return new BrokerTlsPolicy(); + } + + /** + * Returns the read-only actuator endpoint. + * + * @param registry the validated destination registry + * @param backpressure the in-flight counters + * @return the endpoint + */ + @Bean + @ConditionalOnMissingBean + public MessagingEndpoint messagingEndpoint( + ValidatedDestinationRegistry registry, BackpressureController backpressure) { + return new MessagingEndpoint(registry, backpressure); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpoint.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpoint.java new file mode 100644 index 00000000..faeba7bf --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpoint.java @@ -0,0 +1,74 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.transport.BackpressureController; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.springframework.boot.actuate.endpoint.annotation.Endpoint; +import org.springframework.boot.actuate.endpoint.annotation.ReadOperation; + +/** + * Exposes what the messaging runtime is currently doing, over the actuator. + * + *

Read-only. An actuator endpoint is reachable by anything that can reach the management port, + * so a write operation here would be an unauthenticated pause or purge in most deployments — the + * admin plane exists for those, with approvals and an audit trail this endpoint deliberately does + * not duplicate. + * + *

What it reports is bounded and free of per-message identity: destination names, guarantees, + * capability tiers, and in-flight counts. No message ids, no keys, no payloads — an actuator + * response is a diagnostic surface, and a diagnostic surface that echoes message content is a + * re-identification surface too. + */ +@Endpoint(id = "messaging") +public class MessagingEndpoint { + + private final ValidatedDestinationRegistry registry; + private final BackpressureController backpressure; + + /** + * Creates the endpoint. + * + * @param registry the validated destination registry + * @param backpressure the in-flight counters + */ + public MessagingEndpoint( + ValidatedDestinationRegistry registry, BackpressureController backpressure) { + this.registry = Objects.requireNonNull(registry, "registry must not be null"); + this.backpressure = Objects.requireNonNull(backpressure, "backpressure must not be null"); + } + + /** + * Returns the current messaging state. + * + * @return the destinations, their declared guarantees, and the in-flight counts + */ + @ReadOperation + public Map messaging() { + Map report = new LinkedHashMap<>(); + report.put("globalInFlight", backpressure.globalInFlight()); + report.put("destinations", destinations()); + return Map.copyOf(report); + } + + private List> destinations() { + return registry.all().values().stream().map(this::describe).toList(); + } + + private Map describe(DestinationProfile profile) { + Map entry = new LinkedHashMap<>(); + entry.put("name", profile.name().value()); + entry.put("broker", profile.broker()); + entry.put("kind", profile.kind().name()); + entry.put("deliveryGuarantee", profile.deliveryGuarantee().name()); + entry.put("orderingScope", profile.orderingScope().name()); + entry.put("tier", profile.tier().name()); + entry.put("production", profile.production()); + entry.put("deadLetterEnabled", profile.deadLetter().enabled()); + entry.put("retryMode", profile.retry().mode().name()); + entry.put("inFlight", backpressure.inFlight(profile.name().value())); + return Map.copyOf(entry); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingProperties.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingProperties.java new file mode 100644 index 00000000..2984fd37 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingProperties.java @@ -0,0 +1,210 @@ +package dev.caskeleton.messaging.autoconfigure; + +import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * The platform's bound configuration. + * + *

Every Experimental and bridge flag defaults to {@code false}. An adapter whose contract suite + * is still being proven must not become load-bearing because a default turned it on, and the same + * applies to the Spring Cloud Stream bridge, which bypasses the platform's retry and dead letter + * policy by design. + */ +@ConfigurationProperties(prefix = "backend.messaging") +public class MessagingProperties { + + private final Experimental experimental = new Experimental(); + private final Bridge bridge = new Bridge(); + private final Backpressure backpressure = new Backpressure(); + private final Shutdown shutdown = new Shutdown(); + + /** + * Returns the experimental adapter switches. + * + * @return the experimental settings + */ + public Experimental getExperimental() { + return experimental; + } + + /** + * Returns the optional bridge switches. + * + * @return the bridge settings + */ + public Bridge getBridge() { + return bridge; + } + + /** + * Returns the backpressure limits. + * + * @return the backpressure settings + */ + public Backpressure getBackpressure() { + return backpressure; + } + + /** + * Returns the shutdown settings. + * + * @return the shutdown settings + */ + public Shutdown getShutdown() { + return shutdown; + } + + /** Experimental adapter switches, all off by default. */ + public static class Experimental { + + private boolean kafkaShare; + private boolean pulsar; + private boolean nats; + + /** + * Reports whether the Kafka Share Group adapter is enabled. + * + * @return true when enabled + */ + public boolean isKafkaShare() { + return kafkaShare; + } + + /** + * Enables or disables the Kafka Share Group adapter. + * + * @param kafkaShare whether to enable it + */ + public void setKafkaShare(boolean kafkaShare) { + this.kafkaShare = kafkaShare; + } + + /** + * Reports whether the Pulsar adapter is enabled. + * + * @return true when enabled + */ + public boolean isPulsar() { + return pulsar; + } + + /** + * Enables or disables the Pulsar adapter. + * + * @param pulsar whether to enable it + */ + public void setPulsar(boolean pulsar) { + this.pulsar = pulsar; + } + + /** + * Reports whether the NATS JetStream adapter is enabled. + * + * @return true when enabled + */ + public boolean isNats() { + return nats; + } + + /** + * Enables or disables the NATS JetStream adapter. + * + * @param nats whether to enable it + */ + public void setNats(boolean nats) { + this.nats = nats; + } + } + + /** Optional integration bridges, off by default. */ + public static class Bridge { + + private boolean springCloudStream; + + /** + * Reports whether the Spring Cloud Stream bridge is enabled. + * + * @return true when enabled + */ + public boolean isSpringCloudStream() { + return springCloudStream; + } + + /** + * Enables or disables the Spring Cloud Stream bridge. + * + * @param springCloudStream whether to enable it + */ + public void setSpringCloudStream(boolean springCloudStream) { + this.springCloudStream = springCloudStream; + } + } + + /** In-flight limits. */ + public static class Backpressure { + + private int globalLimit = 512; + private int perDestinationLimit = 64; + + /** + * Returns the process-wide in-flight ceiling. + * + * @return the global limit + */ + public int getGlobalLimit() { + return globalLimit; + } + + /** + * Sets the process-wide in-flight ceiling. + * + * @param globalLimit the global limit + */ + public void setGlobalLimit(int globalLimit) { + this.globalLimit = globalLimit; + } + + /** + * Returns the per-destination in-flight ceiling. + * + * @return the per-destination limit + */ + public int getPerDestinationLimit() { + return perDestinationLimit; + } + + /** + * Sets the per-destination in-flight ceiling. + * + * @param perDestinationLimit the per-destination limit + */ + public void setPerDestinationLimit(int perDestinationLimit) { + this.perDestinationLimit = perDestinationLimit; + } + } + + /** Graceful shutdown settings. */ + public static class Shutdown { + + private Duration drainDeadline = Duration.ofSeconds(30); + + /** + * Returns how long a drain waits for in-flight work. + * + * @return the drain deadline + */ + public Duration getDrainDeadline() { + return drainDeadline; + } + + /** + * Sets how long a drain waits for in-flight work. + * + * @param drainDeadline the drain deadline + */ + public void setDrainDeadline(Duration drainDeadline) { + this.drainDeadline = drainDeadline; + } + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingReliabilityAutoConfiguration.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingReliabilityAutoConfiguration.java new file mode 100644 index 00000000..706c0f12 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingReliabilityAutoConfiguration.java @@ -0,0 +1,113 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.inbox.IdempotentConsumer; +import dev.caskeleton.messaging.inbox.InboxCleanupJob; +import dev.caskeleton.messaging.inbox.InboxRetentionPolicy; +import dev.caskeleton.messaging.inbox.TransactionalInboxHandler; +import dev.caskeleton.messaging.outbox.OutboxCleanupJob; +import dev.caskeleton.messaging.outbox.OutboxProperties; +import dev.caskeleton.messaging.outbox.OutboxRetryScheduler; +import dev.caskeleton.messaging.reliability.InboxRepository; +import dev.caskeleton.messaging.reliability.OutboxRepository; +import java.time.Duration; +import java.time.Instant; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; + +/** + * Wires the outbox and inbox operational beans. + * + *

Every bean here is conditional on the application having supplied the corresponding + * repository. The platform cannot provide those: they write inside the application's own + * transaction, against the application's own datasource, and a default implementation would + * silently write to the wrong place — or to nowhere at all, which is worse because the outbox would + * look healthy while nothing was ever staged. + * + *

The cleanup jobs are beans but no scheduler is registered for them. Scheduling is the + * application's decision: a service running several replicas usually wants one of them to run + * cleanup, and auto-registering a fixed-rate task would have every replica delete the same rows. + */ +@AutoConfiguration(after = MessagingCoreAutoConfiguration.class) +public class MessagingReliabilityAutoConfiguration { + + /** + * Returns the relay's operational settings. + * + * @return the outbox properties + */ + @Bean + @ConditionalOnMissingBean + public OutboxProperties outboxProperties() { + return OutboxProperties.defaults(); + } + + /** + * Returns the relay pass scheduler. + * + * @param properties the relay settings + * @return the scheduler + */ + @Bean + @ConditionalOnMissingBean + public OutboxRetryScheduler outboxRetryScheduler(OutboxProperties properties) { + return new OutboxRetryScheduler(properties, Duration.ofMinutes(1)); + } + + /** + * Returns the outbox cleanup job. + * + * @param outbox the application's outbox repository + * @param properties the relay settings supplying the retention + * @return the cleanup job + */ + @Bean + @ConditionalOnBean(OutboxRepository.class) + @ConditionalOnMissingBean + public OutboxCleanupJob outboxCleanupJob(OutboxRepository outbox, OutboxProperties properties) { + return new OutboxCleanupJob(outbox, properties, 20); + } + + /** + * Returns the inbox retention policy. + * + *

The redelivery window defaults to one day, which suits a broker retaining a day of log. A + * deployment whose broker retains longer must override this bean — the policy validates itself, + * so an unsafe combination fails at startup rather than duplicating a side effect later. + * + * @return the retention policy + */ + @Bean + @ConditionalOnMissingBean + public InboxRetentionPolicy inboxRetentionPolicy() { + return new InboxRetentionPolicy(InboxRetentionPolicy.DEFAULT_RETENTION, Duration.ofDays(1)); + } + + /** + * Returns the inbox cleanup job. + * + * @param inbox the application's inbox repository + * @param policy the retention policy + * @return the cleanup job + */ + @Bean + @ConditionalOnBean(InboxRepository.class) + @ConditionalOnMissingBean + public InboxCleanupJob inboxCleanupJob(InboxRepository inbox, InboxRetentionPolicy policy) { + return new InboxCleanupJob(inbox, policy, 20); + } + + /** + * Returns the transactional inbox handler. + * + * @param consumer the reservation-and-effect runner the application supplied + * @return the handler + */ + @Bean + @ConditionalOnBean(IdempotentConsumer.class) + @ConditionalOnMissingBean + public TransactionalInboxHandler transactionalInboxHandler(IdempotentConsumer consumer) { + return new TransactionalInboxHandler<>(consumer, Instant::now); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/PublishResults.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/PublishResults.java new file mode 100644 index 00000000..4dd0178d --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/PublishResults.java @@ -0,0 +1,71 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.error.MessagingException; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.CompletionException; + +/** + * Converts an exception that escaped a publish into a result the caller can act on. + * + *

A {@link MessagingException} already carries a sanitized descriptor, including whether it is + * retryable, so its own classification is used. Anything else is reported {@code AMBIGUOUS}: an + * unrecognised exception proves nothing about whether the message reached the broker, and the + * conservative reading is the one that does not turn a lost confirmation into a duplicate. + */ +final class PublishResults { + + private PublishResults() {} + + /** + * Converts a failure into a publish result. + * + * @param failure the exception that escaped the publish + * @return the equivalent result + */ + static PublishResult from(Throwable failure) { + Throwable cause = failure instanceof CompletionException ? failure.getCause() : failure; + + if (cause instanceof MessagingException messagingFailure) { + FailureDescriptor descriptor = messagingFailure.failure(); + boolean transmitted = descriptor.category() == FailureCategory.AMBIGUOUS; + return new PublishResult( + transmitted ? PublishCompletion.AMBIGUOUS : PublishCompletion.REJECTED, + transmitted + ? new PublishEvidence( + true, + TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED, + false, + ConfirmationLevel.NONE) + : PublishEvidence.notTransmitted(), + transmitted ? RoutingOutcome.UNKNOWN : RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of(descriptor)); + } + + return new PublishResult( + PublishCompletion.AMBIGUOUS, + new PublishEvidence( + true, TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED, false, ConfirmationLevel.NONE), + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of( + FailureDescriptor.of( + FailureCategory.AMBIGUOUS, + "UNCLASSIFIED_PUBLISH_FAILURE", + "the publish failed with an unrecognised error: " + + (cause == null ? "unknown" : cause.getClass().getSimpleName())))); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/RabbitMessagingAutoConfiguration.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/RabbitMessagingAutoConfiguration.java new file mode 100644 index 00000000..be25ea45 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/RabbitMessagingAutoConfiguration.java @@ -0,0 +1,62 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.rabbit.RabbitProfileValidator; +import dev.caskeleton.messaging.rabbit.RabbitPublishFailureClassifier; +import dev.caskeleton.messaging.rabbit.RabbitSecurityConfigurer; +import dev.caskeleton.messaging.security.BrokerTlsPolicy; +import dev.caskeleton.messaging.security.CredentialRuntimeRegistry; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; + +/** + * Wires the RabbitMQ adapter's validators and security configurer. + * + *

Conditional on the AMQP client, for the same reason the Kafka configuration is conditional on + * the Kafka client: a service using one broker must not need the other on its classpath. + * + *

No connection or channel bean is created here. A channel opened before the application has + * declared its security profile would connect with whatever defaults the client carries, which for + * RabbitMQ means {@code guest/guest} against localhost. + */ +@AutoConfiguration(after = MessagingCoreAutoConfiguration.class) +@ConditionalOnClass(name = "com.rabbitmq.client.Channel") +public class RabbitMessagingAutoConfiguration { + + /** + * Returns the RabbitMQ destination profile validator. + * + * @return the validator + */ + @Bean + @ConditionalOnMissingBean + public RabbitProfileValidator rabbitProfileValidator() { + return new RabbitProfileValidator(); + } + + /** + * Returns the publish failure classifier. + * + * @return the classifier + */ + @Bean + @ConditionalOnMissingBean + public RabbitPublishFailureClassifier rabbitPublishFailureClassifier() { + return new RabbitPublishFailureClassifier(); + } + + /** + * Returns the AMQP security configurer. + * + * @param credentials the credential runtime registry + * @param tlsPolicy the transport security policy + * @return the configurer + */ + @Bean + @ConditionalOnMissingBean + public RabbitSecurityConfigurer rabbitSecurityConfigurer( + CredentialRuntimeRegistry credentials, BrokerTlsPolicy tlsPolicy) { + return new RabbitSecurityConfigurer(credentials, tlsPolicy); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/ReactiveMessagePublisher.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/ReactiveMessagePublisher.java new file mode 100644 index 00000000..f59bdf12 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/ReactiveMessagePublisher.java @@ -0,0 +1,32 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import reactor.core.publisher.Mono; + +/** + * The Reactor facade over the asynchronous publisher. + * + *

Lives in the starter rather than the core so that {@code messaging-core-api} never depends on + * Reactor. A service that does not use Reactor should not inherit it through a messaging contract. + * + *

The returned {@code Mono} is cold and cancellable: nothing is published until it is + * subscribed, and cancelling the subscription cancels the underlying stage rather than leaving an + * orphaned publish in flight. + */ +public interface ReactiveMessagePublisher { + + /** + * Publishes one message. + * + * @param the payload type + * @param destination the logical destination + * @param message the envelope to publish + * @param options per-call options + * @return a cold {@code Mono} emitting the outcome and its evidence + */ + Mono publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options); +} diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/ValidatedDestinationRegistry.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/ValidatedDestinationRegistry.java new file mode 100644 index 00000000..ebb66283 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/ValidatedDestinationRegistry.java @@ -0,0 +1,59 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * The registry of destination profiles that passed startup validation. + * + *

Constructing this bean is what proves the registry is coherent; nothing downstream + * re-validates it. Resolving an unregistered destination is a configuration failure rather than an + * empty {@code Optional}, because a publish to a destination nobody declared has no profile, no + * limits, and no dead letter route. + */ +public final class ValidatedDestinationRegistry { + + private final Map byName; + + /** + * Creates a registry over validated profiles. + * + * @param profiles the validated profiles + */ + public ValidatedDestinationRegistry(List profiles) { + Objects.requireNonNull(profiles, "profiles must not be null"); + Map registry = new LinkedHashMap<>(); + profiles.forEach(profile -> registry.put(profile.name(), profile)); + this.byName = Map.copyOf(registry); + } + + /** + * Resolves a destination profile. + * + * @param destination the logical destination + * @return the profile + */ + public DestinationProfile require(DestinationName destination) { + DestinationProfile profile = byName.get(destination); + if (profile == null) { + throw new MessagingConfigurationException( + "DESTINATION_NOT_REGISTERED", + "no destination profile is registered for " + destination.value()); + } + return profile; + } + + /** + * Returns every registered profile. + * + * @return the profiles, keyed by logical name + */ + public Map all() { + return byName; + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/src/messaging/messaging-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 00000000..fba3e8cc --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1,5 @@ +dev.caskeleton.messaging.autoconfigure.MessagingCoreAutoConfiguration +dev.caskeleton.messaging.autoconfigure.KafkaMessagingAutoConfiguration +dev.caskeleton.messaging.autoconfigure.RabbitMessagingAutoConfiguration +dev.caskeleton.messaging.autoconfigure.MessagingReliabilityAutoConfiguration +dev.caskeleton.messaging.autoconfigure.MessagingAdminAutoConfiguration diff --git a/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/BatchPublisherTest.java b/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/BatchPublisherTest.java new file mode 100644 index 00000000..04ce35f9 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/BatchPublisherTest.java @@ -0,0 +1,227 @@ +package dev.caskeleton.messaging.autoconfigure; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.error.MessageTooLargeException; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.publish.BatchPublishOptions; +import dev.caskeleton.messaging.api.publish.BatchPublishResult; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishRequest; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import org.junit.jupiter.api.Test; + +class BatchPublisherTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + private static PublishResult confirmed() { + return new PublishResult( + PublishCompletion.CONFIRMED, + new PublishEvidence( + true, + TransmissionEvidence.TRANSMITTED, + true, + ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK), + RoutingOutcome.ROUTED, + Optional.empty(), + 1, + Duration.ZERO, + Optional.empty()); + } + + /** Publishes according to a per-index script, so partial failures are reproducible. */ + private static MessagePublisher scripted(List script, List calls) { + return new MessagePublisher() { + private int index; + + @Override + public CompletionStage publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options) { + int current = index++; + calls.add(current); + return switch (script.get(current)) { + case CONFIRM -> CompletableFuture.completedFuture(confirmed()); + case FAIL_ASYNC -> + CompletableFuture.failedFuture( + new MessageTooLargeException("PAYLOAD_LIMIT_EXCEEDED", "too large")); + case THROW_SYNC -> + throw new MessageTooLargeException("PAYLOAD_LIMIT_EXCEEDED", "too large"); + }; + } + }; + } + + private enum Outcome { + CONFIRM, + FAIL_ASYNC, + THROW_SYNC + } + + private static PublishRequest request() { + MessageDestination destination = + new MessageDestination<>( + new DestinationName("orders.v1"), new MessageType("order.created"), String.class); + MessageEnvelope envelope = + new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + NOW, + Optional.of(NOW), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + "{}"); + return PublishRequest.of(destination, envelope); + } + + private static List> requests(int count) { + List> requests = new ArrayList<>(); + for (int index = 0; index < count; index++) { + requests.add(request()); + } + return requests; + } + + private static BatchPublishResult publish(List script, BatchPublishOptions options) { + List calls = new ArrayList<>(); + return new DefaultBatchMessagePublisher(scripted(script, calls), () -> NOW) + .publish(requests(script.size()), options) + .toCompletableFuture() + .join(); + } + + @Test + void everyEntryGetsItsOwnResultKeyedByIndex() { + BatchPublishResult result = + publish( + List.of(Outcome.CONFIRM, Outcome.CONFIRM, Outcome.CONFIRM), + BatchPublishOptions.defaults()); + + assertThat(result.items()).hasSize(3); + assertThat(result.items()).extracting(item -> item.index()).containsExactly(0, 1, 2); + assertThat(result.allConfirmed()).isTrue(); + } + + @Test + void oneFailedEntryDoesNotHideTheOthers() { + BatchPublishResult result = + publish( + List.of(Outcome.CONFIRM, Outcome.FAIL_ASYNC, Outcome.CONFIRM), + BatchPublishOptions.defaults()); + + assertThat(result.withCompletion(PublishCompletion.CONFIRMED)).hasSize(2); + assertThat(result.withCompletion(PublishCompletion.REJECTED)) + .as("the caller needs to know exactly which entries to resubmit") + .hasSize(1); + } + + @Test + void aSynchronousFailureStillProducesAnEntryResult() { + BatchPublishResult result = + publish(List.of(Outcome.THROW_SYNC, Outcome.CONFIRM), BatchPublishOptions.defaults()); + + assertThat(result.items()).hasSize(2); + assertThat(result.items().get(0).result().completion()).isEqualTo(PublishCompletion.REJECTED); + } + + @Test + void aRejectedEntryKeepsItsSanitizedFailureCode() { + BatchPublishResult result = + publish(List.of(Outcome.FAIL_ASYNC), BatchPublishOptions.defaults()); + + assertThat(result.items().get(0).result().failure()) + .hasValueSatisfying( + failure -> assertThat(failure.code()).isEqualTo("PAYLOAD_LIMIT_EXCEEDED")); + } + + @Test + void aBatchLargerThanItsLimitIsRefusedBeforeAnythingIsPublished() { + List calls = new ArrayList<>(); + DefaultBatchMessagePublisher publisher = + new DefaultBatchMessagePublisher(scripted(List.of(Outcome.CONFIRM), calls), () -> NOW); + + assertThatThrownBy( + () -> + publisher.publish( + requests(5), new BatchPublishOptions(Duration.ofSeconds(5), 2, false))) + .isInstanceOf(MessageTooLargeException.class); + assertThat(calls).isEmpty(); + } + + @Test + void stopOnFirstRejectionSkipsTheRemainingSubmissions() { + List calls = new ArrayList<>(); + BatchPublishResult result = + new DefaultBatchMessagePublisher( + scripted( + List.of(Outcome.CONFIRM, Outcome.THROW_SYNC, Outcome.CONFIRM, Outcome.CONFIRM), + calls), + () -> NOW) + .publish(requests(4), new BatchPublishOptions(Duration.ofSeconds(5), 100, true)) + .toCompletableFuture() + .join(); + + assertThat(calls).containsExactly(0, 1); + assertThat(result.items()).hasSize(2); + } + + @Test + void withoutStopOnFirstRejectionEveryEntryIsAttempted() { + List calls = new ArrayList<>(); + new DefaultBatchMessagePublisher( + scripted(List.of(Outcome.THROW_SYNC, Outcome.CONFIRM, Outcome.CONFIRM), calls), + () -> NOW) + .publish(requests(3), BatchPublishOptions.defaults()) + .toCompletableFuture() + .join(); + + assertThat(calls).containsExactly(0, 1, 2); + } + + @Test + void aBatchIsNotATransactionSoConfirmedEntriesStayConfirmed() { + BatchPublishResult result = + publish(List.of(Outcome.CONFIRM, Outcome.FAIL_ASYNC), BatchPublishOptions.defaults()); + + assertThat(result.items().get(0).result().completion()) + .as("nothing can un-publish an entry the broker already confirmed") + .isEqualTo(PublishCompletion.CONFIRMED); + assertThat(result.allConfirmed()).isFalse(); + } + + @Test + void theDefaultOptionsAttemptEveryEntry() { + assertThat(BatchPublishOptions.defaults().stopOnFirstRejection()).isFalse(); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/BlockingFacadeTest.java b/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/BlockingFacadeTest.java new file mode 100644 index 00000000..52b399bc --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/BlockingFacadeTest.java @@ -0,0 +1,149 @@ +package dev.caskeleton.messaging.autoconfigure; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.error.MessagePublishTimeoutException; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.publish.BatchPublishItemResult; +import dev.caskeleton.messaging.api.publish.BatchPublishResult; +import dev.caskeleton.messaging.api.publish.BlockingMessagePublisher; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import org.junit.jupiter.api.Test; + +class BlockingFacadeTest { + + private static final MessageDestination ORDERS = + new MessageDestination<>( + new DestinationName("order-events"), new MessageType("order.created"), String.class); + + @Test + void aConfirmedPublishIsReturnedDirectly() { + BlockingMessagePublisher publisher = + new DefaultBlockingMessagePublisher( + new MessagePublisher() { + @Override + public CompletionStage publish( + MessageDestination destination, + MessageEnvelope message, + PublishOptions options) { + return CompletableFuture.completedFuture(confirmed()); + } + }); + + PublishResult result = publisher.publish(ORDERS, envelope(), PublishOptions.defaults()); + + assertThat(result.completion()).isEqualTo(PublishCompletion.CONFIRMED); + } + + @Test + void aStalledPublishTimesOutAsAmbiguousRatherThanBlockingForever() { + MessagePublisher stalled = + new MessagePublisher() { + @Override + public CompletionStage publish( + MessageDestination destination, + MessageEnvelope message, + PublishOptions options) { + return new CompletableFuture<>(); + } + }; + PublishOptions shortDeadline = + new PublishOptions( + Duration.ofMillis(50), + dev.caskeleton.messaging.api.destination.ConfirmationRequirement + .REPLICATION_OR_PERSISTENCE_ACK, + Optional.empty(), + java.util.Map.of()); + + assertThatThrownBy( + () -> + new DefaultBlockingMessagePublisher(stalled) + .publish(ORDERS, envelope(), shortDeadline)) + .isInstanceOf(MessagePublishTimeoutException.class) + .satisfies( + thrown -> + assertThat(((MessagePublishTimeoutException) thrown).category()) + .isEqualTo(FailureCategory.AMBIGUOUS)); + } + + @Test + void aBatchKeepsPerItemOutcomes() { + BatchPublishResult batch = + new BatchPublishResult( + List.of( + new BatchPublishItemResult(0, MessageId.newId(), confirmed()), + new BatchPublishItemResult(1, MessageId.newId(), ambiguous())), + Duration.ofMillis(20)); + + assertThat(batch.allConfirmed()).isFalse(); + assertThat(batch.withCompletion(PublishCompletion.CONFIRMED)).hasSize(1); + assertThat(batch.withCompletion(PublishCompletion.AMBIGUOUS)).hasSize(1); + } + + private static MessageEnvelope envelope() { + return new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + Instant.parse("2026-08-10T09:15:00Z"), + Optional.of(Instant.parse("2026-08-10T09:15:00Z")), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + "payload"); + } + + private static PublishResult confirmed() { + return new PublishResult( + PublishCompletion.CONFIRMED, + PublishEvidence.confirmed(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ofMillis(3), + Optional.empty()); + } + + private static PublishResult ambiguous() { + return new PublishResult( + PublishCompletion.AMBIGUOUS, + PublishEvidence.ambiguous(), + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + Duration.ofSeconds(5), + Optional.of( + FailureDescriptor.of( + FailureCategory.AMBIGUOUS, "CONFIRM_TIMEOUT", "confirm timed out"))); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingAutoConfigurationTest.java b/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingAutoConfigurationTest.java new file mode 100644 index 00000000..24fc68fc --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingAutoConfigurationTest.java @@ -0,0 +1,238 @@ +package dev.caskeleton.messaging.autoconfigure; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.delivery.ExternalSideEffectGuarantee; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.DestinationKind; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.publish.BlockingMessagePublisher; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.policy.CapabilityTier; +import dev.caskeleton.messaging.policy.ConsumerPolicy; +import dev.caskeleton.messaging.policy.DeadLetterPolicy; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.PayloadPolicy; +import dev.caskeleton.messaging.policy.PhysicalDestination; +import dev.caskeleton.messaging.policy.ProducerPolicy; +import dev.caskeleton.messaging.policy.RetryPolicy; +import dev.caskeleton.messaging.policy.SchemaPolicy; +import dev.caskeleton.messaging.schema.SchemaCompatibility; +import dev.caskeleton.messaging.transport.BackpressureController; +import dev.caskeleton.messaging.transport.GracefulShutdownCoordinator; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +class MessagingAutoConfigurationTest { + + private final ApplicationContextRunner runner = + new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(MessagingCoreAutoConfiguration.class)) + .withUserConfiguration(PublisherConfiguration.class); + + @Test + void wiresThePolicyTransportAndObservabilityPrimitives() { + runner.run( + context -> + assertThat(context) + .hasSingleBean(ValidatedDestinationRegistry.class) + .hasSingleBean(BlockingMessagePublisher.class) + .hasSingleBean(ReactiveMessagePublisher.class) + .hasSingleBean(BackpressureController.class) + .hasSingleBean(GracefulShutdownCoordinator.class)); + } + + @Test + void experimentalAdaptersAndTheBridgeAreOffByDefault() { + runner.run( + context -> { + MessagingProperties properties = context.getBean(MessagingProperties.class); + assertThat(properties.getExperimental().isKafkaShare()).isFalse(); + assertThat(properties.getExperimental().isPulsar()).isFalse(); + assertThat(properties.getExperimental().isNats()).isFalse(); + assertThat(properties.getBridge().isSpringCloudStream()).isFalse(); + }); + } + + @Test + void propertiesBindFromConfiguration() { + runner + .withPropertyValues( + "backend.messaging.experimental.pulsar=true", + "backend.messaging.backpressure.global-limit=64", + "backend.messaging.backpressure.per-destination-limit=8", + "backend.messaging.shutdown.drain-deadline=15s") + .run( + context -> { + MessagingProperties properties = context.getBean(MessagingProperties.class); + assertThat(properties.getExperimental().isPulsar()).isTrue(); + assertThat(properties.getBackpressure().getGlobalLimit()).isEqualTo(64); + assertThat(properties.getShutdown().getDrainDeadline()) + .isEqualTo(java.time.Duration.ofSeconds(15)); + }); + } + + @Test + void aCoherentDestinationProfileIsRegistered() { + runner + .withUserConfiguration(CoherentProfileConfiguration.class) + .run( + context -> + assertThat( + context + .getBean(ValidatedDestinationRegistry.class) + .require(new DestinationName("order-events"))) + .isNotNull()); + } + + @Test + void aContradictoryDestinationProfileFailsTheContext() { + runner + .withUserConfiguration(ContradictoryProfileConfiguration.class) + .run( + context -> + assertThat(context).hasFailed().getFailure().hasMessageContaining("ordering")); + } + + @Test + void backpressureLimitsAreValidatedAtStartup() { + runner + .withPropertyValues( + "backend.messaging.backpressure.global-limit=4", + "backend.messaging.backpressure.per-destination-limit=8") + .run(context -> assertThat(context).hasFailed()); + } + + /** Supplies the publisher the facades and orchestrator depend on. */ + @Configuration(proxyBeanMethods = false) + static class PublisherConfiguration { + + @Bean + MessagePublisher messagePublisher() { + return new MessagePublisher() { + @Override + public CompletionStage publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options) { + return CompletableFuture.failedFuture(new UnsupportedOperationException("not used")); + } + }; + } + } + + @Test + void aDeadLetterDestinationThatIsNotRegisteredFailsTheContext() { + runner + .withUserConfiguration(DanglingDeadLetterConfiguration.class) + .run( + context -> + assertThat(context) + .hasFailed() + .getFailure() + .hasMessageContaining("not registered")); + } + + /** A destination whose policy is internally consistent, plus the dead letter it names. */ + @Configuration(proxyBeanMethods = false) + static class CoherentProfileConfiguration { + + @Bean + DestinationProfile orderEvents() { + return profile(OrderingScope.NONE, RetryPolicy.none()); + } + + @Bean + DestinationProfile orderEventsDeadLetter() { + return named("order-events-dlq", DeadLetterPolicy.disabled()); + } + } + + /** A destination naming a dead letter destination nobody declared. */ + @Configuration(proxyBeanMethods = false) + static class DanglingDeadLetterConfiguration { + + @Bean + DestinationProfile orderEvents() { + return profile(OrderingScope.NONE, RetryPolicy.none()); + } + } + + private static DestinationProfile named(String name, DeadLetterPolicy deadLetter) { + DestinationProfile base = profile(OrderingScope.NONE, RetryPolicy.none()); + return new DestinationProfile( + new DestinationName(name), + base.broker(), + base.kind(), + base.physical(), + base.schema(), + base.deliveryGuarantee(), + base.orderingScope(), + base.externalSideEffectGuarantee(), + base.producer(), + base.consumer(), + base.retry(), + deadLetter, + base.payload(), + base.tier(), + base.production(), + base.keyResolverConfigured(), + base.topologyAutoCreate()); + } + + /** An ordered destination configured with a reordering retry. */ + @Configuration(proxyBeanMethods = false) + static class ContradictoryProfileConfiguration { + + @Bean + DestinationProfile orderEvents() { + return profile( + OrderingScope.PARTITION, + new RetryPolicy( + dev.caskeleton.messaging.policy.RetryMode.RETRY_DESTINATION, + 3, + java.time.Duration.ofSeconds(1), + java.time.Duration.ofMinutes(1), + 2.0, + true, + dev.caskeleton.messaging.policy.OrderingImpact.PRESERVE, + Set.of(), + Set.of())); + } + } + + private static DestinationProfile profile(OrderingScope ordering, RetryPolicy retry) { + return new DestinationProfile( + new DestinationName("order-events"), + "kafka-primary", + DestinationKind.EVENT_STREAM, + PhysicalDestination.kafkaTopic("order.events.v1"), + new SchemaPolicy( + ContentType.JSON, + SchemaCompatibility.BACKWARD_TRANSITIVE, + Set.of(new MessageType("order.created"))), + DeliveryGuarantee.AT_LEAST_ONCE, + ordering, + ExternalSideEffectGuarantee.IDEMPOTENCY_REQUIRED, + ProducerPolicy.defaults(), + ConsumerPolicy.defaults("order-projection"), + retry, + DeadLetterPolicy.to(new DestinationName("order-events-dlq")), + PayloadPolicy.defaults(), + CapabilityTier.M1, + false, + false, + false); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpointTest.java b/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpointTest.java new file mode 100644 index 00000000..58862dda --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpointTest.java @@ -0,0 +1,126 @@ +package dev.caskeleton.messaging.autoconfigure; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.delivery.ExternalSideEffectGuarantee; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.DestinationKind; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.policy.CapabilityTier; +import dev.caskeleton.messaging.policy.ConsumerPolicy; +import dev.caskeleton.messaging.policy.DeadLetterPolicy; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.PayloadPolicy; +import dev.caskeleton.messaging.policy.PhysicalDestination; +import dev.caskeleton.messaging.policy.ProducerPolicy; +import dev.caskeleton.messaging.policy.RetryPolicy; +import dev.caskeleton.messaging.policy.SchemaPolicy; +import dev.caskeleton.messaging.schema.SchemaCompatibility; +import dev.caskeleton.messaging.transport.BackpressureController; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class MessagingEndpointTest { + + private static DestinationProfile profile() { + return new DestinationProfile( + new DestinationName("orders.v1"), + "kafka-primary", + DestinationKind.EVENT_STREAM, + PhysicalDestination.kafkaTopic("orders.v1"), + new SchemaPolicy( + ContentType.JSON, + SchemaCompatibility.BACKWARD, + Set.of(new MessageType("order.created"))), + DeliveryGuarantee.AT_LEAST_ONCE, + OrderingScope.KEY, + ExternalSideEffectGuarantee.IDEMPOTENCY_REQUIRED, + ProducerPolicy.defaults(), + ConsumerPolicy.defaults("orders"), + RetryPolicy.none(), + DeadLetterPolicy.to(new DestinationName("orders.v1.dlq")), + PayloadPolicy.defaults(), + CapabilityTier.M1, + true, + true, + false); + } + + private static MessagingEndpoint endpoint(BackpressureController backpressure) { + return new MessagingEndpoint( + new ValidatedDestinationRegistry(List.of(profile())), backpressure); + } + + @SuppressWarnings("unchecked") + private static List> destinations(Map report) { + return (List>) report.get("destinations"); + } + + @Test + void theReportNamesEveryRegisteredDestination() { + Map report = endpoint(new BackpressureController(100, 10)).messaging(); + + assertThat(destinations(report)) + .singleElement() + .satisfies(entry -> assertThat(entry.get("name")).isEqualTo("orders.v1")); + } + + @Test + void theReportCarriesTheDeclaredGuaranteesAnOperatorNeeds() { + Map entry = + destinations(endpoint(new BackpressureController(100, 10)).messaging()).get(0); + + assertThat(entry) + .containsEntry("deliveryGuarantee", "AT_LEAST_ONCE") + .containsEntry("orderingScope", "KEY") + .containsEntry("deadLetterEnabled", true) + .containsEntry("production", true); + } + + @Test + void theReportShowsCurrentInFlightWork() { + BackpressureController backpressure = new BackpressureController(100, 10); + backpressure.tryAcquire("orders.v1"); + backpressure.tryAcquire("orders.v1"); + + Map report = endpoint(backpressure).messaging(); + + assertThat(report).containsEntry("globalInFlight", 2); + assertThat(destinations(report).get(0)).containsEntry("inFlight", 2); + } + + @Test + void theReportCarriesNoPerMessageIdentity() { + Map entry = + destinations(endpoint(new BackpressureController(100, 10)).messaging()).get(0); + + assertThat(entry.keySet()) + .as("an actuator response that echoes message content is a re-identification surface") + .doesNotContain("messageId", "key", "partitionKey", "offset", "payload", "tenantId"); + } + + @Test + void theEndpointIsReadOnly() { + long writeOperations = + java.util.Arrays.stream(MessagingEndpoint.class.getDeclaredMethods()) + .filter( + method -> + method.isAnnotationPresent( + org.springframework.boot.actuate.endpoint.annotation.WriteOperation + .class) + || method.isAnnotationPresent( + org.springframework.boot.actuate.endpoint.annotation.DeleteOperation + .class)) + .count(); + + assertThat(writeOperations) + .as( + "the management port is usually unauthenticated; pause and purge belong to the admin plane") + .isZero(); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/ReactiveFacadeTest.java b/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/ReactiveFacadeTest.java new file mode 100644 index 00000000..21931ea8 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/ReactiveFacadeTest.java @@ -0,0 +1,134 @@ +package dev.caskeleton.messaging.autoconfigure; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +class ReactiveFacadeTest { + + private static final MessageDestination ORDERS = + new MessageDestination<>( + new DestinationName("order-events"), new MessageType("order.created"), String.class); + + private final AtomicInteger publishes = new AtomicInteger(); + + @Test + void nothingIsPublishedUntilTheMonoIsSubscribed() { + ReactiveMessagePublisher publisher = new DefaultReactiveMessagePublisher(counting()); + + Mono mono = publisher.publish(ORDERS, envelope(), PublishOptions.defaults()); + + assertThat(publishes.get()).as("assembly must not publish").isZero(); + + mono.block(); + + assertThat(publishes.get()).isEqualTo(1); + } + + @Test + void eachSubscriptionPublishesOnce() { + ReactiveMessagePublisher publisher = new DefaultReactiveMessagePublisher(counting()); + + Mono mono = publisher.publish(ORDERS, envelope(), PublishOptions.defaults()); + mono.block(); + mono.block(); + + assertThat(publishes.get()).isEqualTo(2); + } + + @Test + void theOutcomeIsEmittedAndTheMonoCompletes() { + ReactiveMessagePublisher publisher = new DefaultReactiveMessagePublisher(counting()); + + StepVerifier.create(publisher.publish(ORDERS, envelope(), PublishOptions.defaults())) + .assertNext( + result -> assertThat(result.completion()).isEqualTo(PublishCompletion.CONFIRMED)) + .verifyComplete(); + } + + @Test + void aFailedPublishPropagatesAsAnErrorSignal() { + MessagePublisher failing = + new MessagePublisher() { + @Override + public CompletionStage publish( + MessageDestination destination, + MessageEnvelope message, + PublishOptions options) { + return CompletableFuture.failedFuture(new IllegalStateException("broker down")); + } + }; + + StepVerifier.create( + new DefaultReactiveMessagePublisher(failing) + .publish(ORDERS, envelope(), PublishOptions.defaults())) + .expectErrorMessage("broker down") + .verify(); + } + + private MessagePublisher counting() { + return new MessagePublisher() { + @Override + public CompletionStage publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options) { + publishes.incrementAndGet(); + return CompletableFuture.completedFuture(confirmed()); + } + }; + } + + private static PublishResult confirmed() { + return new PublishResult( + PublishCompletion.CONFIRMED, + PublishEvidence.confirmed(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ofMillis(2), + Optional.empty()); + } + + private static MessageEnvelope envelope() { + return new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + Instant.parse("2026-08-10T09:15:00Z"), + Optional.of(Instant.parse("2026-08-10T09:15:00Z")), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + "payload"); + } +} diff --git a/src/messaging/messaging-spring-boot-starter/src/test/resources/application-invalid-ordering.yml b/src/messaging/messaging-spring-boot-starter/src/test/resources/application-invalid-ordering.yml new file mode 100644 index 00000000..c332eedf --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/test/resources/application-invalid-ordering.yml @@ -0,0 +1,11 @@ +# A configuration the platform must refuse. +# +# The per-destination in-flight limit exceeds the global one, which means the global limit is not +# a limit at all: a single destination could hold more work in flight than the whole process is +# allowed to. The controller rejects this at construction, so the context fails to start rather +# than running with a ceiling that does not bound anything. +backend: + messaging: + backpressure: + global-limit: 64 + per-destination-limit: 512 diff --git a/src/messaging/messaging-spring-boot-starter/src/test/resources/application-valid.yml b/src/messaging/messaging-spring-boot-starter/src/test/resources/application-valid.yml new file mode 100644 index 00000000..8101b6e9 --- /dev/null +++ b/src/messaging/messaging-spring-boot-starter/src/test/resources/application-valid.yml @@ -0,0 +1,24 @@ +# A configuration the platform accepts. +# +# Every experimental adapter is off and the bridge is off, which is the shipped default: an +# adapter whose contract suite is still being proven must never become load-bearing because a +# configuration defaulted it on. +backend: + messaging: + experimental: + kafka-share: false + pulsar: false + nats: false + bridge: + spring-cloud-stream: false + backpressure: + # The per-destination limit stays below the global one; a per-destination limit at or above + # the global limit would let one destination consume every slot in the process. + global-limit: 512 + per-destination-limit: 64 + shutdown: + # The design's default drain budget. Work still running at the deadline is abandoned + # unsettled so the broker redelivers it. + drain-deadline: 30s + admin: + enabled: false diff --git a/src/messaging/messaging-spring-cloud-stream-bridge/build.gradle b/src/messaging/messaging-spring-cloud-stream-bridge/build.gradle new file mode 100644 index 00000000..111aaa73 --- /dev/null +++ b/src/messaging/messaging-spring-cloud-stream-bridge/build.gradle @@ -0,0 +1,9 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-policy') + api project(':messaging:messaging-transport-spi') + + implementation 'org.springframework:spring-context' +} diff --git a/src/messaging/messaging-spring-cloud-stream-bridge/gradle.lockfile b/src/messaging/messaging-spring-cloud-stream-bridge/gradle.lockfile new file mode 100644 index 00000000..81b3df37 --- /dev/null +++ b/src/messaging/messaging-spring-cloud-stream-bridge/gradle.lockfile @@ -0,0 +1,91 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +commons-logging:commons-logging:1.3.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +io.micrometer:micrometer-commons:1.16.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.micrometer:micrometer-observation:1.16.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,compileClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.springframework:spring-aop:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-beans:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-context:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-core:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.springframework:spring-expression:7.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty= diff --git a/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/BindingCapabilityReport.java b/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/BindingCapabilityReport.java new file mode 100644 index 00000000..fb34a1ae --- /dev/null +++ b/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/BindingCapabilityReport.java @@ -0,0 +1,88 @@ +package dev.caskeleton.messaging.streambridge; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.util.List; +import java.util.Objects; + +/** + * States, per binding, which platform guarantees the bridge does not provide. + * + *

An explicit report rather than silence. The binder does provide retry and dead-lettering of + * its own, so a binding looks like it has them; what it does not have is the platform's versions — + * bounded attempts under the destination's retry policy, and a dead-letter publish confirmed before + * the source is settled. An operator comparing a bridged binding to a native one needs that + * difference written down, because nothing at runtime will show it. + * + * @param destination the logical destination + * @param bindingName the Stream binding name + * @param publishEvidenceAvailable whether the bridge can report broker publish evidence + * @param platformRetryApplied whether the destination's retry policy is enforced + * @param confirmedDeadLetterApplied whether dead-letter publishes are confirmed before settlement + * @param orderingEnforced whether the destination's ordering scope is enforced + */ +public record BindingCapabilityReport( + DestinationName destination, + String bindingName, + boolean publishEvidenceAvailable, + boolean platformRetryApplied, + boolean confirmedDeadLetterApplied, + boolean orderingEnforced) { + + public BindingCapabilityReport { + Objects.requireNonNull(destination, "destination must not be null"); + if (bindingName == null || bindingName.isBlank()) { + throw new IllegalArgumentException("bindingName must not be blank"); + } + } + + /** + * Returns the report for a binding served through the bridge. + * + *

Every platform guarantee is false, because the bridge hands the message to the binder and + * the binder is what decides these. + * + * @param destination the logical destination + * @param bindingName the Stream binding name + * @return the report + */ + public static BindingCapabilityReport bridged(DestinationName destination, String bindingName) { + return new BindingCapabilityReport(destination, bindingName, false, false, false, false); + } + + /** + * Returns the guarantees this binding does not carry, as sanitized text. + * + * @return the gaps, empty when nothing is missing + */ + public List gaps() { + List gaps = new java.util.ArrayList<>(); + if (!publishEvidenceAvailable) { + gaps.add( + "publish evidence: the binder reports a send, not a broker confirmation, so an " + + "ambiguous publish is indistinguishable from a confirmed one"); + } + if (!platformRetryApplied) { + gaps.add( + "retry: the binder's own retry runs instead of the destination's retry policy, with its " + + "own attempt budget and backoff"); + } + if (!confirmedDeadLetterApplied) { + gaps.add( + "dead letter: the binder settles the source without waiting for the dead-letter publish " + + "to confirm, so a dead-letter outage loses the message"); + } + if (!orderingEnforced) { + gaps.add("ordering: the binder's concurrency settings decide ordering, not the profile"); + } + return List.copyOf(gaps); + } + + /** + * Reports whether this binding carries every platform guarantee. + * + * @return true when nothing is missing + */ + public boolean isFullyGuaranteed() { + return gaps().isEmpty(); + } +} diff --git a/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/BindingProfileValidator.java b/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/BindingProfileValidator.java new file mode 100644 index 00000000..cb3d1590 --- /dev/null +++ b/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/BindingProfileValidator.java @@ -0,0 +1,97 @@ +package dev.caskeleton.messaging.streambridge; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * Checks a binding against its destination before the bridge is wired. + * + *

Runs on top of {@link StreamBridgePolicyGuard}: the guard decides whether a destination is + * eligible for the bridge at all, and this decides whether a particular binding is configured + * coherently for it. + * + *

The binder's extended properties are the sharp edge. Stream lets a binding override the + * serializer, the acknowledgement mode, and the concurrency, and each of those silently replaces + * something the destination profile already decided. Rather than merging the two — which produces a + * configuration nobody can read — a conflicting extended property is rejected and the operator is + * told which side to remove. + */ +public final class BindingProfileValidator { + + private static final Pattern BINDING_NAME = Pattern.compile("[a-zA-Z][a-zA-Z0-9-]{0,63}"); + + /** Extended properties that would override a platform decision. */ + private static final java.util.Set CONFLICTING_PROPERTIES = + java.util.Set.of( + "autoBindDlq", + "republishToDlq", + "maxAttempts", + "backOffInitialInterval", + "autoCommitOffset", + "ackMode", + "useNativeEncoding", + "contentType"); + + private final StreamBridgePolicyGuard guard = new StreamBridgePolicyGuard(); + + /** + * Validates one binding. + * + * @param profile the destination profile + * @param bindingName the Stream binding name + * @param extendedProperties the binder extended properties configured for the binding + * @param enabled whether the bridge is switched on + * @return the capability report for the accepted binding + */ + public BindingCapabilityReport validate( + DestinationProfile profile, + String bindingName, + java.util.Map extendedProperties, + boolean enabled) { + Objects.requireNonNull(profile, "profile must not be null"); + Objects.requireNonNull(extendedProperties, "extendedProperties must not be null"); + + guard.validate(profile, enabled); + + if (bindingName == null || !BINDING_NAME.matcher(bindingName).matches()) { + throw new MessagingConfigurationException( + "INVALID_BINDING_NAME", + "binding name must match %s: %s".formatted(BINDING_NAME.pattern(), bindingName)); + } + + for (String property : extendedProperties.keySet()) { + if (CONFLICTING_PROPERTIES.contains(property)) { + throw new MessagingConfigurationException( + "BINDING_OVERRIDES_PLATFORM_POLICY", + "binding %s sets %s, which overrides what the %s destination profile already decides; " + .formatted(bindingName, property, profile.name().value()) + + "remove it from the binding or move the destination to the native adapter"); + } + } + + if (profile.production()) { + throw new MessagingConfigurationException( + "BRIDGE_ON_PRODUCTION_DESTINATION", + "destination %s is marked production; the bridge does not carry the platform's publish " + .formatted(profile.name().value()) + + "evidence, retry, or confirmed dead lettering"); + } + + return BindingCapabilityReport.bridged(profile.name(), bindingName); + } + + /** + * Returns the report for a destination that is not bridged at all. + * + * @param destination the logical destination + * @param bindingName the Stream binding name + * @return a report stating every guarantee holds + */ + public static BindingCapabilityReport nativeAdapter( + DestinationName destination, String bindingName) { + return new BindingCapabilityReport(destination, bindingName, true, true, true, true); + } +} diff --git a/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/MessagingBindingBridge.java b/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/MessagingBindingBridge.java new file mode 100644 index 00000000..49be29e6 --- /dev/null +++ b/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/MessagingBindingBridge.java @@ -0,0 +1,34 @@ +package dev.caskeleton.messaging.streambridge; + +import dev.caskeleton.messaging.api.destination.DestinationName; + +/** + * Connects a logical destination to a Spring Cloud Stream binding. + * + *

The bridge is an interoperability seam, not a second messaging API. Its whole reason to exist + * is that a service already has Stream bindings and needs to reach the same destinations without a + * rewrite. + * + *

Binder semantics are never promoted to platform guarantees. Stream's binder has its own retry, + * its own dead-letter, and its own acknowledgement mode, and they look enough like the platform's + * to be mistaken for them — so a destination that actually relies on the platform's versions is + * refused by {@link StreamBridgePolicyGuard} rather than served with the binder's. + */ +public interface MessagingBindingBridge { + + /** + * Binds a Stream output to a logical destination. + * + * @param destination the logical destination + * @param bindingName the Stream binding name + */ + void bindPublisher(DestinationName destination, String bindingName); + + /** + * Binds a Stream input to a logical destination. + * + * @param destination the logical destination + * @param bindingName the Stream binding name + */ + void bindConsumer(DestinationName destination, String bindingName); +} diff --git a/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/SpringCloudStreamConsumerBridge.java b/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/SpringCloudStreamConsumerBridge.java new file mode 100644 index 00000000..e0b304f7 --- /dev/null +++ b/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/SpringCloudStreamConsumerBridge.java @@ -0,0 +1,87 @@ +package dev.caskeleton.messaging.streambridge; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Receives from a Spring Cloud Stream input binding and hands the message to a platform handler. + * + *

Settlement stays with the binder. The bridge cannot acknowledge, retry, or dead-letter a + * message itself, because Stream's binder already owns the acknowledgement for that binding and two + * things settling one message is worse than either doing it alone. + * + *

What the bridge does own is the translation and the honesty about it: a handler failure is + * rethrown so the binder's error channel sees it, rather than being converted into a platform + * {@code HandleResult} that nothing downstream would act on. + */ +public final class SpringCloudStreamConsumerBridge { + + private final Map handlers = new ConcurrentHashMap<>(); + private final Map destinations = new ConcurrentHashMap<>(); + + /** + * Registers a handler for a binding. + * + * @param destination the logical destination + * @param bindingName the Stream binding name + * @param handler the handler to invoke + */ + public void register(DestinationName destination, String bindingName, BridgedHandler handler) { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(handler, "handler must not be null"); + if (bindingName == null || bindingName.isBlank()) { + throw new IllegalArgumentException("bindingName must not be blank"); + } + handlers.put(bindingName, handler); + destinations.put(bindingName, destination); + } + + /** + * Dispatches one message received on a binding. + * + * @param bindingName the Stream binding name + * @param payload the encoded payload + * @param headers the received headers + * @throws MessagingConfigurationException when nothing is registered for the binding + */ + public void dispatch(String bindingName, byte[] payload, Map headers) { + Objects.requireNonNull(payload, "payload must not be null"); + Objects.requireNonNull(headers, "headers must not be null"); + + BridgedHandler handler = handlers.get(bindingName); + DestinationName destination = destinations.get(bindingName); + if (handler == null || destination == null) { + throw new MessagingConfigurationException( + "NO_BRIDGED_HANDLER", "no handler is registered for Stream binding " + bindingName); + } + // Not caught. The binder's error channel is what retries and dead-letters this binding, and + // swallowing the failure here would acknowledge a message nothing handled. + handler.handle(destination, payload, headers); + } + + /** + * Returns how many bindings have a handler. + * + * @return the registered binding count + */ + public int registeredBindings() { + return handlers.size(); + } + + /** A platform handler reached through a Stream binding. */ + @FunctionalInterface + public interface BridgedHandler { + + /** + * Handles one bridged message. + * + * @param destination the logical destination + * @param payload the encoded payload + * @param headers the received headers + */ + void handle(DestinationName destination, byte[] payload, Map headers); + } +} diff --git a/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/SpringCloudStreamPublisherBridge.java b/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/SpringCloudStreamPublisherBridge.java new file mode 100644 index 00000000..7e2b68d4 --- /dev/null +++ b/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/SpringCloudStreamPublisherBridge.java @@ -0,0 +1,149 @@ +package dev.caskeleton.messaging.streambridge; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import java.time.Duration; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Sends through a Spring Cloud Stream output binding and reports what that actually proves. + * + *

The result is deliberately {@code AMBIGUOUS} rather than {@code CONFIRMED}. A Stream {@code + * send} returns a boolean from the message channel — it says the binder accepted the message, not + * that a broker did. Reporting that as confirmed would put the platform's strongest word on the + * binder's weakest evidence, and a caller reading {@code CONFIRMED} would stop worrying about a + * message that may never have left the process. + * + *

A caller that needs real publish evidence has to use the native adapter. That is the honest + * trade the bridge exists to make visible. + */ +public final class SpringCloudStreamPublisherBridge implements MessagingBindingBridge { + + private final Map outputBindings = new ConcurrentHashMap<>(); + private final Map inputBindings = new ConcurrentHashMap<>(); + private final ChannelSend send; + + /** + * Creates a publisher bridge. + * + * @param send the Stream channel send operation + */ + public SpringCloudStreamPublisherBridge(ChannelSend send) { + this.send = Objects.requireNonNull(send, "send must not be null"); + } + + @Override + public void bindPublisher(DestinationName destination, String bindingName) { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(bindingName, "bindingName must not be null"); + outputBindings.put(destination, bindingName); + } + + @Override + public void bindConsumer(DestinationName destination, String bindingName) { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(bindingName, "bindingName must not be null"); + inputBindings.put(destination, bindingName); + } + + /** + * Publishes through the bound output binding. + * + * @param destination the logical destination + * @param payload the encoded payload + * @param headers the headers to carry + * @return the outcome, never stronger than the binder can prove + */ + public PublishResult publish( + DestinationName destination, byte[] payload, Map headers) { + Objects.requireNonNull(destination, "destination must not be null"); + Objects.requireNonNull(payload, "payload must not be null"); + Objects.requireNonNull(headers, "headers must not be null"); + + String binding = outputBindings.get(destination); + if (binding == null) { + throw new MessagingConfigurationException( + "NO_OUTPUT_BINDING", "no Stream output binding is bound to " + destination.value()); + } + + boolean accepted = send.send(binding, payload, headers); + return accepted ? acceptedByBinder() : rejectedByBinder(); + } + + /** + * Returns the binding bound to a destination for consuming. + * + * @param destination the logical destination + * @return the binding name, when one is bound + */ + public Optional consumerBinding(DestinationName destination) { + return Optional.ofNullable(inputBindings.get(destination)); + } + + /** + * Returns the binding bound to a destination for publishing. + * + * @param destination the logical destination + * @return the binding name, when one is bound + */ + public Optional publisherBinding(DestinationName destination) { + return Optional.ofNullable(outputBindings.get(destination)); + } + + private static PublishResult acceptedByBinder() { + return new PublishResult( + // The binder accepted it; no broker has confirmed anything. AMBIGUOUS is the only honest + // answer, and it is what makes the bridge's weaker guarantee visible to the caller. + PublishCompletion.AMBIGUOUS, + new PublishEvidence( + true, TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED, false, ConfirmationLevel.NONE), + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of( + dev.caskeleton.messaging.api.error.FailureDescriptor.of( + dev.caskeleton.messaging.api.error.FailureCategory.AMBIGUOUS, + "STREAM_BRIDGE_NO_BROKER_EVIDENCE", + "the Stream binder accepted the message; no broker confirmation is available"))); + } + + private static PublishResult rejectedByBinder() { + return new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of( + dev.caskeleton.messaging.api.error.FailureDescriptor.of( + dev.caskeleton.messaging.api.error.FailureCategory.TRANSIENT_INFRASTRUCTURE, + "STREAM_BRIDGE_SEND_REFUSED", + "the Stream channel refused the message"))); + } + + /** The Stream channel send, isolated so the bridge is testable without a binder. */ + @FunctionalInterface + public interface ChannelSend { + + /** + * Sends one message to a binding. + * + * @param bindingName the Stream binding name + * @param payload the encoded payload + * @param headers the headers to carry + * @return whether the channel accepted the message + */ + boolean send(String bindingName, byte[] payload, Map headers); + } +} diff --git a/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/StreamBridgePolicyGuard.java b/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/StreamBridgePolicyGuard.java new file mode 100644 index 00000000..e676f350 --- /dev/null +++ b/src/messaging/messaging-spring-cloud-stream-bridge/src/main/java/dev/caskeleton/messaging/streambridge/StreamBridgePolicyGuard.java @@ -0,0 +1,52 @@ +package dev.caskeleton.messaging.streambridge; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.util.Objects; + +/** + * Keeps the optional Spring Cloud Stream bridge inside platform policy. + * + *

The bridge exists for interoperability with existing Spring Cloud Stream bindings, and its + * risk is specific: Stream owns its own binder configuration, so a binding can quietly acquire its + * own serializer, its own error handling, and its own acknowledgement mode — none of which the + * destination profile knows about. + * + *

So the bridge is only permitted where the platform's guarantees are not the thing being relied + * on: a destination that declares an ordering scope, a retry policy, or a dead letter destination + * must go through the native adapter, where those are actually enforced. + */ +public final class StreamBridgePolicyGuard { + + /** + * Validates that a destination may be served through the bridge. + * + * @param profile the destination profile + * @param enabled whether the bridge is switched on + */ + public void validate(DestinationProfile profile, boolean enabled) { + Objects.requireNonNull(profile, "profile must not be null"); + + if (!enabled) { + throw new MessagingConfigurationException( + "STREAM_BRIDGE_DISABLED", + "the Spring Cloud Stream bridge is optional and disabled unless " + + "backend.messaging.bridge.spring-cloud-stream=true"); + } + if (profile.isOrdered()) { + throw new MessagingConfigurationException( + "STREAM_BRIDGE_ORDERING_UNSUPPORTED", + "an ordered destination must use the native adapter: " + profile.name().value()); + } + if (profile.retry().mode() != dev.caskeleton.messaging.policy.RetryMode.NONE) { + throw new MessagingConfigurationException( + "STREAM_BRIDGE_RETRY_UNSUPPORTED", + "the bridge does not apply the platform retry policy: " + profile.name().value()); + } + if (profile.deadLetter().enabled()) { + throw new MessagingConfigurationException( + "STREAM_BRIDGE_DLQ_UNSUPPORTED", + "the bridge does not apply confirmed dead lettering: " + profile.name().value()); + } + } +} diff --git a/src/messaging/messaging-spring-cloud-stream-bridge/src/test/java/dev/caskeleton/messaging/streambridge/BindingProfileValidatorTest.java b/src/messaging/messaging-spring-cloud-stream-bridge/src/test/java/dev/caskeleton/messaging/streambridge/BindingProfileValidatorTest.java new file mode 100644 index 00000000..1f5f4035 --- /dev/null +++ b/src/messaging/messaging-spring-cloud-stream-bridge/src/test/java/dev/caskeleton/messaging/streambridge/BindingProfileValidatorTest.java @@ -0,0 +1,164 @@ +package dev.caskeleton.messaging.streambridge; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.delivery.ExternalSideEffectGuarantee; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.DestinationKind; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.policy.CapabilityTier; +import dev.caskeleton.messaging.policy.ConsumerPolicy; +import dev.caskeleton.messaging.policy.DeadLetterPolicy; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.PayloadPolicy; +import dev.caskeleton.messaging.policy.PhysicalDestination; +import dev.caskeleton.messaging.policy.ProducerPolicy; +import dev.caskeleton.messaging.policy.RetryPolicy; +import dev.caskeleton.messaging.policy.SchemaPolicy; +import dev.caskeleton.messaging.schema.SchemaCompatibility; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class BindingProfileValidatorTest { + + private final BindingProfileValidator validator = new BindingProfileValidator(); + + private static DestinationProfile profile( + OrderingScope ordering, RetryPolicy retry, DeadLetterPolicy deadLetter, boolean production) { + return new DestinationProfile( + new DestinationName("orders.v1"), + "kafka-primary", + DestinationKind.EVENT_STREAM, + PhysicalDestination.kafkaTopic("orders.v1"), + new SchemaPolicy( + ContentType.JSON, + SchemaCompatibility.BACKWARD, + Set.of(new MessageType("order.created"))), + DeliveryGuarantee.AT_LEAST_ONCE, + ordering, + ExternalSideEffectGuarantee.IDEMPOTENCY_REQUIRED, + ProducerPolicy.defaults(), + ConsumerPolicy.defaults("orders"), + retry, + deadLetter, + PayloadPolicy.defaults(), + CapabilityTier.M1, + production, + false, + false); + } + + private static DestinationProfile bridgeable() { + return profile(OrderingScope.NONE, RetryPolicy.none(), DeadLetterPolicy.disabled(), false); + } + + @Test + void aBridgeableDestinationIsAccepted() { + BindingCapabilityReport report = validator.validate(bridgeable(), "orders-out", Map.of(), true); + + assertThat(report.bindingName()).isEqualTo("orders-out"); + } + + @Test + void aDisabledBridgeRefusesEverything() { + assertThatThrownBy(() -> validator.validate(bridgeable(), "orders-out", Map.of(), false)) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("disabled"); + } + + @Test + void anOrderedDestinationMustUseTheNativeAdapter() { + assertThatThrownBy( + () -> + validator.validate( + profile( + OrderingScope.KEY, RetryPolicy.none(), DeadLetterPolicy.disabled(), false), + "orders-out", + Map.of(), + true)) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("ordered"); + } + + @Test + void aDestinationWithADeadLetterMustUseTheNativeAdapter() { + assertThatThrownBy( + () -> + validator.validate( + profile( + OrderingScope.NONE, + RetryPolicy.none(), + DeadLetterPolicy.to(new DestinationName("orders.v1.dlq")), + false), + "orders-out", + Map.of(), + true)) + .as("the binder settles the source without confirming the dead-letter publish") + .isInstanceOf(MessagingConfigurationException.class); + } + + @Test + void aProductionDestinationIsRefused() { + assertThatThrownBy( + () -> + validator.validate( + profile( + OrderingScope.NONE, RetryPolicy.none(), DeadLetterPolicy.disabled(), true), + "orders-out", + Map.of(), + true)) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("production"); + } + + @Test + void aBinderPropertyThatOverridesPlatformPolicyIsRejectedRatherThanMerged() { + assertThatThrownBy( + () -> validator.validate(bridgeable(), "orders-out", Map.of("maxAttempts", "10"), true)) + .as("merging the two produces a configuration nobody can read") + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("maxAttempts"); + } + + @Test + void aHarmlessBinderPropertyIsAllowedThrough() { + assertThatCode( + () -> validator.validate(bridgeable(), "orders-out", Map.of("concurrency", "4"), true)) + .doesNotThrowAnyException(); + } + + @Test + void anInvalidBindingNameIsRejected() { + assertThatThrownBy(() -> validator.validate(bridgeable(), "9-bad name!", Map.of(), true)) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("binding name"); + } + + @Test + void aBridgedBindingReportsEveryGuaranteeItDoesNotCarry() { + BindingCapabilityReport report = validator.validate(bridgeable(), "orders-out", Map.of(), true); + + assertThat(report.isFullyGuaranteed()).isFalse(); + assertThat(report.gaps()).hasSize(4); + assertThat(String.join(" ", report.gaps())) + .contains("publish evidence") + .contains("retry") + .contains("dead letter") + .contains("ordering"); + } + + @Test + void aNativeBindingReportsNoGaps() { + assertThat( + BindingProfileValidator.nativeAdapter(new DestinationName("orders.v1"), "orders-out") + .isFullyGuaranteed()) + .isTrue(); + } +} diff --git a/src/messaging/messaging-spring-cloud-stream-bridge/src/test/java/dev/caskeleton/messaging/streambridge/BridgePublishEvidenceTest.java b/src/messaging/messaging-spring-cloud-stream-bridge/src/test/java/dev/caskeleton/messaging/streambridge/BridgePublishEvidenceTest.java new file mode 100644 index 00000000..411f04f1 --- /dev/null +++ b/src/messaging/messaging-spring-cloud-stream-bridge/src/test/java/dev/caskeleton/messaging/streambridge/BridgePublishEvidenceTest.java @@ -0,0 +1,131 @@ +package dev.caskeleton.messaging.streambridge; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class BridgePublishEvidenceTest { + + private static final DestinationName ORDERS = new DestinationName("orders.v1"); + private static final byte[] PAYLOAD = "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8); + + private static SpringCloudStreamPublisherBridge bridge(boolean channelAccepts) { + SpringCloudStreamPublisherBridge bridge = + new SpringCloudStreamPublisherBridge((binding, payload, headers) -> channelAccepts); + bridge.bindPublisher(ORDERS, "orders-out"); + return bridge; + } + + @Test + void anAcceptedSendIsAmbiguousBecauseNoBrokerConfirmedIt() { + PublishResult result = bridge(true).publish(ORDERS, PAYLOAD, Map.of()); + + assertThat(result.completion()) + .as("a channel send says the binder took it, not that a broker did") + .isEqualTo(PublishCompletion.AMBIGUOUS); + assertThat(result.evidence().brokerAccepted()).isFalse(); + assertThat(result.evidence().confirmationLevel()).isEqualTo(ConfirmationLevel.NONE); + } + + @Test + void anAcceptedSendReportsThatTransmissionIsUnknown() { + assertThat(bridge(true).publish(ORDERS, PAYLOAD, Map.of()).evidence().transmission()) + .isEqualTo(TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED); + } + + @Test + void theFailureDescriptorNamesTheMissingEvidence() { + assertThat(bridge(true).publish(ORDERS, PAYLOAD, Map.of()).failure()) + .hasValueSatisfying( + failure -> assertThat(failure.code()).isEqualTo("STREAM_BRIDGE_NO_BROKER_EVIDENCE")); + } + + @Test + void aRefusedChannelSendIsRejectedNotAmbiguous() { + PublishResult result = bridge(false).publish(ORDERS, PAYLOAD, Map.of()); + + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.evidence().transmission()).isEqualTo(TransmissionEvidence.NOT_TRANSMITTED); + } + + @Test + void publishingToAnUnboundDestinationFailsLoudly() { + SpringCloudStreamPublisherBridge bridge = + new SpringCloudStreamPublisherBridge((binding, payload, headers) -> true); + + assertThatThrownBy(() -> bridge.publish(ORDERS, PAYLOAD, Map.of())) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("orders.v1"); + } + + @Test + void thePayloadReachesTheBoundBinding() { + List bindings = new ArrayList<>(); + SpringCloudStreamPublisherBridge bridge = + new SpringCloudStreamPublisherBridge( + (binding, payload, headers) -> { + bindings.add(binding); + return true; + }); + bridge.bindPublisher(ORDERS, "orders-out"); + + bridge.publish(ORDERS, PAYLOAD, Map.of()); + + assertThat(bindings).containsExactly("orders-out"); + } + + @Test + void bindingsAreQueryableInBothDirections() { + SpringCloudStreamPublisherBridge bridge = bridge(true); + bridge.bindConsumer(ORDERS, "orders-in"); + + assertThat(bridge.publisherBinding(ORDERS)).hasValue("orders-out"); + assertThat(bridge.consumerBinding(ORDERS)).hasValue("orders-in"); + } + + @Test + void aHandlerFailureIsRethrownSoTheBindersErrorChannelSeesIt() { + SpringCloudStreamConsumerBridge consumer = new SpringCloudStreamConsumerBridge(); + consumer.register( + ORDERS, + "orders-in", + (destination, payload, headers) -> { + throw new IllegalStateException("handler failed"); + }); + + assertThatThrownBy(() -> consumer.dispatch("orders-in", PAYLOAD, Map.of())) + .as("swallowing it would acknowledge a message nothing handled") + .isInstanceOf(IllegalStateException.class); + } + + @Test + void dispatchingToAnUnregisteredBindingFailsLoudly() { + assertThatThrownBy( + () -> new SpringCloudStreamConsumerBridge().dispatch("orders-in", PAYLOAD, Map.of())) + .isInstanceOf(MessagingConfigurationException.class); + } + + @Test + void aRegisteredHandlerReceivesTheDestinationItWasBoundTo() { + List seen = new ArrayList<>(); + SpringCloudStreamConsumerBridge consumer = new SpringCloudStreamConsumerBridge(); + consumer.register( + ORDERS, "orders-in", (destination, payload, headers) -> seen.add(destination)); + + consumer.dispatch("orders-in", PAYLOAD, Map.of()); + + assertThat(seen).containsExactly(ORDERS); + assertThat(consumer.registeredBindings()).isEqualTo(1); + } +} diff --git a/src/messaging/messaging-testkit/build.gradle b/src/messaging/messaging-testkit/build.gradle new file mode 100644 index 00000000..d77b5db6 --- /dev/null +++ b/src/messaging/messaging-testkit/build.gradle @@ -0,0 +1,11 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-schema-api') + api project(':messaging:messaging-policy') + api project(':messaging:messaging-transport-spi') + + api 'org.junit.jupiter:junit-jupiter' + api 'org.assertj:assertj-core' +} diff --git a/src/messaging/messaging-testkit/gradle.lockfile b/src/messaging/messaging-testkit/gradle.lockfile new file mode 100644 index 00000000..d1ba4a93 --- /dev/null +++ b/src/messaging/messaging-testkit/gradle.lockfile @@ -0,0 +1,87 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,jmhAnnotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,jmhAnnotationProcessor,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,jmhAnnotationProcessor,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +net.sf.jopt-simple:jopt-simple:5.0.4=jmhAnnotationProcessor,jmhCompileClasspath,jmhRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-math3:3.6.1=jmhAnnotationProcessor,jmhCompileClasspath,jmhRuntimeClasspath +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=compileClasspath,jmhCompileClasspath,testCompileClasspath +org.assertj:assertj-core:3.27.6=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,compileClasspath,jmhAnnotationProcessor,jmhCompileClasspath,testAnnotationProcessor,testCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=jmhRuntimeClasspath,runtimeClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=jmhRuntimeClasspath,runtimeClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=jmhRuntimeClasspath,testRuntimeClasspath +org.junit:junit-bom:6.0.1=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.openjdk.jmh:jmh-core:1.37=jmhAnnotationProcessor,jmhCompileClasspath,jmhRuntimeClasspath +org.openjdk.jmh:jmh-generator-annprocess:1.37=jmhAnnotationProcessor +org.opentest4j:opentest4j:1.3.0=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,jmhAnnotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty= diff --git a/src/messaging/messaging-testkit/src/jmh/java/dev/caskeleton/messaging/testkit/EnvelopeCodecBenchmark.java b/src/messaging/messaging-testkit/src/jmh/java/dev/caskeleton/messaging/testkit/EnvelopeCodecBenchmark.java new file mode 100644 index 00000000..4a7135d8 --- /dev/null +++ b/src/messaging/messaging-testkit/src/jmh/java/dev/caskeleton/messaging/testkit/EnvelopeCodecBenchmark.java @@ -0,0 +1,126 @@ +package dev.caskeleton.messaging.testkit; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.UuidV7; +import dev.caskeleton.messaging.api.header.HeaderName; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measures the per-message cost the platform adds before any broker is involved. + * + *

This is the number the platform is accountable for. Broker latency dominates any real publish + * and varies with the network, so measuring it would tell you about the test environment; envelope + * construction, header validation, and id generation happen on the calling thread on every message + * and are entirely the platform's. + * + *

Header validation is benchmarked separately from envelope construction because they scale + * differently: construction is constant, while validation is linear in the header count and is the + * part that a caller can make expensive by accident. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +@Fork(1) +@State(Scope.Benchmark) +public class EnvelopeCodecBenchmark { + + private Map fewHeaders; + private Map manyHeaders; + private Instant now; + + /** Builds the header maps once, so the benchmark measures validation and not map construction. */ + @Setup + public void setUp() { + now = Instant.parse("2026-08-10T09:15:00Z"); + fewHeaders = headers(3); + manyHeaders = headers(32); + } + + private static Map headers(int count) { + Map headers = new LinkedHashMap<>(); + for (int index = 0; index < count; index++) { + headers.put(new HeaderName("x-app-" + index), new HeaderValue("value-" + index)); + } + return headers; + } + + /** + * Measures monotonic id generation, which every published message pays for. + * + * @param blackhole consumes the result + */ + @Benchmark + public void generateMessageId(Blackhole blackhole) { + blackhole.consume(UuidV7.next()); + } + + /** + * Measures header validation with a typical header count. + * + * @param blackhole consumes the result + */ + @Benchmark + public void validateFewHeaders(Blackhole blackhole) { + blackhole.consume(MessageHeaders.application(fewHeaders)); + } + + /** + * Measures header validation near the count limit, where the cost is linear. + * + * @param blackhole consumes the result + */ + @Benchmark + public void validateManyHeaders(Blackhole blackhole) { + blackhole.consume(MessageHeaders.application(manyHeaders)); + } + + /** + * Measures building one complete envelope. + * + * @param blackhole consumes the result + */ + @Benchmark + public void buildEnvelope(Blackhole blackhole) { + blackhole.consume( + new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + now, + Optional.of(now), + new ProducerId("order-api"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + MessageHeaders.application(fewHeaders), + "{\"orderId\":\"o-1\"}")); + } +} diff --git a/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/BrokerFailureMatrix.java b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/BrokerFailureMatrix.java new file mode 100644 index 00000000..34bfd741 --- /dev/null +++ b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/BrokerFailureMatrix.java @@ -0,0 +1,131 @@ +package dev.caskeleton.messaging.testkit; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Which failure scenarios each adapter has actually been run against. + * + *

Records coverage, not capability. An adapter that has never been run under {@code + * connection-cut-after-write} may well handle it correctly — the point is that nobody knows, and a + * support matrix that does not distinguish "verified" from "probably fine" is how an unproven + * adapter ends up in production. + * + *

A Stable adapter must cover every scenario. That rule is enforced by a test rather than + * documented, because a promotion to Stable is exactly the moment the gap would otherwise be + * overlooked. + */ +public final class BrokerFailureMatrix { + + private final Map> coverage = new LinkedHashMap<>(); + + /** How a scenario is covered for one adapter. */ + public enum Coverage { + /** Run against a deterministic harness. */ + DETERMINISTIC, + /** Run against a live broker in a container. */ + LIVE_BROKER, + /** Not run. */ + NOT_COVERED + } + + /** + * Records how one adapter covers one scenario. + * + * @param adapter the adapter module name + * @param scenario the scenario + * @param level how it is covered + * @return this matrix + */ + public BrokerFailureMatrix record(String adapter, NetworkFaultScenario scenario, Coverage level) { + Objects.requireNonNull(scenario, "scenario must not be null"); + Objects.requireNonNull(level, "level must not be null"); + if (adapter == null || adapter.isBlank()) { + throw new IllegalArgumentException("adapter must not be blank"); + } + coverage.computeIfAbsent(adapter, key -> new LinkedHashMap<>()).put(scenario.name(), level); + return this; + } + + /** + * Returns how an adapter covers a scenario. + * + * @param adapter the adapter module name + * @param scenario the scenario + * @return the coverage level, {@link Coverage#NOT_COVERED} when nothing was recorded + */ + public Coverage coverageOf(String adapter, NetworkFaultScenario scenario) { + return Optional.ofNullable(coverage.get(adapter)) + .map(scenarios -> scenarios.get(scenario.name())) + .orElse(Coverage.NOT_COVERED); + } + + /** + * Returns the scenarios an adapter has never been run against. + * + * @param adapter the adapter module name + * @return the uncovered scenarios + */ + public List gapsFor(String adapter) { + return NetworkFaultScenario.all().stream() + .filter(scenario -> coverageOf(adapter, scenario) == Coverage.NOT_COVERED) + .toList(); + } + + /** + * Reports whether an adapter covers every scenario. + * + * @param adapter the adapter module name + * @return true when nothing is uncovered + */ + public boolean isComplete(String adapter) { + return gapsFor(adapter).isEmpty(); + } + + /** + * Returns the adapters this matrix has any record for. + * + * @return the recorded adapter names + */ + public List adapters() { + return List.copyOf(coverage.keySet()); + } + + /** + * Returns the matrix the platform ships with. + * + *

Kafka and RabbitMQ are Stable and cover everything. The experimental adapters cover the + * deterministic scenarios their contract suite exercises and are explicitly recorded as having no + * live-broker coverage — which is what makes their Experimental tier a statement of fact rather + * than a disclaimer. + * + * @return the shipped coverage matrix + */ + public static BrokerFailureMatrix shipped() { + BrokerFailureMatrix matrix = new BrokerFailureMatrix(); + for (NetworkFaultScenario scenario : NetworkFaultScenario.all()) { + matrix.record("messaging-kafka", scenario, Coverage.LIVE_BROKER); + matrix.record("messaging-rabbit", scenario, Coverage.LIVE_BROKER); + } + matrix.record( + "messaging-pulsar-experimental", + NetworkFaultScenario.CONNECTION_CUT_AFTER_WRITE, + Coverage.DETERMINISTIC); + matrix.record( + "messaging-pulsar-experimental", + NetworkFaultScenario.CONFIRM_TIMEOUT, + Coverage.DETERMINISTIC); + matrix.record( + "messaging-nats-experimental", + NetworkFaultScenario.CONNECTION_CUT_AFTER_WRITE, + Coverage.DETERMINISTIC); + matrix.record( + "messaging-nats-experimental", + NetworkFaultScenario.CONFIRM_TIMEOUT, + Coverage.DETERMINISTIC); + return matrix; + } +} diff --git a/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/CompatibilityMatrix.java b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/CompatibilityMatrix.java new file mode 100644 index 00000000..a3f5d6fc --- /dev/null +++ b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/CompatibilityMatrix.java @@ -0,0 +1,115 @@ +package dev.caskeleton.messaging.testkit; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * The certified broker versions and support tier for every adapter. + * + *

Held as code with a test over it rather than as a document, because a support matrix that + * lives only in prose drifts the first time an adapter is added. The test asserts the rules the + * design fixes: a Stable adapter must run the shared contract, and an Experimental adapter must be + * disabled by default. + */ +public final class CompatibilityMatrix { + + private CompatibilityMatrix() {} + + /** The support tier an adapter has earned. */ + public enum Tier { + /** Passes the shared adapter contract unchanged. */ + STABLE, + + /** Contract coverage is still being proven; disabled by default. */ + EXPERIMENTAL, + + /** Adapter SPI only; outside the supported set. */ + EXTENSION + } + + /** + * One certified adapter. + * + * @param adapter the module name + * @param brokerVersions the broker versions the adapter is certified against + * @param tier the support tier + * @param runsSharedContract whether the module runs {@link MessagingAdapterContract} + * @param enabledByDefault whether the adapter is active without an explicit opt-in + * @param hasLiveBrokerCertification whether a container-backed suite exercises a real broker + */ + public record Entry( + String adapter, + List brokerVersions, + Tier tier, + boolean runsSharedContract, + boolean enabledByDefault, + boolean hasLiveBrokerCertification) { + + public Entry { + Objects.requireNonNull(tier, "tier must not be null"); + Objects.requireNonNull(brokerVersions, "brokerVersions must not be null"); + if (adapter == null || adapter.isBlank()) { + throw new IllegalArgumentException("adapter must not be blank"); + } + if (brokerVersions.isEmpty()) { + throw new IllegalArgumentException("an adapter must certify at least one broker version"); + } + brokerVersions = List.copyOf(brokerVersions); + } + } + + private static final Map ENTRIES = + Map.of( + "messaging-kafka", + new Entry("messaging-kafka", List.of("4.2", "4.3"), Tier.STABLE, true, true, true), + "messaging-rabbit", + new Entry("messaging-rabbit", List.of("4.3"), Tier.STABLE, true, true, true), + "messaging-kafka-share-experimental", + new Entry( + "messaging-kafka-share-experimental", + List.of("4.2", "4.3"), + Tier.EXPERIMENTAL, + false, + false, + false), + "messaging-pulsar-experimental", + new Entry( + "messaging-pulsar-experimental", + List.of("4.0", "4.2"), + Tier.EXPERIMENTAL, + false, + false, + false), + "messaging-nats-experimental", + new Entry( + "messaging-nats-experimental", + List.of("2.14"), + Tier.EXPERIMENTAL, + false, + false, + false)); + + /** + * Returns every certified adapter. + * + * @return the entries, keyed by module name + */ + public static Map entries() { + return ENTRIES; + } + + /** + * Returns one adapter's entry. + * + * @param adapter the module name + * @return the entry + */ + public static Entry of(String adapter) { + Entry entry = ENTRIES.get(adapter); + if (entry == null) { + throw new IllegalArgumentException("adapter is not in the compatibility matrix: " + adapter); + } + return entry; + } +} diff --git a/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/ContractAssertions.java b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/ContractAssertions.java new file mode 100644 index 00000000..2bed0f11 --- /dev/null +++ b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/ContractAssertions.java @@ -0,0 +1,51 @@ +package dev.caskeleton.messaging.testkit; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; + +/** Assertions every adapter's contract run shares. */ +public final class ContractAssertions { + + private ContractAssertions() {} + + /** + * Asserts a publish confirmed with real broker evidence. + * + * @param result the publish outcome + */ + public static void assertConfirmed(PublishResult result) { + assertThat(result.completion()).isEqualTo(PublishCompletion.CONFIRMED); + assertThat(result.evidence().brokerAccepted()).isTrue(); + assertThat(result.evidence().confirmationLevel()).isNotEqualTo(ConfirmationLevel.NONE); + assertThat(result.evidence().transmission()).isEqualTo(TransmissionEvidence.TRANSMITTED); + } + + /** + * Asserts a publish reported ambiguity without claiming any confirmation. + * + * @param result the publish outcome + */ + public static void assertAmbiguous(PublishResult result) { + assertThat(result.completion()).isEqualTo(PublishCompletion.AMBIGUOUS); + assertThat(result.evidence().confirmationLevel()).isEqualTo(ConfirmationLevel.NONE); + assertThat(result.evidence().brokerAccepted()).isFalse(); + assertThat(result.evidence().transmission()) + .isEqualTo(TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED); + assertThat(result.mayHaveBeenStored()).isTrue(); + } + + /** + * Asserts a publish was refused before anything left the process. + * + * @param result the publish outcome + */ + public static void assertRejectedLocally(PublishResult result) { + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.evidence().transmission()).isEqualTo(TransmissionEvidence.NOT_TRANSMITTED); + assertThat(result.failure()).isPresent(); + } +} diff --git a/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/ContractMessage.java b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/ContractMessage.java new file mode 100644 index 00000000..7c9239fe --- /dev/null +++ b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/ContractMessage.java @@ -0,0 +1,87 @@ +package dev.caskeleton.messaging.testkit; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * The standard messages every adapter contract run uses. + * + *

Fixed rather than random so that a failure in one adapter can be compared byte for byte + * against another. The oversized variant exists to prove the payload limit is enforced locally: an + * adapter that lets it reach the broker turns a deterministic rejection into a broker-specific + * error. + * + * @param envelope the encoded envelope to publish + */ +public record ContractMessage(MessageEnvelope envelope) { + + private static final Instant FIXED_TIME = Instant.parse("2026-08-10T09:15:00Z"); + + public ContractMessage { + Objects.requireNonNull(envelope, "envelope must not be null"); + } + + /** + * Returns the standard small event. + * + * @return a contract message under every limit + */ + public static ContractMessage orderCreated() { + return of("{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8)); + } + + /** + * Returns a message one byte above the one mebibyte portability limit. + * + * @return an oversized contract message + */ + public static ContractMessage oversized() { + return of(new byte[1_048_577]); + } + + /** + * Builds a contract message around explicit payload bytes. + * + * @param payload the encoded payload + * @return a contract message + */ + public static ContractMessage of(byte[] payload) { + return new ContractMessage( + new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + FIXED_TIME, + Optional.of(FIXED_TIME), + new ProducerId("contract-suite"), + Optional.empty(), + Optional.empty(), + ContentType.JSON, + Optional.of("acct-1"), + Optional.of("acct-1"), + Optional.empty(), + TraceContext.none(), + MessageHeaders.empty(), + new EncodedMessage(payload, ContentType.JSON, Optional.empty()))); + } + + /** + * Returns the logical identity of this message. + * + * @return the message id + */ + public MessageId messageId() { + return envelope.messageId(); + } +} diff --git a/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/DockerAvailability.java b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/DockerAvailability.java new file mode 100644 index 00000000..542eb36a --- /dev/null +++ b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/DockerAvailability.java @@ -0,0 +1,35 @@ +package dev.caskeleton.messaging.testkit; + +/** + * Reports whether a container runtime is reachable. + * + *

Live-broker suites are guarded on this rather than assumed. A developer machine or a build + * agent without Docker should skip them with a stated reason, not fail with a connection error that + * looks like a product defect — and the release gate checks separately that the suites actually ran + * where they were supposed to. + */ +public final class DockerAvailability { + + private static final boolean AVAILABLE = probe(); + + private DockerAvailability() {} + + /** + * Reports whether a container runtime is available. + * + * @return true when containers can be started + */ + public static boolean isAvailable() { + return AVAILABLE; + } + + private static boolean probe() { + try { + Class factory = Class.forName("org.testcontainers.DockerClientFactory"); + Object instance = factory.getMethod("instance").invoke(null); + return (boolean) factory.getMethod("isDockerAvailable").invoke(instance); + } catch (ReflectiveOperationException | RuntimeException exception) { + return false; + } + } +} diff --git a/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/FaultController.java b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/FaultController.java new file mode 100644 index 00000000..3cc62f01 --- /dev/null +++ b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/FaultController.java @@ -0,0 +1,26 @@ +package dev.caskeleton.messaging.testkit; + +/** + * Injects the faults an adapter must survive. + * + *

Each fault targets an acknowledgement, not the operation itself. That is the point: + * losing a confirm is what makes an outcome ambiguous, and an adapter is only trustworthy if it + * reports the ambiguity rather than guessing which way it went. + */ +public interface FaultController { + + /** Makes the next publish complete without a broker confirmation. */ + void dropPublishConfirmation(); + + /** Makes the next settlement complete without a broker confirmation. */ + void dropSettlementConfirmation(); + + /** Makes publishes to the dead letter destination fail. */ + void failDeadLetterPublish(); + + /** Makes the next publish be refused outright by the broker. */ + void rejectPublish(); + + /** Clears every injected fault. */ + void reset(); +} diff --git a/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/HandleOutcome.java b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/HandleOutcome.java new file mode 100644 index 00000000..460030fb --- /dev/null +++ b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/HandleOutcome.java @@ -0,0 +1,14 @@ +package dev.caskeleton.messaging.testkit; + +/** What the contract suite's handler decides for a drained delivery. */ +public enum HandleOutcome { + + /** The handler succeeded and the platform may settle. */ + SUCCESS, + + /** The handler failed transiently and the message should be retried. */ + RETRY, + + /** The handler failed permanently and the message should be dead lettered. */ + DEAD_LETTER +} diff --git a/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/MessagingAdapterContract.java b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/MessagingAdapterContract.java new file mode 100644 index 00000000..d96d9d5d --- /dev/null +++ b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/MessagingAdapterContract.java @@ -0,0 +1,139 @@ +package dev.caskeleton.messaging.testkit; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.publish.PublishResult; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * The behaviour every adapter must exhibit, regardless of broker. + * + *

This suite is the platform's actual definition of "supported". A broker is Stable when it + * passes these unchanged — not when it has an adapter that compiles. Each test pins one of the + * guarantees the design refuses to weaken: ambiguity is reported rather than guessed, settlement + * follows handler success, logical identity survives retry and dead lettering, and a source is + * never acknowledged on the strength of a dead letter publish that failed. + */ +public abstract class MessagingAdapterContract { + + /** + * Returns a fresh harness for one test. + * + * @return the adapter harness under test + */ + protected abstract MessagingAdapterHarness harness(); + + @Test + void publishesAndConfirms() { + try (MessagingAdapterHarness harness = harness()) { + PublishResult result = join(harness.publish(ContractMessage.orderCreated())); + + ContractAssertions.assertConfirmed(result); + } + } + + @Test + void returnsAmbiguousWhenConfirmIsLost() { + try (MessagingAdapterHarness harness = harness()) { + harness.faults().dropPublishConfirmation(); + + PublishResult result = join(harness.publish(ContractMessage.orderCreated())); + + ContractAssertions.assertAmbiguous(result); + } + } + + @Test + void redeliversWhenSettlementIsLost() { + try (MessagingAdapterHarness harness = harness()) { + ContractMessage message = ContractMessage.orderCreated(); + join(harness.publish(message)); + harness.faults().dropSettlementConfirmation(); + + List first = harness.drain(HandleOutcome.SUCCESS); + assertThat(first) + .singleElement() + .satisfies( + delivery -> { + assertThat(delivery.messageId()).isEqualTo(message.messageId()); + assertThat(delivery.settled()).isFalse(); + }); + + List second = harness.drain(HandleOutcome.SUCCESS); + assertThat(second) + .singleElement() + .satisfies( + delivery -> { + assertThat(delivery.messageId()).isEqualTo(message.messageId()); + assertThat(delivery.attempt()).isEqualTo(2); + assertThat(delivery.redelivered()).isTrue(); + }); + } + } + + @Test + void preservesMessageIdAcrossRetryAndDlq() { + try (MessagingAdapterHarness harness = harness()) { + ContractMessage message = ContractMessage.orderCreated(); + join(harness.publish(message)); + MessageId original = message.messageId(); + + List retried = harness.drain(HandleOutcome.RETRY); + assertThat(retried) + .singleElement() + .extracting(ObservedDelivery::messageId) + .isEqualTo(original); + + List redelivered = harness.drain(HandleOutcome.DEAD_LETTER); + assertThat(redelivered) + .singleElement() + .extracting(ObservedDelivery::messageId) + .isEqualTo(original); + + assertThat(harness.deadLettered()).containsExactly(original); + } + } + + @Test + void keepsSourceUnsettledWhenDlqPublishFails() { + try (MessagingAdapterHarness harness = harness()) { + ContractMessage message = ContractMessage.orderCreated(); + join(harness.publish(message)); + harness.faults().failDeadLetterPublish(); + + List observed = harness.drain(HandleOutcome.DEAD_LETTER); + + assertThat(observed).singleElement().extracting(ObservedDelivery::settled).isEqualTo(false); + assertThat(harness.deadLettered()).isEmpty(); + assertThat(harness.unsettled()).contains(message.messageId()); + } + } + + @Test + void rejectsOversizedPayloadBeforeTransport() { + try (MessagingAdapterHarness harness = harness()) { + PublishResult result = join(harness.publish(ContractMessage.oversized())); + + ContractAssertions.assertRejectedLocally(result); + } + } + + @Test + void stopsAcceptingNewWorkDuringShutdown() { + try (MessagingAdapterHarness harness = harness()) { + assertThat(harness.isAcceptingWork()).isTrue(); + + harness.beginShutdown(); + + assertThat(harness.isAcceptingWork()).isFalse(); + PublishResult result = join(harness.publish(ContractMessage.orderCreated())); + ContractAssertions.assertRejectedLocally(result); + } + } + + private static PublishResult join(java.util.concurrent.CompletionStage stage) { + return stage.toCompletableFuture().join(); + } +} diff --git a/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/MessagingAdapterHarness.java b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/MessagingAdapterHarness.java new file mode 100644 index 00000000..4df6fa84 --- /dev/null +++ b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/MessagingAdapterHarness.java @@ -0,0 +1,73 @@ +package dev.caskeleton.messaging.testkit; + +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.publish.PublishResult; +import java.util.List; +import java.util.concurrent.CompletionStage; + +/** + * The seam each broker adapter implements to run the common contract. + * + *

The harness exposes outcomes, not internals: what the platform decided, what reached the dead + * letter destination, and what is still unsettled. Kafka and RabbitMQ answer those three questions + * with completely different machinery, and the contract only passes if the answers match. + */ +public interface MessagingAdapterHarness extends AutoCloseable { + + /** + * Returns the broker under test. + * + * @return the broker name + */ + String brokerName(); + + /** + * Returns the fault injector for this harness. + * + * @return the fault controller + */ + FaultController faults(); + + /** + * Publishes one contract message. + * + * @param message the message to publish + * @return a stage completing with the outcome and evidence + */ + CompletionStage publish(ContractMessage message); + + /** + * Delivers everything currently pending to a handler that returns the given outcome. + * + * @param outcome what the handler decides for each delivery + * @return what the harness observed, in delivery order + */ + List drain(HandleOutcome outcome); + + /** + * Returns the identities that reached the dead letter destination. + * + * @return dead lettered message identities + */ + List deadLettered(); + + /** + * Returns the identities still unsettled at the source. + * + * @return unsettled message identities + */ + List unsettled(); + + /** Begins a graceful drain. */ + void beginShutdown(); + + /** + * Reports whether the harness still accepts new work. + * + * @return true until shutdown begins + */ + boolean isAcceptingWork(); + + @Override + void close(); +} diff --git a/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/NetworkFaultScenario.java b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/NetworkFaultScenario.java new file mode 100644 index 00000000..665afb8e --- /dev/null +++ b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/NetworkFaultScenario.java @@ -0,0 +1,127 @@ +package dev.caskeleton.messaging.testkit; + +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; + +/** + * A named network fault, and the outcome an adapter must report while it is active. + * + *

The expected outcome is part of the scenario rather than left to each test, because the whole + * value of a fault suite is that every adapter answers the same way. A scenario that let each + * adapter declare its own expectation would pass while the adapters disagreed — which is exactly + * the situation the shared contract exists to catch. + * + *

The distinction across these scenarios is what evidence survives. A connection refused before + * any bytes leave proves nothing was published; a connection cut after the frame was written proves + * nothing at all, and the honest answer is ambiguous. + * + * @param name the scenario name, used in reports + * @param phase when the fault is injected relative to the publish + * @param expectation what the adapter must report + * @param duration how long the fault holds, when it is time-bounded + * @param rationale why this outcome is the correct one + */ +public record NetworkFaultScenario( + String name, + Phase phase, + Expectation expectation, + Optional duration, + String rationale) { + + /** When a fault takes effect relative to the operation under test. */ + public enum Phase { + /** Before any bytes leave the client. */ + BEFORE_TRANSMISSION, + /** After the frame was written but before the acknowledgement. */ + AFTER_TRANSMISSION, + /** While the consumer holds an unsettled delivery. */ + DURING_SETTLEMENT + } + + /** What the adapter must report under a fault. */ + public enum Expectation { + /** A definite rejection, with no ambiguity. */ + REJECTED, + /** An unknown outcome, reported as such rather than guessed. */ + AMBIGUOUS, + /** The message is redelivered because it was never settled. */ + REDELIVERED + } + + public NetworkFaultScenario { + Objects.requireNonNull(phase, "phase must not be null"); + Objects.requireNonNull(expectation, "expectation must not be null"); + Objects.requireNonNull(duration, "duration must not be null"); + if (name == null || name.isBlank()) { + throw new IllegalArgumentException("name must not be blank"); + } + if (rationale == null || rationale.isBlank()) { + throw new IllegalArgumentException( + "a scenario without a rationale cannot be reviewed; state why this outcome is correct"); + } + } + + /** The connection is refused before the publish frame is written. */ + public static final NetworkFaultScenario CONNECTION_REFUSED = + new NetworkFaultScenario( + "connection-refused", + Phase.BEFORE_TRANSMISSION, + Expectation.REJECTED, + Optional.empty(), + "no bytes left the client, so the broker cannot hold the message and a retry cannot " + + "duplicate it"); + + /** The connection is cut after the frame is written but before the confirm. */ + public static final NetworkFaultScenario CONNECTION_CUT_AFTER_WRITE = + new NetworkFaultScenario( + "connection-cut-after-write", + Phase.AFTER_TRANSMISSION, + Expectation.AMBIGUOUS, + Optional.empty(), + "the broker may have stored the message and only the confirm was lost; reporting a " + + "rejection here is what turns one lost confirmation into two orders"); + + /** The confirm is delayed past the publish deadline. */ + public static final NetworkFaultScenario CONFIRM_TIMEOUT = + new NetworkFaultScenario( + "confirm-timeout", + Phase.AFTER_TRANSMISSION, + Expectation.AMBIGUOUS, + Optional.of(Duration.ofSeconds(10)), + "a timeout is the absence of evidence, not evidence of absence"); + + /** The broker becomes unreachable while a delivery is unsettled. */ + public static final NetworkFaultScenario SETTLEMENT_LOST = + new NetworkFaultScenario( + "settlement-lost", + Phase.DURING_SETTLEMENT, + Expectation.REDELIVERED, + Optional.empty(), + "an unsettled delivery is redelivered by design; the handler must be idempotent rather " + + "than the platform pretending the settlement landed"); + + /** High latency without a disconnect, which slows confirms without losing them. */ + public static final NetworkFaultScenario HIGH_LATENCY = + new NetworkFaultScenario( + "high-latency", + Phase.AFTER_TRANSMISSION, + Expectation.AMBIGUOUS, + Optional.of(Duration.ofSeconds(30)), + "latency above the publish deadline is indistinguishable from a lost confirm at the " + + "moment the decision has to be made"); + + /** + * Returns every scenario the shared fault suite runs. + * + * @return the scenarios + */ + public static java.util.List all() { + return java.util.List.of( + CONNECTION_REFUSED, + CONNECTION_CUT_AFTER_WRITE, + CONFIRM_TIMEOUT, + SETTLEMENT_LOST, + HIGH_LATENCY); + } +} diff --git a/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/ObservedDelivery.java b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/ObservedDelivery.java new file mode 100644 index 00000000..79e01a4c --- /dev/null +++ b/src/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/ObservedDelivery.java @@ -0,0 +1,23 @@ +package dev.caskeleton.messaging.testkit; + +import dev.caskeleton.messaging.api.MessageId; +import java.util.Objects; + +/** + * What the harness observed about one delivery attempt. + * + * @param messageId the logical identity delivered + * @param attempt the attempt number, counting the first delivery as one + * @param redelivered whether the broker flagged this as a redelivery + * @param settled whether the source was settled after handling + */ +public record ObservedDelivery( + MessageId messageId, int attempt, boolean redelivered, boolean settled) { + + public ObservedDelivery { + Objects.requireNonNull(messageId, "messageId must not be null"); + if (attempt < 1) { + throw new IllegalArgumentException("attempt counts the first delivery as 1"); + } + } +} diff --git a/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CompatibilityMatrixTest.java b/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CompatibilityMatrixTest.java new file mode 100644 index 00000000..1476bf51 --- /dev/null +++ b/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CompatibilityMatrixTest.java @@ -0,0 +1,99 @@ +package dev.caskeleton.messaging.testkit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; + +class CompatibilityMatrixTest { + + private static final List REQUIRED_CONTRACT_TESTS = + List.of( + "publishesAndConfirms", + "returnsAmbiguousWhenConfirmIsLost", + "redeliversWhenSettlementIsLost", + "preservesMessageIdAcrossRetryAndDlq", + "keepsSourceUnsettledWhenDlqPublishFails", + "rejectsOversizedPayloadBeforeTransport", + "stopsAcceptingNewWorkDuringShutdown"); + + @Test + void everyStableAdapterRunsTheSharedContract() { + assertThat(CompatibilityMatrix.entries().values()) + .filteredOn(entry -> entry.tier() == CompatibilityMatrix.Tier.STABLE) + .isNotEmpty() + .allSatisfy(entry -> assertThat(entry.runsSharedContract()).isTrue()); + } + + @Test + void noExperimentalAdapterIsEnabledByDefault() { + assertThat(CompatibilityMatrix.entries().values()) + .filteredOn(entry -> entry.tier() == CompatibilityMatrix.Tier.EXPERIMENTAL) + .isNotEmpty() + .allSatisfy(entry -> assertThat(entry.enabledByDefault()).isFalse()); + } + + @Test + void theStableSetIsExactlyKafkaAndRabbit() { + assertThat(CompatibilityMatrix.entries().values()) + .filteredOn(entry -> entry.tier() == CompatibilityMatrix.Tier.STABLE) + .extracting(CompatibilityMatrix.Entry::adapter) + .containsExactlyInAnyOrder("messaging-kafka", "messaging-rabbit"); + } + + @Test + void kafkaIsCertifiedAgainstFourTwoAndFourThree() { + assertThat(CompatibilityMatrix.of("messaging-kafka").brokerVersions()) + .containsExactly("4.2", "4.3"); + } + + @Test + void theSharedContractStillDeclaresEveryRequiredTest() { + List declared = + Arrays.stream(MessagingAdapterContract.class.getDeclaredMethods()) + .filter(method -> method.isAnnotationPresent(org.junit.jupiter.api.Test.class)) + .map(Method::getName) + .toList(); + + assertThat(declared).containsExactlyInAnyOrderElementsOf(REQUIRED_CONTRACT_TESTS); + } + + @Test + void anUnknownAdapterIsNotSilentlyTreatedAsSupported() { + assertThatThrownBy(() -> CompatibilityMatrix.of("messaging-artemis")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void anAdapterMustCertifyAtLeastOneBrokerVersion() { + assertThatThrownBy( + () -> + new CompatibilityMatrix.Entry( + "messaging-x", List.of(), CompatibilityMatrix.Tier.STABLE, true, true, true)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void everyStableAdapterIsCertifiedAgainstALiveBroker() { + assertThat(CompatibilityMatrix.entries().values()) + .filteredOn(entry -> entry.tier() == CompatibilityMatrix.Tier.STABLE) + .isNotEmpty() + .allSatisfy( + entry -> + assertThat(entry.hasLiveBrokerCertification()) + .as( + "%s claims Stable, so a container-backed suite must exercise a real broker", + entry.adapter()) + .isTrue()); + } + + @Test + void noExperimentalAdapterClaimsLiveBrokerCertification() { + assertThat(CompatibilityMatrix.entries().values()) + .filteredOn(entry -> entry.tier() == CompatibilityMatrix.Tier.EXPERIMENTAL) + .allSatisfy(entry -> assertThat(entry.hasLiveBrokerCertification()).isFalse()); + } +} diff --git a/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CrossBrokerContractSuite.java b/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CrossBrokerContractSuite.java new file mode 100644 index 00000000..f4dce867 --- /dev/null +++ b/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CrossBrokerContractSuite.java @@ -0,0 +1,111 @@ +package dev.caskeleton.messaging.testkit; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.testkit.BrokerFailureMatrix.Coverage; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * The release gate over the whole adapter set. + * + *

Every other suite proves one adapter behaves correctly. This one proves the adapters agree — + * that the same fault produces the same reported outcome whichever broker is underneath, and that + * nothing has been promoted to Stable without the evidence that tier claims. + * + *

It asserts on the coverage records rather than re-running the fault suites, because it is the + * bookkeeping that rots: an adapter's own tests keep passing while the matrix quietly + * stops reflecting what was actually run. + */ +class CrossBrokerContractSuite { + + private final BrokerFailureMatrix matrix = BrokerFailureMatrix.shipped(); + + private static List stableAdapters() { + return CompatibilityMatrix.entries().values().stream() + .filter(entry -> entry.tier() == CompatibilityMatrix.Tier.STABLE) + .map(CompatibilityMatrix.Entry::adapter) + .toList(); + } + + @Test + void everyStableAdapterCoversEveryFaultScenario() { + assertThat(stableAdapters()) + .isNotEmpty() + .allSatisfy( + adapter -> + assertThat(matrix.gapsFor(adapter)) + .as("%s is Stable, so no fault scenario may be unrun", adapter) + .isEmpty()); + } + + @Test + void everyStableAdapterCoversTheFaultsAgainstALiveBroker() { + assertThat(stableAdapters()) + .allSatisfy( + adapter -> + assertThat(NetworkFaultScenario.all()) + .allSatisfy( + scenario -> + assertThat(matrix.coverageOf(adapter, scenario)) + .as("%s / %s", adapter, scenario.name()) + .isEqualTo(Coverage.LIVE_BROKER))); + } + + @Test + void noExperimentalAdapterClaimsLiveBrokerFaultCoverage() { + List experimental = + CompatibilityMatrix.entries().values().stream() + .filter(entry -> entry.tier() == CompatibilityMatrix.Tier.EXPERIMENTAL) + .map(CompatibilityMatrix.Entry::adapter) + .toList(); + + assertThat(experimental) + .allSatisfy( + adapter -> + assertThat(NetworkFaultScenario.all()) + .allSatisfy( + scenario -> + assertThat(matrix.coverageOf(adapter, scenario)) + .isNotEqualTo(Coverage.LIVE_BROKER))); + } + + @Test + void anExperimentalAdapterIsAllowedGapsButTheyAreRecorded() { + assertThat(matrix.isComplete("messaging-pulsar-experimental")) + .as("the Experimental tier is a statement of fact, not a disclaimer") + .isFalse(); + assertThat(matrix.gapsFor("messaging-pulsar-experimental")).isNotEmpty(); + } + + @Test + void aFaultThatWasNeverRecordedReadsAsUncoveredRatherThanPassing() { + assertThat(matrix.coverageOf("messaging-artemis", NetworkFaultScenario.CONFIRM_TIMEOUT)) + .isEqualTo(Coverage.NOT_COVERED); + } + + @Test + void everyAdapterAgreesThatAWriteFollowedByALostConfirmIsAmbiguous() { + assertThat(NetworkFaultScenario.CONNECTION_CUT_AFTER_WRITE.expectation()) + .as("the shared expectation is what stops two adapters answering differently") + .isEqualTo(NetworkFaultScenario.Expectation.AMBIGUOUS); + assertThat(NetworkFaultScenario.CONFIRM_TIMEOUT.expectation()) + .isEqualTo(NetworkFaultScenario.Expectation.AMBIGUOUS); + } + + @Test + void aFailureBeforeTransmissionIsTheOnlyOneReportedAsRejected() { + assertThat(NetworkFaultScenario.all()) + .filteredOn(scenario -> scenario.expectation() == NetworkFaultScenario.Expectation.REJECTED) + .allSatisfy( + scenario -> + assertThat(scenario.phase()) + .isEqualTo(NetworkFaultScenario.Phase.BEFORE_TRANSMISSION)); + } + + @Test + void everyScenarioStatesWhyItsOutcomeIsCorrect() { + assertThat(NetworkFaultScenario.all()) + .allSatisfy(scenario -> assertThat(scenario.rationale()).isNotBlank()); + } +} diff --git a/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/InMemoryHarnessContractTest.java b/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/InMemoryHarnessContractTest.java new file mode 100644 index 00000000..5cd052a0 --- /dev/null +++ b/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/InMemoryHarnessContractTest.java @@ -0,0 +1,22 @@ +package dev.caskeleton.messaging.testkit; + +import org.junit.jupiter.api.Nested; + +/** + * Runs the shared adapter contract against the in-memory harness. + * + *

This is the contract testing itself. Every broker adapter adds the same nested class over its + * own harness, so a guarantee can only be weakened by editing the contract, where the change is + * visible, rather than by an adapter quietly not implementing it. + */ +class InMemoryHarnessContractTest { + + @Nested + class Contract extends MessagingAdapterContract { + + @Override + protected MessagingAdapterHarness harness() { + return InMemoryMessagingHarness.create(); + } + } +} diff --git a/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/InMemoryMessagingHarness.java b/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/InMemoryMessagingHarness.java new file mode 100644 index 00000000..9f3c9c0c --- /dev/null +++ b/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/InMemoryMessagingHarness.java @@ -0,0 +1,260 @@ +package dev.caskeleton.messaging.testkit; + +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * A deterministic harness that exists only to prove the contract suite itself is sound. + * + *

It is not a production adapter and must never become one. Its value is that it has no broker + * to hide behind: if a contract test passes here and fails on Kafka, the difference is in the + * adapter, not in the test. + */ +final class InMemoryMessagingHarness implements MessagingAdapterHarness { + + private static final int MAX_PAYLOAD_BYTES = 1_048_576; + + private final Deque pending = new ArrayDeque<>(); + private final List deadLetters = new ArrayList<>(); + private final Set unsettled = new LinkedHashSet<>(); + private final Faults faults = new Faults(); + + private boolean shuttingDown; + + private InMemoryMessagingHarness() {} + + static InMemoryMessagingHarness create() { + return new InMemoryMessagingHarness(); + } + + @Override + public String brokerName() { + return "in-memory"; + } + + @Override + public FaultController faults() { + return faults; + } + + @Override + public CompletionStage publish(ContractMessage message) { + if (shuttingDown) { + return completed( + rejected("SHUTTING_DOWN", "the consumer is draining and accepts no new work")); + } + if (message.envelope().payload().size() > MAX_PAYLOAD_BYTES) { + return completed( + rejected("PAYLOAD_TOO_LARGE", "encoded payload exceeds " + MAX_PAYLOAD_BYTES + " bytes")); + } + if (faults.consumeRejectPublish()) { + return completed(rejected("BROKER_REJECTED", "the broker refused the publish")); + } + + if (faults.consumeDropPublishConfirmation()) { + return completed( + new PublishResult( + PublishCompletion.AMBIGUOUS, + PublishEvidence.ambiguous(), + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + Duration.ofMillis(1), + Optional.of( + FailureDescriptor.of( + FailureCategory.AMBIGUOUS, + "CONFIRM_LOST", + "the publish confirmation was not received")))); + } + + pending.addLast(new Pending(message.messageId(), 1, false)); + return completed( + new PublishResult( + PublishCompletion.CONFIRMED, + PublishEvidence.confirmed(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK), + RoutingOutcome.ROUTED, + Optional.empty(), + 1, + Duration.ofMillis(1), + Optional.empty())); + } + + @Override + public List drain(HandleOutcome outcome) { + List batch = new ArrayList<>(pending); + pending.clear(); + + List observed = new ArrayList<>(); + for (Pending delivery : batch) { + boolean settled = + switch (outcome) { + case SUCCESS -> settleAfterSuccess(delivery); + case RETRY -> { + unsettled.add(delivery.messageId()); + redeliver(delivery); + yield false; + } + case DEAD_LETTER -> deadLetter(delivery); + }; + observed.add( + new ObservedDelivery( + delivery.messageId(), delivery.attempt(), delivery.redelivered(), settled)); + } + return List.copyOf(observed); + } + + /** + * Settles the source only when the settlement itself was confirmed. + * + *

An unconfirmed settlement leaves the message in flight, which is what produces the + * redelivery the contract expects. + */ + private boolean settleAfterSuccess(Pending delivery) { + if (faults.consumeDropSettlementConfirmation()) { + unsettled.add(delivery.messageId()); + redeliver(delivery); + return false; + } + unsettled.remove(delivery.messageId()); + return true; + } + + /** + * Publishes to the dead letter destination before settling the source. + * + *

When that publish fails the source stays unsettled and in flight. Acknowledging it here + * would destroy the last copy of a message the dead letter destination never received. + */ + private boolean deadLetter(Pending delivery) { + if (faults.deadLetterPublishFails()) { + unsettled.add(delivery.messageId()); + redeliver(delivery); + return false; + } + deadLetters.add(delivery.messageId()); + unsettled.remove(delivery.messageId()); + return true; + } + + private void redeliver(Pending delivery) { + pending.addLast(new Pending(delivery.messageId(), delivery.attempt() + 1, true)); + } + + @Override + public List deadLettered() { + return List.copyOf(deadLetters); + } + + @Override + public List unsettled() { + return List.copyOf(unsettled); + } + + @Override + public void beginShutdown() { + shuttingDown = true; + } + + @Override + public boolean isAcceptingWork() { + return !shuttingDown; + } + + @Override + public void close() { + pending.clear(); + } + + private static PublishResult rejected(String code, String message) { + return new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ZERO, + Optional.of(FailureDescriptor.of(FailureCategory.PERMANENT_BUSINESS, code, message))); + } + + private static CompletionStage completed(PublishResult result) { + return CompletableFuture.completedFuture(result); + } + + /** One message waiting to be delivered. */ + private record Pending(MessageId messageId, int attempt, boolean redelivered) {} + + /** One-shot fault flags, consumed by the operation they target. */ + private static final class Faults implements FaultController { + + private boolean dropPublishConfirmation; + private boolean dropSettlementConfirmation; + private boolean failDeadLetterPublish; + private boolean rejectPublish; + + @Override + public void dropPublishConfirmation() { + dropPublishConfirmation = true; + } + + @Override + public void dropSettlementConfirmation() { + dropSettlementConfirmation = true; + } + + @Override + public void failDeadLetterPublish() { + failDeadLetterPublish = true; + } + + @Override + public void rejectPublish() { + rejectPublish = true; + } + + @Override + public void reset() { + dropPublishConfirmation = false; + dropSettlementConfirmation = false; + failDeadLetterPublish = false; + rejectPublish = false; + } + + boolean consumeDropPublishConfirmation() { + boolean active = dropPublishConfirmation; + dropPublishConfirmation = false; + return active; + } + + boolean consumeDropSettlementConfirmation() { + boolean active = dropSettlementConfirmation; + dropSettlementConfirmation = false; + return active; + } + + boolean consumeRejectPublish() { + boolean active = rejectPublish; + rejectPublish = false; + return active; + } + + boolean deadLetterPublishFails() { + return failDeadLetterPublish; + } + } +} diff --git a/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/MessagingDocumentationContractTest.java b/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/MessagingDocumentationContractTest.java new file mode 100644 index 00000000..0b7471a5 --- /dev/null +++ b/src/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/MessagingDocumentationContractTest.java @@ -0,0 +1,145 @@ +package dev.caskeleton.messaging.testkit; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * Checks that the shipped documentation still describes the code. + * + *

Docs rot silently. A support matrix claiming an adapter is Stable outlives the day someone + * demoted it, and nothing fails — the tests still pass, the build is green, and the only signal is + * an operator making a decision on a page that stopped being true months ago. + * + *

The assertions are deliberately narrow: they check the claims a reader would act on, not + * prose. Asserting on wording would make every edit a test failure and the check would be deleted. + */ +class MessagingDocumentationContractTest { + + private static Path docsRoot() { + Path candidate = Path.of("").toAbsolutePath(); + while (candidate != null && !Files.isDirectory(candidate.resolve("docs/messaging"))) { + candidate = candidate.getParent(); + } + if (candidate == null) { + throw new IllegalStateException("could not locate docs/messaging from the test cwd"); + } + return candidate.resolve("docs/messaging"); + } + + private static String read(String name) { + try { + return Files.readString(docsRoot().resolve(name)); + } catch (IOException failure) { + throw new IllegalStateException("could not read " + name, failure); + } + } + + @Test + void everyDocumentTheSupportMatrixPromisesExists() { + List expected = + List.of( + "support-matrix.md", + "delivery-guarantees.md", + "retry-dlq-redrive.md", + "outbox-inbox.md", + "security.md", + "operations.md", + "configuration-reference.md", + "experimental-policy.md", + "migration-guide.md"); + + assertThat(expected) + .allSatisfy(name -> assertThat(Files.exists(docsRoot().resolve(name))).isTrue()); + } + + @Test + void theSupportMatrixNamesExactlyTheAdaptersTheCodeCallsStable() { + String matrix = read("support-matrix.md"); + + CompatibilityMatrix.entries().values().stream() + .filter(entry -> entry.tier() == CompatibilityMatrix.Tier.STABLE) + .forEach(entry -> assertThat(matrix).contains(brokerLabel(entry.adapter()))); + } + + @Test + void theSupportMatrixDoesNotCallAnExperimentalAdapterStable() { + String matrix = read("support-matrix.md"); + + CompatibilityMatrix.entries().values().stream() + .filter(entry -> entry.tier() == CompatibilityMatrix.Tier.EXPERIMENTAL) + .forEach( + entry -> + assertThat(matrix) + .as("%s is Experimental in code", entry.adapter()) + .doesNotContain("| %s | Stable".formatted(brokerLabel(entry.adapter())))); + } + + @Test + void theDocumentedKafkaVersionsMatchWhatTheCodeCertifies() { + String matrix = read("support-matrix.md"); + + CompatibilityMatrix.of("messaging-kafka") + .brokerVersions() + .forEach(version -> assertThat(matrix).contains(version)); + } + + @Test + void theUnsupportedListStillNamesTheTwoConstantsThatDoNotExist() { + String matrix = read("support-matrix.md"); + + assertThat(matrix) + .as("a reader must be able to confirm the platform never offers these") + .contains("EXACTLY_ONCE") + .contains("GLOBAL"); + } + + @Test + void noEnumConstantTheDocsDenyActuallyExists() { + assertThat( + java.util.Arrays.stream( + dev.caskeleton.messaging.api.delivery.DeliveryGuarantee.values()) + .map(Enum::name)) + .doesNotContain("EXACTLY_ONCE"); + assertThat( + java.util.Arrays.stream(dev.caskeleton.messaging.api.delivery.OrderingScope.values()) + .map(Enum::name)) + .doesNotContain("GLOBAL"); + } + + @Test + void theExperimentalPolicyStatesThatExperimentalIsOffByDefault() { + assertThat(read("experimental-policy.md")).contains("false"); + } + + @Test + void everyDocumentHasContent() { + assertThat( + List.of( + "support-matrix.md", + "delivery-guarantees.md", + "retry-dlq-redrive.md", + "outbox-inbox.md", + "security.md", + "operations.md", + "configuration-reference.md", + "experimental-policy.md", + "migration-guide.md")) + .allSatisfy(name -> assertThat(read(name).length()).isGreaterThan(500)); + } + + private static String brokerLabel(String adapter) { + return switch (adapter) { + case "messaging-kafka" -> "Kafka"; + case "messaging-rabbit" -> "RabbitMQ"; + case "messaging-pulsar-experimental" -> "Pulsar"; + case "messaging-nats-experimental" -> "NATS"; + case "messaging-kafka-share-experimental" -> "Kafka Share"; + default -> adapter; + }; + } +} diff --git a/src/messaging/messaging-transport-spi/build.gradle b/src/messaging/messaging-transport-spi/build.gradle new file mode 100644 index 00000000..283c584f --- /dev/null +++ b/src/messaging/messaging-transport-spi/build.gradle @@ -0,0 +1,7 @@ +apply plugin: 'java-library' + +dependencies { + api project(':messaging:messaging-core-api') + api project(':messaging:messaging-schema-api') + api project(':messaging:messaging-policy') +} diff --git a/src/messaging/messaging-transport-spi/gradle.lockfile b/src/messaging/messaging-transport-spi/gradle.lockfile new file mode 100644 index 00000000..599ff921 --- /dev/null +++ b/src/messaging/messaging-transport-spi/gradle.lockfile @@ -0,0 +1,83 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ben-manes.caffeine:caffeine:3.2.3=annotationProcessor,testAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs +com.github.spotbugs:spotbugs:4.10.2=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.errorprone:error_prone_annotations:2.47.0=checkstyle +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=checkstyle +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins +com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +commons-beanutils:commons-beanutils:1.11.0=checkstyle +commons-collections:commons-collections:3.2.2=checkstyle +commons-io:commons-io:2.21.0=spotbugs +info.picocli:picocli:4.7.7=checkstyle +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs +org.antlr:antlr4-runtime:4.13.2=checkstyle +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.commons:commons-text:1.3=checkstyle +org.apache.httpcomponents:httpclient:4.5.13=checkstyle +org.apache.httpcomponents:httpcore:4.4.16=checkstyle +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apache.maven.doxia:doxia-core:1.12.0=checkstyle +org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle +org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle +org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle +org.apache.xbean:xbean-reflect:3.7=checkstyle +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.6=testCompileClasspath,testRuntimeClasspath +org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle +org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle +org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle +org.codehaus.plexus:plexus-utils:3.3.0=checkstyle +org.dom4j:dom4j:2.2.0=spotbugs +org.javassist:javassist:3.28.0-GA=checkstyle +org.jspecify:jspecify:1.0.0=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:6.0.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:6.0.1=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:6.0.1=testRuntimeClasspath +org.junit:junit-bom:6.0.1=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.0=spotbugs +org.mockito:mockito-core:5.20.0=mockitoAgent +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-commons:9.10.1=spotbugs +org.ow2.asm:asm-tree:9.10.1=spotbugs +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm:9.10.1=spotbugs +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.reflections:reflections:0.10.2=checkstyle +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=checkstyle,spotbugsSlf4j +org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +empty=compileClasspath,runtimeClasspath diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/BackpressureController.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/BackpressureController.java new file mode 100644 index 00000000..2c0cba45 --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/BackpressureController.java @@ -0,0 +1,114 @@ +package dev.caskeleton.messaging.transport; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Bounds in-flight work per destination and across the process. + * + *

Two limits, because one is not enough. A per-destination limit stops a single slow downstream + * from consuming every worker, and a global limit stops the sum of well-behaved destinations from + * exhausting memory. Without the global bound, adding a destination silently raises the process + * ceiling. + * + *

Refusing work is the point. An unbounded in-flight queue does not remove backpressure, it + * moves it into the heap, where it surfaces as an out-of-memory error instead of a visible, + * recoverable slowdown. + */ +public final class BackpressureController { + + private final int globalLimit; + private final int perDestinationLimit; + private final AtomicInteger global = new AtomicInteger(); + private final Map perDestination = new ConcurrentHashMap<>(); + + /** + * Creates a controller. + * + * @param globalLimit the process-wide in-flight ceiling + * @param perDestinationLimit the per-destination in-flight ceiling + */ + public BackpressureController(int globalLimit, int perDestinationLimit) { + if (globalLimit < 1) { + throw new IllegalArgumentException("globalLimit must be at least 1"); + } + if (perDestinationLimit < 1) { + throw new IllegalArgumentException("perDestinationLimit must be at least 1"); + } + if (perDestinationLimit > globalLimit) { + throw new IllegalArgumentException( + "perDestinationLimit cannot exceed the global limit, or the global limit is not a limit"); + } + this.globalLimit = globalLimit; + this.perDestinationLimit = perDestinationLimit; + } + + /** + * Tries to admit one unit of work. + * + * @param destination the logical destination + * @return true when the work may start + */ + public boolean tryAcquire(String destination) { + Objects.requireNonNull(destination, "destination must not be null"); + AtomicInteger counter = perDestination.computeIfAbsent(destination, key -> new AtomicInteger()); + + if (!increment(counter, perDestinationLimit)) { + return false; + } + if (!increment(global, globalLimit)) { + counter.decrementAndGet(); + return false; + } + return true; + } + + /** + * Releases one unit of work. + * + * @param destination the logical destination + */ + public void release(String destination) { + AtomicInteger counter = perDestination.get(destination); + if (counter != null && counter.get() > 0) { + counter.decrementAndGet(); + } + if (global.get() > 0) { + global.decrementAndGet(); + } + } + + /** + * Returns the process-wide in-flight count. + * + * @return the global in-flight count + */ + public int globalInFlight() { + return global.get(); + } + + /** + * Returns a destination's in-flight count. + * + * @param destination the logical destination + * @return the in-flight count + */ + public int inFlight(String destination) { + AtomicInteger counter = perDestination.get(destination); + return counter == null ? 0 : counter.get(); + } + + private static boolean increment(AtomicInteger counter, int limit) { + while (true) { + int current = counter.get(); + if (current >= limit) { + return false; + } + if (counter.compareAndSet(current, current + 1)) { + return true; + } + } + } +} diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/DefaultMessagingRuntimeRegistry.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/DefaultMessagingRuntimeRegistry.java new file mode 100644 index 00000000..b5326ec3 --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/DefaultMessagingRuntimeRegistry.java @@ -0,0 +1,189 @@ +package dev.caskeleton.messaging.transport; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Reference-counted runtime generations with an atomic swap on install. + * + *

Rotation works by publishing a new generation and letting the old one drain. The map entry is + * replaced atomically so a caller either sees the old generation or the new one, never a + * half-rebuilt connection pool; the retired generation stays open until its last lease closes, so + * an in-flight publish still gets its confirm on the connection it started on. + * + *

A drain deadline bounds that wait. Without one, a single leaked lease would pin a revoked + * credential open indefinitely, which turns a rotation into a security non-event. + */ +public final class DefaultMessagingRuntimeRegistry implements MessagingRuntimeRegistry { + + private static final Duration DEFAULT_DRAIN_DEADLINE = Duration.ofSeconds(30); + + private final Map current = new ConcurrentHashMap<>(); + private final List draining = new ArrayList<>(); + private final Duration drainDeadline; + + /** Creates a registry with the default thirty second drain deadline. */ + public DefaultMessagingRuntimeRegistry() { + this(DEFAULT_DRAIN_DEADLINE); + } + + /** + * Creates a registry with an explicit drain deadline. + * + * @param drainDeadline how long a retired generation may wait for its leases + */ + public DefaultMessagingRuntimeRegistry(Duration drainDeadline) { + Objects.requireNonNull(drainDeadline, "drainDeadline must not be null"); + if (drainDeadline.isNegative()) { + throw new IllegalArgumentException("drainDeadline must not be negative"); + } + this.drainDeadline = drainDeadline; + } + + @Override + public void install(MessagingRuntime runtime) { + Objects.requireNonNull(runtime, "runtime must not be null"); + Generation retired = current.put(runtime.brokerName(), new Generation(runtime)); + if (retired == null) { + return; + } + retired.retire(); + if (!retired.closeIfIdle()) { + synchronized (draining) { + draining.add(retired); + } + } + } + + @Override + public MessagingRuntimeLease acquire(String brokerName) { + Objects.requireNonNull(brokerName, "brokerName must not be null"); + Generation generation = + current.computeIfPresent( + brokerName, + (key, value) -> { + value.leases.incrementAndGet(); + return value; + }); + if (generation == null) { + throw new MessagingConfigurationException( + "RUNTIME_NOT_INSTALLED", "no messaging runtime is installed for broker " + brokerName); + } + return new Lease(generation); + } + + /** + * Closes retired generations whose drain deadline has passed. + * + *

Called by the platform's scheduler. Exposed as a plain method taking the current instant so + * that deadline behaviour is testable without sleeping. + * + * @param now the current instant + * @param retiredAt when the generations were retired + * @return how many generations were force closed + */ + public int closeExpiredDraining(Instant now, Instant retiredAt) { + Objects.requireNonNull(now, "now must not be null"); + Objects.requireNonNull(retiredAt, "retiredAt must not be null"); + if (now.isBefore(retiredAt.plus(drainDeadline))) { + return 0; + } + List expired; + synchronized (draining) { + expired = List.copyOf(draining); + draining.clear(); + } + int closed = 0; + for (Generation generation : expired) { + if (generation.forceClose()) { + closed++; + } + } + return closed; + } + + /** + * Returns how many retired generations are still draining. + * + * @return the draining count + */ + public int drainingCount() { + synchronized (draining) { + return draining.size(); + } + } + + /** One installed generation plus its live lease count. */ + private static final class Generation { + + private final MessagingRuntime runtime; + private final AtomicInteger leases = new AtomicInteger(); + private final AtomicBoolean retired = new AtomicBoolean(); + private final AtomicBoolean closed = new AtomicBoolean(); + + private Generation(MessagingRuntime runtime) { + this.runtime = runtime; + } + + void retire() { + retired.set(true); + } + + /** + * Closes the runtime when it is retired and no lease remains. + * + * @return true when this call closed the runtime + */ + boolean closeIfIdle() { + if (retired.get() && leases.get() == 0) { + return forceClose(); + } + return false; + } + + boolean forceClose() { + if (closed.compareAndSet(false, true)) { + runtime.close(); + return true; + } + return false; + } + + void release() { + if (leases.decrementAndGet() == 0) { + closeIfIdle(); + } + } + } + + /** A single borrowed reference, safe to close more than once. */ + private static final class Lease implements MessagingRuntimeLease { + + private final Generation generation; + private final AtomicBoolean released = new AtomicBoolean(); + + private Lease(Generation generation) { + this.generation = generation; + } + + @Override + public MessagingRuntime runtime() { + return generation.runtime; + } + + @Override + public void close() { + if (released.compareAndSet(false, true)) { + generation.release(); + } + } + } +} diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/GracefulShutdownCoordinator.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/GracefulShutdownCoordinator.java new file mode 100644 index 00000000..a27fbf2f --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/GracefulShutdownCoordinator.java @@ -0,0 +1,133 @@ +package dev.caskeleton.messaging.transport; + +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Drains in-flight work without starting any more. + * + *

Shutdown has three phases, in order: stop accepting new work, let what is running finish, then + * close. Skipping the middle phase is what produces the classic shutdown bug — a handler is + * interrupted between its side effect and its settlement, so the message is redelivered and the + * effect happens twice. + * + *

The deadline exists because draining cannot be unbounded: a stuck handler would otherwise hold + * the process open forever. Work still running at the deadline is abandoned unsettled, so + * the broker redelivers it rather than the platform pretending it completed. + * + *

No retry attempt is created once draining begins. Starting a fresh attempt during shutdown + * guarantees it will be abandoned at the deadline. + */ +public final class GracefulShutdownCoordinator { + + private final Duration drainDeadline; + private final AtomicBoolean draining = new AtomicBoolean(); + private final AtomicInteger inFlight = new AtomicInteger(); + + private volatile Instant drainStartedAt; + + /** + * Creates a coordinator. + * + * @param drainDeadline how long to wait for in-flight work + */ + public GracefulShutdownCoordinator(Duration drainDeadline) { + Objects.requireNonNull(drainDeadline, "drainDeadline must not be null"); + if (drainDeadline.isNegative()) { + throw new IllegalArgumentException("drainDeadline must not be negative"); + } + this.drainDeadline = drainDeadline; + } + + /** + * Tries to start one unit of work. + * + * @return true when the work may start; false once draining has begun + */ + public boolean tryBeginWork() { + if (draining.get()) { + return false; + } + inFlight.incrementAndGet(); + if (draining.get()) { + inFlight.decrementAndGet(); + return false; + } + return true; + } + + /** Records that one unit of work finished. */ + public void endWork() { + inFlight.decrementAndGet(); + } + + /** + * Begins draining. + * + * @param now the current instant + */ + public void beginDrain(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + if (draining.compareAndSet(false, true)) { + drainStartedAt = now; + } + } + + /** + * Reports whether new work is still accepted. + * + * @return true until draining begins + */ + public boolean isAcceptingWork() { + return !draining.get(); + } + + /** + * Reports whether a retry attempt may be created. + * + * @return true until draining begins + */ + public boolean mayCreateRetryAttempt() { + return !draining.get(); + } + + /** + * Returns how many units of work are still running. + * + * @return the in-flight count + */ + public int inFlight() { + return inFlight.get(); + } + + /** + * Reports whether the drain has finished, either by completing or by hitting its deadline. + * + * @param now the current instant + * @return true when it is safe to close + */ + public boolean isDrained(Instant now) { + Objects.requireNonNull(now, "now must not be null"); + if (!draining.get()) { + return false; + } + if (inFlight.get() == 0) { + return true; + } + Instant startedAt = drainStartedAt; + return startedAt != null && !now.isBefore(startedAt.plus(drainDeadline)); + } + + /** + * Reports whether work was abandoned because the deadline passed. + * + * @param now the current instant + * @return true when in-flight work outlived the deadline + */ + public boolean abandonedWorkAtDeadline(Instant now) { + return isDrained(now) && inFlight.get() > 0; + } +} diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingLifecycle.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingLifecycle.java new file mode 100644 index 00000000..e45aa2bb --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingLifecycle.java @@ -0,0 +1,67 @@ +package dev.caskeleton.messaging.transport; + +import java.time.Duration; + +/** + * The ordered lifecycle every messaging runtime implements. + * + *

The order in {@link ShutdownPhase} is the contract, not an implementation detail. Closing + * connections before settlements have been transmitted loses the settlements, and pausing consumers + * after draining lets fresh deliveries arrive into a runtime that is already shutting down. Each + * adapter implements the phases; none of them chooses the order. + * + *

Implementations are driven by the Spring lifecycle rather than a JVM shutdown hook alone. A + * shutdown hook runs after the context has already begun disposing beans, so a handler mid-drain + * can find its datasource closed underneath it. + */ +public interface MessagingLifecycle { + + /** The fixed phases of an orderly shutdown, in the order they must run. */ + enum ShutdownPhase { + /** Refuse new publishes. Nothing in flight is affected. */ + STOP_PUBLISH_ADMISSION, + /** Refuse to start new delivery handlers. */ + STOP_NEW_HANDLERS, + /** Ask the broker to stop delivering. */ + PAUSE_CONSUMERS, + /** Let running handlers finish. */ + DRAIN_HANDLERS, + /** Transmit the settlements those handlers produced. */ + FLUSH_SETTLEMENTS, + /** Wait for outstanding producer confirms so publishes are not left ambiguous. */ + AWAIT_PRODUCER_CONFIRMS, + /** Return outbox leases so another relay can claim the rows immediately. */ + RELEASE_OUTBOX_LEASES, + /** Close connections and channels. */ + CLOSE_CONNECTIONS + } + + /** The default drain budget from the design. */ + Duration DEFAULT_DRAIN_DEADLINE = Duration.ofSeconds(30); + + /** + * Starts the runtime. + * + * @throws IllegalStateException when the runtime was already started + */ + void start(); + + /** + * Runs the shutdown phases in order, bounded by the drain deadline. + * + *

Work still running at the deadline is abandoned unsettled so the broker redelivers + * it. Unconfirmed publishes are recorded as ambiguous rather than as successes. + * + * @param drainDeadline how long in-flight work may take + * @return the phase that was still running when the deadline passed, or {@link + * ShutdownPhase#CLOSE_CONNECTIONS} when the shutdown completed + */ + ShutdownPhase shutdown(Duration drainDeadline); + + /** + * Reports whether the runtime is running and accepting work. + * + * @return true between {@link #start()} and the first shutdown phase + */ + boolean isRunning(); +} diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingRuntime.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingRuntime.java new file mode 100644 index 00000000..4155ce5c --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingRuntime.java @@ -0,0 +1,43 @@ +package dev.caskeleton.messaging.transport; + +/** + * One immutable generation of a broker's connections, credentials, and topology view. + * + *

Credential rotation and topology reload replace a whole generation rather than mutating a live + * one. In-flight publishes keep the generation they started on, which is what makes a rotation + * invisible to callers instead of a burst of authentication failures. + */ +public interface MessagingRuntime extends AutoCloseable { + + /** + * Returns the broker this runtime serves. + * + * @return the broker name + */ + String brokerName(); + + /** + * Returns the generation number, increasing with each replacement. + * + * @return the generation + */ + long generation(); + + /** + * Returns the transport bound to this generation. + * + * @return the transport + */ + MessagingTransport transport(); + + /** + * Reports whether this generation has been closed. + * + * @return true once closed + */ + boolean isClosed(); + + /** Closes this generation's resources. Must be idempotent. */ + @Override + void close(); +} diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingRuntimeLease.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingRuntimeLease.java new file mode 100644 index 00000000..2977936e --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingRuntimeLease.java @@ -0,0 +1,22 @@ +package dev.caskeleton.messaging.transport; + +/** + * A borrowed reference to a runtime generation. + * + *

Holding a lease pins the generation open. A caller that has begun a publish keeps the runtime + * it started on even after a rotation installs a newer one, so a rotation never yanks a connection + * out from under an in-flight confirm. + */ +public interface MessagingRuntimeLease extends AutoCloseable { + + /** + * Returns the pinned runtime. + * + * @return the runtime generation + */ + MessagingRuntime runtime(); + + /** Releases the reference. Must be idempotent. */ + @Override + void close(); +} diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingRuntimeRegistry.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingRuntimeRegistry.java new file mode 100644 index 00000000..edf990de --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingRuntimeRegistry.java @@ -0,0 +1,20 @@ +package dev.caskeleton.messaging.transport; + +/** Holds the current runtime generation for each broker. */ +public interface MessagingRuntimeRegistry { + + /** + * Installs a new generation, retiring any previous one. + * + * @param runtime the new generation + */ + void install(MessagingRuntime runtime); + + /** + * Borrows the current generation for a broker. + * + * @param brokerName the broker name + * @return a lease that pins the generation until closed + */ + MessagingRuntimeLease acquire(String brokerName); +} diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingTransport.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingTransport.java new file mode 100644 index 00000000..5a3abd11 --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/MessagingTransport.java @@ -0,0 +1,57 @@ +package dev.caskeleton.messaging.transport; + +import dev.caskeleton.messaging.api.destination.DestinationCapabilities; +import dev.caskeleton.messaging.api.destination.DestinationName; +import java.util.concurrent.CompletionStage; + +/** + * The SPI every broker adapter implements. + * + *

No method returns a native client object. Handing back a raw producer or channel would let an + * application bypass destination policy, payload limits, and the settlement ordering in one call, + * and the resulting code would silently stop working the moment the broker changed. + */ +public interface MessagingTransport extends AutoCloseable { + + /** + * Publishes one already-encoded message. + * + * @param request the publish request + * @return a stage completing with the transport outcome + */ + CompletionStage publish(TransportPublishRequest request); + + /** + * Starts consuming a destination. + * + * @param spec what to consume and where to deliver it + * @return the live registration + */ + TransportConsumerRegistration register(TransportConsumerSpec spec); + + /** + * Returns what this adapter can prove for a destination. + * + * @param destination the logical destination + * @return the capability snapshot + */ + DestinationCapabilities capabilities(DestinationName destination); + + /** + * Returns the broker family name. + * + * @return a stable, low-cardinality name + */ + String brokerName(); + + /** + * Returns the runtime generation this transport belongs to. + * + * @return the generation number + */ + long generation(); + + /** Releases broker resources. */ + @Override + void close(); +} diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportConsumerRegistration.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportConsumerRegistration.java new file mode 100644 index 00000000..8beeffe2 --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportConsumerRegistration.java @@ -0,0 +1,40 @@ +package dev.caskeleton.messaging.transport; + +import java.util.concurrent.CompletionStage; + +/** + * A live consumer subscription. + * + *

Pause and resume operate on an ordering unit rather than the whole consumer, because that is + * what makes {@code PAUSE_PARTITION} retry possible: one stuck key must not stall every other + * partition on the same connection. + */ +public interface TransportConsumerRegistration extends AutoCloseable { + + /** + * Pauses one ordering unit, or the whole registration when the scope is empty. + * + * @param scope the ordering unit, or an empty string for all + * @return a stage completing when the pause has taken effect + */ + CompletionStage pause(String scope); + + /** + * Resumes a previously paused scope. + * + * @param scope the ordering unit, or an empty string for all + * @return a stage completing when the resume has taken effect + */ + CompletionStage resume(String scope); + + /** + * Reports whether the registration is still delivering. + * + * @return true while active + */ + boolean isActive(); + + /** Stops delivering and releases broker resources. */ + @Override + void close(); +} diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportConsumerSpec.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportConsumerSpec.java new file mode 100644 index 00000000..2bf901c3 --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportConsumerSpec.java @@ -0,0 +1,21 @@ +package dev.caskeleton.messaging.transport; + +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.util.Objects; +import java.util.concurrent.CompletionStage; +import java.util.function.Function; + +/** + * What an adapter needs to start consuming a destination. + * + * @param profile the validated destination profile + * @param sink the platform callback invoked for each delivery + */ +public record TransportConsumerSpec( + DestinationProfile profile, Function> sink) { + + public TransportConsumerSpec { + Objects.requireNonNull(profile, "profile must not be null"); + Objects.requireNonNull(sink, "sink must not be null"); + } +} diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportDelivery.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportDelivery.java new file mode 100644 index 00000000..9a980073 --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportDelivery.java @@ -0,0 +1,29 @@ +package dev.caskeleton.messaging.transport; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.delivery.DeliveryMetadata; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.util.Objects; + +/** + * One still-encoded delivery handed up from an adapter. + * + *

Decoding happens above the transport so that a payload the consumer cannot parse is classified + * as a schema failure by the platform, and parked, rather than being turned into an + * adapter-specific exception each broker reports differently. + * + * @param envelope the envelope carrying the encoded payload + * @param metadata transport-side delivery facts + * @param settlement the handle for settling this delivery + */ +public record TransportDelivery( + MessageEnvelope envelope, + DeliveryMetadata metadata, + TransportSettlement settlement) { + + public TransportDelivery { + Objects.requireNonNull(envelope, "envelope must not be null"); + Objects.requireNonNull(metadata, "metadata must not be null"); + Objects.requireNonNull(settlement, "settlement must not be null"); + } +} diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportPublishRequest.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportPublishRequest.java new file mode 100644 index 00000000..30ad41a8 --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportPublishRequest.java @@ -0,0 +1,28 @@ +package dev.caskeleton.messaging.transport; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.util.Objects; + +/** + * One publish handed to an adapter. + * + *

The payload arrives already encoded and the profile arrives already validated, so an adapter + * never chooses a codec or a limit for itself. That is what keeps two adapters from disagreeing + * about what "the same message" means. + * + * @param profile the validated destination profile + * @param envelope the envelope carrying the encoded payload + * @param options the per-call publish options + */ +public record TransportPublishRequest( + DestinationProfile profile, MessageEnvelope envelope, PublishOptions options) { + + public TransportPublishRequest { + Objects.requireNonNull(profile, "profile must not be null"); + Objects.requireNonNull(envelope, "envelope must not be null"); + Objects.requireNonNull(options, "options must not be null"); + } +} diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportPublishResult.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportPublishResult.java new file mode 100644 index 00000000..f059a4ab --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportPublishResult.java @@ -0,0 +1,20 @@ +package dev.caskeleton.messaging.transport; + +import dev.caskeleton.messaging.api.publish.PublishResult; +import java.util.Objects; + +/** + * What an adapter reports back from a publish. + * + *

Adapters return the full {@link PublishResult} rather than a boolean plus an exception. An + * adapter that could not establish what the broker did is required to say so through the completion + * and evidence, which is the whole reason the ambiguity survives to the caller. + * + * @param result the outcome and its evidence + */ +public record TransportPublishResult(PublishResult result) { + + public TransportPublishResult { + Objects.requireNonNull(result, "result must not be null"); + } +} diff --git a/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportSettlement.java b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportSettlement.java new file mode 100644 index 00000000..dd39bc1e --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/TransportSettlement.java @@ -0,0 +1,36 @@ +package dev.caskeleton.messaging.transport; + +import dev.caskeleton.messaging.api.settlement.SettlementResult; +import java.time.Duration; +import java.util.concurrent.CompletionStage; + +/** + * The adapter-side handle for settling one delivery. + * + *

Deliberately not exposed to application code. Handlers state an intent; the platform decides + * when and in what order the settlement happens, and this is the seam it uses to do that. + */ +public interface TransportSettlement { + + /** + * Acknowledges the delivery. + * + * @return a stage completing with the settlement outcome + */ + CompletionStage acknowledge(); + + /** + * Returns the delivery for redelivery after a delay. + * + * @param delay how long the broker should wait + * @return a stage completing with the settlement outcome + */ + CompletionStage requeue(Duration delay); + + /** + * Discards the delivery without requeueing. + * + * @return a stage completing with the settlement outcome + */ + CompletionStage discard(); +} diff --git a/src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/BackpressureAndShutdownTest.java b/src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/BackpressureAndShutdownTest.java new file mode 100644 index 00000000..4f89db7a --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/BackpressureAndShutdownTest.java @@ -0,0 +1,108 @@ +package dev.caskeleton.messaging.transport; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Duration; +import java.time.Instant; +import org.junit.jupiter.api.Test; + +class BackpressureAndShutdownTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + @Test + void aSlowDestinationCannotConsumeEveryWorker() { + BackpressureController controller = new BackpressureController(10, 2); + + assertThat(controller.tryAcquire("order-events")).isTrue(); + assertThat(controller.tryAcquire("order-events")).isTrue(); + assertThat(controller.tryAcquire("order-events")).isFalse(); + assertThat(controller.tryAcquire("email-work")).isTrue(); + } + + @Test + void theGlobalLimitBoundsTheSumOfWellBehavedDestinations() { + BackpressureController controller = new BackpressureController(2, 2); + + assertThat(controller.tryAcquire("a")).isTrue(); + assertThat(controller.tryAcquire("b")).isTrue(); + assertThat(controller.tryAcquire("c")).isFalse(); + assertThat(controller.globalInFlight()).isEqualTo(2); + } + + @Test + void aRefusedAcquireDoesNotLeakAPerDestinationSlot() { + BackpressureController controller = new BackpressureController(1, 1); + controller.tryAcquire("a"); + + controller.tryAcquire("b"); + + assertThat(controller.inFlight("b")).isZero(); + } + + @Test + void releasingFreesBothCounters() { + BackpressureController controller = new BackpressureController(4, 2); + controller.tryAcquire("a"); + + controller.release("a"); + + assertThat(controller.inFlight("a")).isZero(); + assertThat(controller.globalInFlight()).isZero(); + } + + @Test + void aPerDestinationLimitAboveTheGlobalOneIsRejected() { + assertThatThrownBy(() -> new BackpressureController(2, 4)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void drainingStopsNewWorkButLetsRunningWorkFinish() { + GracefulShutdownCoordinator coordinator = + new GracefulShutdownCoordinator(Duration.ofSeconds(30)); + assertThat(coordinator.tryBeginWork()).isTrue(); + + coordinator.beginDrain(NOW); + + assertThat(coordinator.isAcceptingWork()).isFalse(); + assertThat(coordinator.tryBeginWork()).isFalse(); + assertThat(coordinator.inFlight()).isEqualTo(1); + assertThat(coordinator.isDrained(NOW)).isFalse(); + + coordinator.endWork(); + + assertThat(coordinator.isDrained(NOW)).isTrue(); + } + + @Test + void noRetryAttemptIsCreatedOnceDrainingBegins() { + GracefulShutdownCoordinator coordinator = + new GracefulShutdownCoordinator(Duration.ofSeconds(30)); + + assertThat(coordinator.mayCreateRetryAttempt()).isTrue(); + coordinator.beginDrain(NOW); + assertThat(coordinator.mayCreateRetryAttempt()).isFalse(); + } + + @Test + void workStillRunningAtTheDeadlineIsAbandonedUnsettled() { + GracefulShutdownCoordinator coordinator = + new GracefulShutdownCoordinator(Duration.ofSeconds(30)); + coordinator.tryBeginWork(); + coordinator.beginDrain(NOW); + + assertThat(coordinator.isDrained(NOW.plusSeconds(29))).isFalse(); + assertThat(coordinator.isDrained(NOW.plusSeconds(30))).isTrue(); + assertThat(coordinator.abandonedWorkAtDeadline(NOW.plusSeconds(30))).isTrue(); + } + + @Test + void anIdleCoordinatorIsNotDrainedBeforeShutdownStarts() { + GracefulShutdownCoordinator coordinator = + new GracefulShutdownCoordinator(Duration.ofSeconds(30)); + + assertThat(coordinator.isDrained(NOW)).isFalse(); + } +} diff --git a/src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/MessagingLifecycleTest.java b/src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/MessagingLifecycleTest.java new file mode 100644 index 00000000..beabf828 --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/MessagingLifecycleTest.java @@ -0,0 +1,59 @@ +package dev.caskeleton.messaging.transport; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.transport.MessagingLifecycle.ShutdownPhase; +import java.time.Duration; +import java.util.List; +import org.junit.jupiter.api.Test; + +class MessagingLifecycleTest { + + @Test + void publishAdmissionClosesBeforeConsumersArePaused() { + List order = List.of(ShutdownPhase.values()); + + assertThat(order.indexOf(ShutdownPhase.STOP_PUBLISH_ADMISSION)) + .isLessThan(order.indexOf(ShutdownPhase.PAUSE_CONSUMERS)); + } + + @Test + void handlersDrainBeforeTheirSettlementsAreFlushed() { + List order = List.of(ShutdownPhase.values()); + + assertThat(order.indexOf(ShutdownPhase.DRAIN_HANDLERS)) + .as("flushing before the handlers finish would lose the settlements they produce") + .isLessThan(order.indexOf(ShutdownPhase.FLUSH_SETTLEMENTS)); + } + + @Test + void connectionsCloseLastOfAll() { + ShutdownPhase[] order = ShutdownPhase.values(); + + assertThat(order[order.length - 1]) + .as("every earlier phase needs the connection that this one closes") + .isEqualTo(ShutdownPhase.CLOSE_CONNECTIONS); + } + + @Test + void producerConfirmsAreAwaitedBeforeTheConnectionGoesAway() { + List order = List.of(ShutdownPhase.values()); + + assertThat(order.indexOf(ShutdownPhase.AWAIT_PRODUCER_CONFIRMS)) + .as("a confirm that arrives after close cannot be observed, leaving the publish ambiguous") + .isLessThan(order.indexOf(ShutdownPhase.CLOSE_CONNECTIONS)); + } + + @Test + void outboxLeasesAreReleasedBeforeClosing() { + List order = List.of(ShutdownPhase.values()); + + assertThat(order.indexOf(ShutdownPhase.RELEASE_OUTBOX_LEASES)) + .isLessThan(order.indexOf(ShutdownPhase.CLOSE_CONNECTIONS)); + } + + @Test + void theDefaultDrainDeadlineMatchesTheDesign() { + assertThat(MessagingLifecycle.DEFAULT_DRAIN_DEADLINE).isEqualTo(Duration.ofSeconds(30)); + } +} diff --git a/src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/MessagingRuntimeRegistryTest.java b/src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/MessagingRuntimeRegistryTest.java new file mode 100644 index 00000000..b054edc6 --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/MessagingRuntimeRegistryTest.java @@ -0,0 +1,188 @@ +package dev.caskeleton.messaging.transport; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.destination.DestinationCapabilities; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.destination.MessagingCapabilities; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.CompletionStage; +import org.junit.jupiter.api.Test; + +class MessagingRuntimeRegistryTest { + + @Test + void replacingRuntimeReturnsNewGenerationAndKeepsOldUntilReleased() { + DefaultMessagingRuntimeRegistry registry = new DefaultMessagingRuntimeRegistry(); + MessagingRuntime first = MessagingRuntimeFixtures.runtime("kafka-primary", 1); + MessagingRuntime second = MessagingRuntimeFixtures.runtime("kafka-primary", 2); + + registry.install(first); + MessagingRuntimeLease lease = registry.acquire("kafka-primary"); + registry.install(second); + + assertThat(lease.runtime().generation()).isEqualTo(1); + assertThat(first.isClosed()).as("a leased generation stays open").isFalse(); + assertThat(registry.acquire("kafka-primary").runtime().generation()).isEqualTo(2); + + lease.close(); + + assertThat(first.isClosed()).isTrue(); + } + + @Test + void anIdleGenerationIsClosedImmediatelyOnReplacement() { + DefaultMessagingRuntimeRegistry registry = new DefaultMessagingRuntimeRegistry(); + MessagingRuntime first = MessagingRuntimeFixtures.runtime("kafka-primary", 1); + + registry.install(first); + registry.install(MessagingRuntimeFixtures.runtime("kafka-primary", 2)); + + assertThat(first.isClosed()).isTrue(); + assertThat(registry.drainingCount()).isZero(); + } + + @Test + void closingALeaseTwiceDoesNotDoubleRelease() { + DefaultMessagingRuntimeRegistry registry = new DefaultMessagingRuntimeRegistry(); + MessagingRuntime first = MessagingRuntimeFixtures.runtime("kafka-primary", 1); + registry.install(first); + + MessagingRuntimeLease lease = registry.acquire("kafka-primary"); + lease.close(); + lease.close(); + + assertThat(first.isClosed()).as("the current generation is not retired").isFalse(); + } + + @Test + void theDrainDeadlineForceClosesALeakedLease() { + DefaultMessagingRuntimeRegistry registry = + new DefaultMessagingRuntimeRegistry(Duration.ofSeconds(30)); + MessagingRuntime first = MessagingRuntimeFixtures.runtime("kafka-primary", 1); + + registry.install(first); + MessagingRuntimeLease leaked = registry.acquire("kafka-primary"); + registry.install(MessagingRuntimeFixtures.runtime("kafka-primary", 2)); + + Instant retiredAt = Instant.parse("2026-08-10T00:00:00Z"); + assertThat(registry.closeExpiredDraining(retiredAt.plusSeconds(29), retiredAt)).isZero(); + assertThat(first.isClosed()).isFalse(); + + assertThat(registry.closeExpiredDraining(retiredAt.plusSeconds(30), retiredAt)).isEqualTo(1); + assertThat(first.isClosed()).isTrue(); + + leaked.close(); + } + + @Test + void acquiringAnUninstalledBrokerIsAConfigurationFailure() { + DefaultMessagingRuntimeRegistry registry = new DefaultMessagingRuntimeRegistry(); + + assertThatThrownBy(() -> registry.acquire("rabbit-primary")) + .isInstanceOf(MessagingConfigurationException.class); + } + + @Test + void generationsAreTrackedPerBroker() { + DefaultMessagingRuntimeRegistry registry = new DefaultMessagingRuntimeRegistry(); + registry.install(MessagingRuntimeFixtures.runtime("kafka-primary", 7)); + registry.install(MessagingRuntimeFixtures.runtime("rabbit-primary", 3)); + + assertThat(registry.acquire("kafka-primary").runtime().generation()).isEqualTo(7); + assertThat(registry.acquire("rabbit-primary").runtime().generation()).isEqualTo(3); + } +} + +/** Builds runtimes that record whether they were closed. */ +final class MessagingRuntimeFixtures { + + private MessagingRuntimeFixtures() {} + + static MessagingRuntime runtime(String brokerName, long generation) { + return new FakeRuntime(brokerName, generation); + } +} + +/** A runtime that closes nothing but remembers that it was asked to. */ +final class FakeRuntime implements MessagingRuntime { + + private final String brokerName; + private final long generation; + private boolean closed; + + FakeRuntime(String brokerName, long generation) { + this.brokerName = brokerName; + this.generation = generation; + } + + @Override + public String brokerName() { + return brokerName; + } + + @Override + public long generation() { + return generation; + } + + @Override + public MessagingTransport transport() { + return new FakeTransport(brokerName, generation); + } + + @Override + public boolean isClosed() { + return closed; + } + + @Override + public void close() { + closed = true; + } +} + +/** A transport that refuses every operation; the registry test never dispatches through it. */ +final class FakeTransport implements MessagingTransport { + + private final String brokerName; + private final long generation; + + FakeTransport(String brokerName, long generation) { + this.brokerName = brokerName; + this.generation = generation; + } + + @Override + public CompletionStage publish(TransportPublishRequest request) { + throw new UnsupportedOperationException("fixture transport does not publish"); + } + + @Override + public TransportConsumerRegistration register(TransportConsumerSpec spec) { + throw new UnsupportedOperationException("fixture transport does not consume"); + } + + @Override + public DestinationCapabilities capabilities(DestinationName destination) { + return new DestinationCapabilities(destination, brokerName, MessagingCapabilities.none()); + } + + @Override + public String brokerName() { + return brokerName; + } + + @Override + public long generation() { + return generation; + } + + @Override + public void close() { + // Nothing to release. + } +} diff --git a/src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/ResourceLeakGateTest.java b/src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/ResourceLeakGateTest.java new file mode 100644 index 00000000..93f881c6 --- /dev/null +++ b/src/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/ResourceLeakGateTest.java @@ -0,0 +1,166 @@ +package dev.caskeleton.messaging.transport; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * Pins the release paths that keep a long-running consumer from leaking. + * + *

Every resource the platform holds is bounded by something that must eventually release it: a + * runtime generation by its last lease, an in-flight slot by its handler finishing, a drain by its + * deadline. Each of those has a failure mode that is invisible in a short test and fatal over days + * — a retired generation whose credential never gets revoked, a partition that never accepts work + * again, a shutdown that never completes. + */ +class ResourceLeakGateTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + @Test + void everyRetiredGenerationIsEventuallyClosed() { + DefaultMessagingRuntimeRegistry registry = + new DefaultMessagingRuntimeRegistry(Duration.ofSeconds(30)); + List generations = new ArrayList<>(); + + for (int generation = 1; generation <= 20; generation++) { + LeakTrackingRuntime runtime = new LeakTrackingRuntime("kafka-primary", generation); + generations.add(runtime); + registry.install(runtime); + registry.acquire("kafka-primary").close(); + } + + assertThat(generations.subList(0, generations.size() - 1)) + .as("only the current generation stays open") + .allSatisfy(runtime -> assertThat(runtime.isClosed()).isTrue()); + assertThat(registry.drainingCount()).isZero(); + } + + @Test + void aLeakedLeaseIsForceClosedAtTheDrainDeadline() { + DefaultMessagingRuntimeRegistry registry = + new DefaultMessagingRuntimeRegistry(Duration.ofSeconds(30)); + LeakTrackingRuntime retired = new LeakTrackingRuntime("kafka-primary", 1); + registry.install(retired); + registry.acquire("kafka-primary"); + + registry.install(new LeakTrackingRuntime("kafka-primary", 2)); + registry.closeExpiredDraining(NOW.plusSeconds(30), NOW); + + assertThat(retired.isClosed()) + .as("a leaked lease must not pin a revoked credential open forever") + .isTrue(); + } + + @Test + void everyAcquiredInFlightSlotIsReleased() { + BackpressureController controller = new BackpressureController(8, 4); + + for (int i = 0; i < 1_000; i++) { + if (controller.tryAcquire("order-events")) { + controller.release("order-events"); + } + } + + assertThat(controller.inFlight("order-events")).isZero(); + assertThat(controller.globalInFlight()).isZero(); + } + + @Test + void aRefusedAcquireLeavesNoSlotBehind() { + BackpressureController controller = new BackpressureController(2, 1); + controller.tryAcquire("a"); + controller.tryAcquire("b"); + + for (int i = 0; i < 100; i++) { + controller.tryAcquire("c"); + } + + assertThat(controller.inFlight("c")).isZero(); + assertThat(controller.globalInFlight()).isEqualTo(2); + } + + @Test + void aDrainAlwaysTerminatesEvenWithStuckWork() { + GracefulShutdownCoordinator coordinator = + new GracefulShutdownCoordinator(Duration.ofSeconds(30)); + coordinator.tryBeginWork(); + coordinator.beginDrain(NOW); + + assertThat(coordinator.isDrained(NOW.plusSeconds(3600))) + .as("a stuck handler must not hold the process open indefinitely") + .isTrue(); + } + + @Test + void aRetiredGenerationIsClosedExactlyOnce() { + DefaultMessagingRuntimeRegistry registry = + new DefaultMessagingRuntimeRegistry(Duration.ofSeconds(30)); + LeakTrackingRuntime retired = new LeakTrackingRuntime("kafka-primary", 1); + registry.install(retired); + MessagingRuntimeLease lease = registry.acquire("kafka-primary"); + registry.install(new LeakTrackingRuntime("kafka-primary", 2)); + + lease.close(); + lease.close(); + registry.closeExpiredDraining(NOW.plusSeconds(60), NOW); + + assertThat(retired.closeCount()) + .as("a second close on a real connection pool throws from a shutdown hook") + .isEqualTo(1); + } +} + +/** A runtime that records whether it was closed, and how many times. */ +final class LeakTrackingRuntime implements MessagingRuntime { + + private final String brokerName; + private final long generation; + private int closeCount; + + LeakTrackingRuntime(String brokerName, long generation) { + this.brokerName = brokerName; + this.generation = generation; + } + + @Override + public String brokerName() { + return brokerName; + } + + @Override + public long generation() { + return generation; + } + + @Override + public MessagingTransport transport() { + throw new UnsupportedOperationException("the leak gate never dispatches"); + } + + @Override + public boolean isClosed() { + return closeCount > 0; + } + + /** + * Returns how many times close was called. + * + *

Closing twice is as much a defect as never closing: a second close on a real connection pool + * throws, and the exception surfaces from a shutdown hook where nothing handles it. + * + * @return the close count + */ + int closeCount() { + return closeCount; + } + + @Override + public void close() { + closeCount++; + } +} diff --git a/src/settings.gradle b/src/settings.gradle index d69e1d66..6943a6e1 100644 --- a/src/settings.gradle +++ b/src/settings.gradle @@ -34,7 +34,7 @@ if (!(moduleRegistry.runtime_compositions instanceof List) || moduleRegistryFile) } -int expectedModuleCount = 19 +int expectedModuleCount = 43 if (moduleRegistry.modules.size() != expectedModuleCount) { throw new GradleException( "Module registry must contain exactly ${expectedModuleCount} modules, " +