From a261df2a2bcb1c5ed5b60e4a40e9dd6d4ffc1b83 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Mon, 24 Aug 2026 18:26:40 +0900 Subject: [PATCH] =?UTF-8?q?refactor:=20=EA=B0=81=20=EC=96=B4=EB=8C=91?= =?UTF-8?q?=ED=84=B0=ED=84=B0=EB=B3=84=20=EB=A6=AC=ED=8E=99=ED=86=A0?= =?UTF-8?q?=EB=A7=81=20=EC=A7=84=ED=96=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/messaging-certification.yml | 59 +++ docs/messaging/configuration-reference.md | 235 +++++---- docs/messaging/support-matrix.md | 24 +- messaging/CLAUDE.md | 63 ++- .../messaging/api/CorrelationId.java | 15 +- .../messaging/api/MessageEnvelope.java | 35 ++ .../caskeleton/messaging/api/MessageType.java | 33 +- .../caskeleton/messaging/api/ProducerId.java | 33 +- .../messaging/api/TraceContext.java | 111 +++++ .../messaging/api/WireSafeText.java | 65 +++ .../api/header/CanonicalEnvelopeHeaders.java | 79 +++ .../messaging/api/header/HeaderValue.java | 15 +- .../messaging/api/header/MessageHeaders.java | 49 +- .../messaging/api/header/ReservedHeaders.java | 27 +- .../messaging/api/publish/PublishOptions.java | 30 +- .../messaging/api/publish/PublishResult.java | 34 +- .../api/WireBoundaryRejectionTest.java | 208 ++++++++ .../api/publish/PublishResultTest.java | 86 +++- messaging/messaging-kafka/build.gradle | 97 ++++ .../messaging/kafka/KafkaDeliveryMapper.java | 6 +- .../messaging/kafka/KafkaHeaderMapper.java | 57 ++- .../kafka/KafkaBrokerCertificationIT.java | 463 ++++++++++++++++++ .../kafka/KafkaEnvelopeRoundTripTest.java | 154 ++++++ .../kafka/ReservedHeaderForgeryTest.java | 2 +- .../nats/NatsAdapterContractTest.java | 3 +- .../messaging-observability/build.gradle | 19 - .../outbox/DebeziumOutboxEventRouter.java | 81 ++- .../outbox/DebeziumOutboxRecordMapper.java | 30 +- .../outbox/JdbcOutboxRepository.java | 88 +++- .../outbox/OutboxEnvelopeFactory.java | 75 ++- .../messaging/outbox/OutboxRelay.java | 56 ++- .../messaging/outbox/OutboxRelayReport.java | 22 +- .../messaging/outbox/OutboxRelayWorker.java | 199 ++++++++ .../outbox/OutboxRetryScheduler.java | 29 +- ...4__messaging_outbox_canonical_metadata.sql | 66 +++ .../DebeziumOutboxRecordMapperTest.java | 67 ++- .../JdbcOutboxTransactionRequirementTest.java | 1 + .../outbox/OutboxEnvelopeFactoryTest.java | 158 ++++++ .../messaging/outbox/OutboxPostgresIT.java | 159 +++++- .../messaging/outbox/OutboxRelayTest.java | 60 +++ .../outbox/OutboxRelayWorkerTest.java | 163 ++++++ .../policy/MessagingAdmissionController.java | 105 +++- .../MessagingAdmissionControllerTest.java | 83 +++- .../pulsar/PulsarAdapterContractTest.java | 3 +- .../rabbit/RabbitDeliveryMapper.java | 6 +- .../messaging/rabbit/RabbitHeaderMapper.java | 49 +- .../rabbit/RabbitEnvelopeRoundTripTest.java | 152 ++++++ .../reliability/OutboxCanonicalMetadata.java | 158 ++++++ .../messaging/reliability/OutboxRecord.java | 11 +- .../runtime/DefaultMessagePublisher.java | 103 +++- .../runtime/DefaultMessagePublisherTest.java | 100 +++- messaging/messaging-schema-avro/build.gradle | 5 +- .../schema/avro/AvroMessageCodec.java | 42 +- .../schema/avro/AvroHostileInputTest.java | 134 +++++ .../messaging-schema-protobuf/build.gradle | 4 +- .../protobuf/ProtobufCompatibilityTest.java | 14 + .../build.gradle | 11 +- .../gradle.lockfile | 1 + .../autoconfigure/BrokerSecuritySettings.java | 50 ++ .../autoconfigure/BrokerSettings.java | 69 +++ .../CompiledMessagingConfiguration.java | 53 ++ .../autoconfigure/DestinationSettings.java | 177 +++++++ .../KafkaMessagingAutoConfiguration.java | 23 + .../MessagingConfigurationCompiler.java | 331 +++++++++++++ .../MessagingConfigurationKeyValidator.java | 227 +++++++++ .../MessagingCoreAutoConfiguration.java | 118 ++++- ...ssagingCredentialRequirementValidator.java | 21 +- .../autoconfigure/MessagingEndpoint.java | 18 +- .../MessagingOutboxRelayLifecycle.java | 69 +++ .../MessagingProviderSelection.java | 42 ++ ...MessagingReliabilityAutoConfiguration.java | 72 +++ .../autoconfigure/MessagingSettings.java | 70 +++ .../MessagingShutdownLifecycle.java | 124 +++++ .../RabbitMessagingAutoConfiguration.java | 24 + .../StartupProfileValidation.java | 46 ++ .../autoconfigure/BlockingFacadeTest.java | 3 +- .../MessagingAutoConfigurationTest.java | 57 ++- .../MessagingConfigurationBindingTest.java | 320 ++++++++++++ .../autoconfigure/MessagingEndpointTest.java | 33 +- ...ssagingLiveRoundTripQualificationTest.java | 6 +- .../MessagingOutboxRelayLifecycleTest.java | 203 ++++++++ .../MessagingShutdownLifecycleTest.java | 78 +++ .../MessagingStarterOffContractTest.java | 35 +- .../testkit/BrokerCertificationEvidence.java | 128 ++++- .../testkit/BrokerFailureMatrix.java | 71 ++- .../messaging/testkit/CertifiedEvidence.java | 84 ++++ .../testkit/CompatibilityMatrix.java | 43 +- .../testkit/NetworkFaultScenario.java | 24 + .../broker-certification-evidence.jsonl | 4 + .../messaging/testkit/CertifiedEvidence.java | 72 --- .../testkit/CertifiedEvidenceTest.java | 127 +++++ .../testkit/CompatibilityMatrixTest.java | 32 +- .../testkit/CrossBrokerContractSuite.java | 35 +- .../MessagingPublicSurfaceContractTest.java | 350 +++++++++++++ .../transport/BackpressureController.java | 125 ----- .../DefaultMessagingRuntimeRegistry.java | 56 ++- .../BackpressureAndShutdownTest.java | 148 ------ .../transport/GracefulShutdownTest.java | 75 +++ .../MessagingRuntimeRegistryTest.java | 66 +++ .../transport/ResourceLeakGateTest.java | 28 -- 100 files changed, 7072 insertions(+), 902 deletions(-) create mode 100644 .github/workflows/messaging-certification.yml create mode 100644 messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/WireSafeText.java create mode 100644 messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/CanonicalEnvelopeHeaders.java create mode 100644 messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/WireBoundaryRejectionTest.java create mode 100644 messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaBrokerCertificationIT.java create mode 100644 messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaEnvelopeRoundTripTest.java create mode 100644 messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelayWorker.java create mode 100644 messaging/messaging-outbox-jdbc-postgresql/src/main/resources/db/migration/messaging/V4__messaging_outbox_canonical_metadata.sql create mode 100644 messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxEnvelopeFactoryTest.java create mode 100644 messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxRelayWorkerTest.java create mode 100644 messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitEnvelopeRoundTripTest.java create mode 100644 messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxCanonicalMetadata.java create mode 100644 messaging/messaging-schema-avro/src/test/java/dev/caskeleton/messaging/schema/avro/AvroHostileInputTest.java create mode 100644 messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/BrokerSecuritySettings.java create mode 100644 messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/BrokerSettings.java create mode 100644 messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/CompiledMessagingConfiguration.java create mode 100644 messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DestinationSettings.java create mode 100644 messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingConfigurationCompiler.java create mode 100644 messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingConfigurationKeyValidator.java create mode 100644 messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingOutboxRelayLifecycle.java create mode 100644 messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingShutdownLifecycle.java create mode 100644 messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/StartupProfileValidation.java create mode 100644 messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingConfigurationBindingTest.java create mode 100644 messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingOutboxRelayLifecycleTest.java create mode 100644 messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingShutdownLifecycleTest.java create mode 100644 messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/CertifiedEvidence.java create mode 100644 messaging/messaging-testkit/src/main/resources/messaging/broker-certification-evidence.jsonl delete mode 100644 messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CertifiedEvidence.java create mode 100644 messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CertifiedEvidenceTest.java create mode 100644 messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/contract/MessagingPublicSurfaceContractTest.java delete mode 100644 messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/BackpressureController.java delete mode 100644 messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/BackpressureAndShutdownTest.java create mode 100644 messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/GracefulShutdownTest.java diff --git a/.github/workflows/messaging-certification.yml b/.github/workflows/messaging-certification.yml new file mode 100644 index 0000000..f90d6e4 --- /dev/null +++ b/.github/workflows/messaging-certification.yml @@ -0,0 +1,59 @@ +# The messaging platform's broker certification lane. +# +# Separate from ci-quality-gates.yml because it needs a container runtime and several minutes of it. +# The lane deliberately carries no Docker guard: every other container suite in the messaging tree +# skips with a stated reason when Docker is absent, and a certification lane that skipped would +# report success for a broker nobody started — which is the exact claim the evidence exists to rule +# out. +# +# The job runs the evidence gate rather than the lane, and the gate depends on the lane. What it +# proves is not only that the scenarios pass but that the committed manifest +# (messaging-testkit/src/main/resources/messaging/broker-certification-evidence.jsonl) is what this +# run produced, so "certified against a live broker" cannot be restored by editing a file. +name: messaging-certification + +on: + pull_request: + paths: + - "src/messaging/**" + - ".github/workflows/messaging-certification.yml" + schedule: + - cron: "41 4 * * 3" + workflow_dispatch: + +permissions: + contents: read + +env: + # Reuse would hand one scenario the broker another scenario had already faulted. + TESTCONTAINERS_REUSE_ENABLE: "false" + +jobs: + broker-certification: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # actions/checkout@v4.2.2 + - name: Validate Gradle wrapper + id: gradle-wrapper-validation + uses: gradle/actions/wrapper-validation@3f131e8634966bd73d06cc69884922b02e6faf92 # gradle/actions@v6 + - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # actions/setup-java@v4.7.1 + with: + distribution: temurin + java-version: "21.0.11+10" + cache: gradle + cache-dependency-path: | + src/**/*.gradle + src/**/gradle-wrapper.properties + src/**/gradle.lockfile + - name: Certify the Kafka adapter against a real broker + working-directory: src + # GITHUB_SHA is read by the lane and written into every evidence line, because "certified" + # is a claim about one source tree. + run: ./gradlew :messaging:messaging-kafka:verifyMessagingCertificationEvidence --no-daemon --stacktrace + - name: Publish the certification evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4.6.2 + with: + name: messaging-broker-certification-evidence + path: src/messaging/messaging-kafka/build/messaging-certification/ + if-no-files-found: warn diff --git a/docs/messaging/configuration-reference.md b/docs/messaging/configuration-reference.md index 0e864ee..31afe67 100644 --- a/docs/messaging/configuration-reference.md +++ b/docs/messaging/configuration-reference.md @@ -7,60 +7,76 @@ > either of the old prefixes now fails startup with a message naming the key — see > `MessagingPrefixMigrationValidator`. +> **이 페이지는 실행된다.** 아래 YAML 블록은 `MessagingConfigurationBindingTest`가 이 파일에서 직접 +> 읽어 컨텍스트에 올린다. 문서가 설명하는 모양이 곧 바인딩되는 모양이라는 뜻이고, 문서를 고치면서 +> 코드를 고치지 않으면 테스트가 깨진다. 이전 판은 destination·broker·security 세 섹션을 설명했지만 +> 어떤 binder도 그것을 읽지 않았다 — 문서대로 설정한 배포는 아무것도 바뀌지 않았고 아무 말도 듣지 +> 못했다 (MSG-008). ## Destination profile ```yaml app: messaging: - destinations: - order-events: - broker: kafka-primary - kind: EVENT_STREAM # ASYNC_COMMAND | DOMAIN_EVENT | INTEGRATION_EVENT + destinations: + order-events: + broker: kafka-primary + kind: EVENT_STREAM # ASYNC_COMMAND | DOMAIN_EVENT | INTEGRATION_EVENT # | WORK_QUEUE | PUBLISH_SUBSCRIBE | EVENT_STREAM | REQUEST_REPLY - tier: M1 # M1 | M2 | M3 - physical: - topic: order.events.v1 - schema: - codec: application/json - compatibility: BACKWARD_TRANSITIVE - message-types: [order.created] - guarantees: - delivery: AT_LEAST_ONCE # AT_MOST_ONCE | AT_LEAST_ONCE - ordering: KEY # NONE | DESTINATION | PARTITION | KEY - external-side-effect: INBOX_TRANSACTIONAL - producer: - confirmation: REPLICATION_OR_PERSISTENCE_ACK - timeout: 5s - mandatory-routing: true - idempotent: true - consumer: - group: order-projection - concurrency: 6 - max-in-flight-per-ordering-unit: 1 - prefetch: 16 - handler-timeout: 30s - manual-settlement: false - retry: - mode: PAUSE_PARTITION # NONE | INLINE | BLOCKING | PAUSE_PARTITION + tier: M1 # M1 | M2 | M3 + physical: + topic: order.events.v1 + schema: + codec: application/json + compatibility: BACKWARD_TRANSITIVE + message-types: [order.created] + guarantees: + delivery: AT_LEAST_ONCE # AT_MOST_ONCE | AT_LEAST_ONCE + ordering: KEY # NONE | DESTINATION | PARTITION | KEY + external-side-effect: INBOX_TRANSACTIONAL + producer: + confirmation: REPLICATION_OR_PERSISTENCE_ACK + timeout: 5s + mandatory-routing: true + idempotent: true + consumer: + group: order-projection + concurrency: 1 # DESTINATION 순서를 요구하면 1이어야 한다 + max-in-flight-per-ordering-unit: 1 + prefetch: 16 + handler-timeout: 30s + manual-settlement: false + retry: + mode: PAUSE_PARTITION # NONE | INLINE | BLOCKING | PAUSE_PARTITION # | RETRY_DESTINATION | BROKER_DELAYED - max-attempts: 3 - initial-delay: 200ms - max-delay: 2s - multiplier: 2.0 - jitter: true - ordering-impact: PRESERVE # PRESERVE | ALLOW_REORDER - dlq: - destination: order-events-dlq - max-redrive-count: 1 - payload: - max-bytes: 1048576 - claim-check-threshold-bytes: 1048576 - key-resolver-configured: true - production: true - topology-auto-create: false + max-attempts: 3 + initial-delay: 200ms + max-delay: 2s + multiplier: 2.0 + jitter: true + ordering-impact: PRESERVE # PRESERVE | ALLOW_REORDER + dlq: + destination: order-events-dlq + max-redrive-count: 1 + payload: + max-bytes: 1048576 + claim-check-threshold-bytes: 1048576 + key-resolver-configured: true + production: false + topology-auto-create: false + order-events-dlq: + broker: kafka-primary + kind: WORK_QUEUE + physical: + topic: order.events.v1.dlt + schema: + message-types: [order.created] ``` +`dlq.destination`이 가리키는 destination도 선언되어야 한다. 선언되지 않은 이름은 부팅 실패이며, +메시지가 갈 곳 없는 DLQ 설정이 조용히 통과하지 않는다. `retry.destination`과 `dlq.destination`이 +섞여 만드는 순환(A의 retry가 B로, B의 dlq가 A로)도 하나의 그래프로 검사되어 경로와 함께 거절된다. + ## 기본값 | 설정 | 기본값 | 근거 | @@ -81,26 +97,36 @@ app: | Outbox polling | 500ms | | | metric dimension 상한 | 200 | cardinality 폭발 방지 | +`schema.codec`은 `application/json`, `schema.compatibility`는 `BACKWARD_TRANSITIVE`, +`guarantees.delivery`는 `AT_LEAST_ONCE`, `retry.mode`는 `NONE`이 기본값이다. 자동 retry가 기본으로 +꺼져 있는 이유는 순서를 흐트러뜨리거나 비멱등 side effect를 두 번 실행하는 retry가 눈에 보이는 +실패보다 나쁘기 때문이다. + ## Broker profile +브로커는 `app.messaging.brokers` 아래에 한 번만 기술한다. `type`이 어느 계열의 설정이 적용되는지 +결정하며, 다른 계열의 키(Kafka 항목의 `prefetch` 같은)는 무시되지 않고 부팅 실패로 거절된다 — +무시하면 그 줄을 쓴 사람은 무언가가 적용됐다고 믿게 된다. + ### Kafka ```yaml app: messaging: - brokers: - kafka-primary: - type: kafka - stable: true - production: true - bootstrap-servers: [broker-1:9093, broker-2:9093] - enable-idempotence: true # stable에서 필수 - acks: all # stable에서 필수 - max-in-flight-requests-per-connection: 5 # 최대 5 - delivery-timeout: 30s - enable-auto-commit: false # 항상 금지 - tls-enabled: true # production 필수 - authentication-enabled: true # production 필수 + brokers: + kafka-primary: + type: kafka + stable: true + production: false + bootstrap-servers: [broker-1:9093, broker-2:9093] + enable-idempotence: true # stable에서 필수 + acks: all # stable에서 필수 + max-in-flight-requests-per-connection: 5 # 최대 5 + delivery-timeout: 30s + enable-auto-commit: false # 항상 금지 + consumer-group: order-projection + tls-enabled: false # production이면 필수 + authentication-enabled: false # production이면 필수 ``` ### RabbitMQ @@ -108,40 +134,52 @@ app: ```yaml app: messaging: - brokers: - rabbit-primary: - type: rabbitmq - stable: true - production: true - addresses: [rabbit-1:5671] - publisher-confirms: true # stable에서 필수 - publisher-returns: true # stable에서 필수 - mandatory: true # stable에서 필수 - confirm-timeout: 5s - auto-ack: false # 항상 금지 - prefetch: 16 - quorum-queues: true # durable work queue 필수 - tls-enabled: true - authentication-enabled: true + brokers: + rabbit-primary: + type: rabbitmq + stable: true + production: false + addresses: [rabbit-1:5671] + publisher-confirms: true # stable에서 필수 + publisher-returns: true # stable에서 필수 + mandatory: true # stable에서 필수 + confirm-timeout: 5s + auto-ack: false # 항상 금지 + prefetch: 16 + quorum-queues: true # durable work queue 필수 + tls-enabled: false + authentication-enabled: false ``` +`production: true`인 브로커는 `tls-enabled`와 `authentication-enabled`가 모두 참이어야 하고, +그렇지 않으면 `KafkaProfileValidator` / `RabbitProfileValidator`가 부팅을 거절한다. 위 예시가 +`production: false`인 것은 이 페이지가 그대로 실행되는 fixture이기 때문이며, 실 배포는 셋 다 참이다. + ## 보안 ```yaml app: messaging: - security: - kafka-primary: - producer: { type: SASL_SCRAM, credential-id: kafka-producer } - consumer: { type: SASL_SCRAM, credential-id: kafka-consumer } - # admin은 application runtime에 설정하지 않는다 - hostname-verification: true - access: - publishable: [order-events] - consumable: [] - administrable: [] + security: + kafka-primary: + producer: { type: SASL_SCRAM, credential-id: kafka-producer } + consumer: { type: SASL_SCRAM, credential-id: kafka-consumer } + # admin은 application runtime에 설정하지 않는다 + hostname-verification: true + access: + publishable: [order-events] + consumable: [] + administrable: [] ``` +키는 `app.messaging.brokers`에 선언된 브로커 이름과 같아야 한다. `tls-enabled`와 `production`은 +브로커 쪽에만 있고 여기에 중복되지 않는다 — 하나의 브로커가 두 곳에서 기술되면 두 값이 어긋나는 +날이 오고, 어느 쪽이 이기는지는 아무도 모른다. + +`credential-id`는 이름일 뿐이고 자격 증명 자체가 아니다. 실제 재료는 `CredentialProvider`가 +연결 시점에 해석하므로, 설정 덤프나 힙 덤프에서 나오는 것은 이름뿐이다. producer와 consumer는 +서로 다른 `credential-id`를 써야 하며, 같으면 부팅에 실패한다. + ## Experimental / Optional 기본값은 전부 `false`다. @@ -149,12 +187,12 @@ app: ```yaml app: messaging: - experimental: - kafka-share: false - pulsar: false - nats: false - bridge: - spring-cloud-stream: false + experimental: + kafka-share: false + pulsar: false + nats: false + bridge: + spring-cloud-stream: false ``` ## Backpressure @@ -162,9 +200,24 @@ app: ```yaml app: messaging: - backpressure: - global-limit: 512 - per-destination-limit: 64 # global-limit 이하여야 한다 + backpressure: + global-limit: 512 + per-destination-limit: 64 # global-limit 이하여야 한다 ``` `per-destination-limit > global-limit`이면 global limit이 limit이 아니게 되므로 부팅에 실패한다. + +## 바인딩되지 않는 키 + +섹션은 바인딩되는데 그 안의 키 하나가 오타인 경우는 접두사 오타와 달리 조용하다 — 섹션은 붙고, +플랫폼은 뜨고, 바꾸러 온 그 설정만 적용되지 않는다. `MessagingConfigurationKeyValidator`가 +`app.messaging.destinations|brokers|security` 아래의 모든 키를 settings 레코드에서 파생한 목록과 +대조하고, 없는 키는 그 키 이름을 담아 부팅을 거절한다. + +허용 키 목록은 이 문서가 아니라 레코드에서 나온다. 문서에 목록을 적으면 필드가 추가된 날 그 +목록이 틀리고, 오타를 잡으라고 만든 검사가 정상 필드를 거절하게 된다. + +환경변수(`APP_MESSAGING_...`)는 이 검사의 대상이 아니다. `APP_MESSAGING_DESTINATIONS_ORDER_EVENTS_ +CONSUMER_PREFETCH`에서 entry 이름과 leaf를 가르는 밑줄은 둘 안에 있는 밑줄과 구별되지 않으므로, +되돌려 쪼개려면 추측해야 한다. 여기서의 추측은 정상 배포를 거절하는 쪽으로 틀리며, 그것은 배포 +매니페스트에 손으로 적어야 하는 변수에서 오타 하나를 놓치는 것보다 나쁘다. diff --git a/docs/messaging/support-matrix.md b/docs/messaging/support-matrix.md index 000cfca..38622f1 100644 --- a/docs/messaging/support-matrix.md +++ b/docs/messaging/support-matrix.md @@ -3,6 +3,12 @@ 플랫폼이 **무엇을 보장하는지**와 **무엇을 보장하지 않는지**를 브로커별로 고정한다. 여기 없는 조합은 지원되지 않는다. +> **등급은 증거를 따른다.** `CompatibilityMatrix.Entry.hasLiveBrokerCertification()`은 선언된 +> boolean이 아니라 `CertifiedEvidence`가 가진 레인 증거에서 파생된다. RabbitMQ가 Stable에서 내려온 +> 이유가 이것이다 — 어댑터는 공유 contract 7개를 통과하고 `RabbitBrokerIT`가 실 컨테이너에서 정상 +> 경로를 돌리지만, 이 저장소의 Stable 기준인 **장애 시나리오 증거**가 하나도 없다. 레인이 생겨 +> 증거를 내면 등급은 코드 수정 없이 따라 올라간다. + > **인증 근거.** 이 표의 버전은 이 저장소의 컨테이너 레인이 실제로 실행한 이미지다. 이전 판은 > Kafka 4.2/4.3을 선언했지만 fixture는 `apache/kafka:4.1.0`, lockfile client는 4.1.1이었다 — 표와 > 코드 상수가 서로 일치했을 뿐 어느 쪽도 실행된 적이 없었다. 장애 시나리오 커버리지도 마찬가지로 @@ -25,7 +31,7 @@ | 브로커 | 등급 | 인증 기준 | Stable 기능 | 제한 | |---|---|---|---|---| | Kafka | Stable | 4.1.x | producer idempotence, consumer group, batch, pause/resume, replay, transaction capability | Share Group은 Experimental | -| RabbitMQ | Stable | 4.3.x | exchange/routing, publisher confirm, mandatory return, manual ACK, quorum queue, retry queue, DLQ | stream 및 특수 plugin 미지원 | +| RabbitMQ | Experimental | 4.3.x | exchange/routing, publisher confirm, mandatory return, manual ACK, quorum queue, retry queue, DLQ | 장애 시나리오 레인 미실행 — 증거 없음. stream 및 특수 plugin 미지원 | | Pulsar | Experimental | 4.0 LTS + 4.2 | typed publish/consume, Shared, Key_Shared, schema | transaction 미승격, 기본 비활성 | | NATS JetStream | Experimental | 2.14.x | stream, durable consumer, explicit ACK, dedupe, replay | native DLQ 없음(플랫폼이 대행), 기본 비활성 | | Artemis/JMS | Extension | 범위 밖 | adapter SPI만 | 별도 ADR + Contract Suite 통과 필요 | @@ -86,11 +92,15 @@ Kafka와 RabbitMQ가 동일한 7개 테스트를 변경 없이 통과한다. 결 |---|---| | `KafkaBrokerIT` | `acks=all`이 실제 replication 증거를 만든다 / 잘못된 토픽은 `REJECTED` / 발행-소비 왕복에서 identity 보존 및 contiguous commit | | `KafkaAmbiguityChaosIT` | 브로커를 `docker pause`로 멈춘 상태의 publish가 **`AMBIGUOUS`** 로 보고된다 (broker acceptance 없음, confirmation level `NONE`, 비-retryable) | +| `KafkaBrokerCertificationIT` | 인증 레인. Toxiproxy를 broker 앞에 두고 connection cut / confirm 유실 / 지연 / settlement 유실을 각각 주입하고, 통과한 시나리오마다 `BrokerCertificationEvidence` 한 줄을 manifest에 쓴다 | | `RabbitBrokerIT` | exchange가 confirm했는데 어떤 큐에도 바인딩되지 않은 publish가 **`REJECTED` + `UNROUTABLE`** 로 보고된다 | | `OutboxPostgresIT` | 롤백된 트랜잭션은 발행 가능한 행을 남기지 않는다 / `SKIP LOCKED` lease가 두 relay를 분리한다 / ambiguous 행이 같은 `messageId`로 재클레임된다 | | `InboxPostgresIT` | 재전달이 side effect를 두 번 적용하지 않는다 / 롤백은 예약도 되돌린다 | Docker가 없으면 `DockerAvailability` 가드로 skip되며, 이 표의 항목은 그때 **검증되지 않은 것**으로 취급한다. +`KafkaBrokerCertificationIT`만 예외다 — 인증 레인은 가드를 달지 않고 Docker가 없으면 실패한다. skip하는 +레인은 아무도 켜지 않은 브로커에 대해 성공을 보고하기 때문이다. 그래서 이 레인은 `test`에서 태그로 +제외되고 `messagingCertificationTest`로만 실행된다. ### 3. 장애 시나리오 커버리지 (`BrokerFailureMatrix`) @@ -109,6 +119,18 @@ Docker가 없으면 `DockerAvailability` 가드로 skip되며, 이 표의 항목 커버해야 하고, Experimental 어댑터는 `LIVE_BROKER` 커버리지를 주장할 수 없다. 커버리지는 *능력*이 아니라 *무엇을 실제로 돌렸는지*의 기록이다. +**증거의 출처.** `CertifiedEvidence`는 더 이상 손으로 쓴 목록이 아니라 +`messaging-testkit/src/main/resources/messaging/broker-certification-evidence.jsonl`을 읽는다. 그 파일은 +`messagingCertificationTest` 레인이 실제 Kafka 컨테이너에 장애를 주입하며 만들어낸 출력이고, +`verifyMessagingCertificationEvidence`가 커밋된 manifest와 이번 실행의 출력을 대조해 다르면 빌드를 +실패시킨다. 즉 **manifest를 손으로 고치면 게이트가 깨지고, 레인을 돌리면 manifest가 다시 쓰인다.** + +오늘 Kafka가 가진 증거는 `connection-cut-after-write` · `confirm-timeout` · `high-latency` · +`settlement-lost` 네 개다. `connection-refused`는 남은 gap이며 그 이유가 있다 — Kafka producer는 연결 +존재 여부를 알기 전에 레코드를 버퍼에 넣으므로, 연결 거부는 전송에 대해 아무것도 증명하지 못하는 +delivery timeout으로 나타난다. 이를 `REJECTED`로 보고하는 것은 이 플랫폼이 금지한 추측이므로, +시나리오는 `CertifiedEvidence.knownGaps`가 이름으로 들고 있는 미커버 항목으로 남는다. + ### 실 브로커가 실제로 잡아낸 결함 이 스위트들은 장식이 아니다. 작성 과정에서 결정적 테스트가 통과하는데 실 인프라에서 실패한 diff --git a/messaging/CLAUDE.md b/messaging/CLAUDE.md index e25f9a8..0fd1942 100644 --- a/messaging/CLAUDE.md +++ b/messaging/CLAUDE.md @@ -31,25 +31,58 @@ leaf의 정확한 id는 registry에서 읽는다. family는 **디렉터리 이 - **vendor 타입을 public signature에 노출하면 그 의존성은 `api`로 선언한다.** 숨긴다고 타입이 숨겨지지 않는다 — consumer가 이 모듈이 문서화한 API에 대해 컴파일하지 못하게 만들 뿐이다. - (`kafka-clients`, `spring-rabbit`, `cloudevents-api`, `micrometer-core`가 이 규칙으로 `api`다.) + (`kafka-clients`, `spring-rabbit`, `cloudevents-api`, `micrometer-core`, `avro`, + `protobuf-java`, `reactor-core`가 이 규칙으로 `api`다.) - 반대로 **노출하지 않는 것은 `implementation`으로 낮춘다.** starter가 노출하지 않는 하위 모듈을 - `implementation`으로 내린 것이 그 예다. + `implementation`으로 내린 것이, JSON codec이 `ObjectMapper`를 private으로만 쓰고 Jackson을 + `implementation`으로 둔 것이 그 예다. +- 이 규칙은 문서가 아니라 `MessagingPublicSurfaceContractTest`가 붙들고 있다. leaf의 production + source에서 public/protected 시그니처에 등장하는 vendor 라이브러리를 뽑아 그 leaf의 + `build.gradle`이 `api`로 선언했는지 대조하고, starter의 `api` closure에 broker client 두 개가 + 들어오지 않는 것도 함께 본다. Spring Framework/Boot는 대상이 아니다 — 조립 경계 안에서만 + 등장하고 adopter가 이름 부를 일이 없다. - 새 public 타입은 그 모듈의 계약이다. 삭제·시그니처 변경은 breaking change로 취급한다. ## Runtime membership이 비어 있다는 것의 의미 -registry의 모든 messaging leaf는 `runtime_memberships`가 비어 있다. 이는 **build-only / -incubating**을 뜻한다: +**이 절은 한동안 사실이 아닌 채로 남아 있었다.** "registry의 모든 messaging leaf는 +`runtime_memberships`가 비어 있고 따라서 build-only"라고 쓰여 있었는데, 다섯 어댑터 remediation이 +`messaging-spring-boot-starter`를 `app-bootstrap` 의존성으로 넣으면서 그 closure 전체가 런타임 +classpath에 올라갔다. 정확한 목록은 registry가 소유하므로 여기서 세지 않는다 — 세는 순간 다시 +drift한다. -- 두 composition root(`app-bootstrap`, `sample-portfolio`) 중 어디에도 런타임으로 들어가 있지 않다. -- 따라서 이 leaf들은 컴파일·테스트·계약 검증의 대상이지, 배포된 애플리케이션이 실행하는 코드가 - 아니다. -- 애플리케이션에 실제로 배선하려면 registry의 `runtime_memberships`를 먼저 바꾸고 +지금 의미는 이렇다: + +- 대부분의 leaf는 `runtime_memberships`에 `app-bootstrap`을 갖는다. **배포된 아티팩트가 싣고 있고, + master switch `app.messaging.enabled`로 켜진다.** 기본값은 `false`다. +- 여전히 비어 있는 leaf(experimental transport, 일부 codec, bridge, testkit)만이 build-only / + incubating이다. +- **런타임에 있다는 것과 자격이 증명됐다는 것은 다르다.** Kafka publish 경로만 실브로커 round trip으로 + 자격 검증됐고(`MessagingLiveRoundTripQualificationTest`), Rabbit·JDBC inbox/outbox·claim check·admin + plane은 shipped, inactive, unqualified다. 이 구분은 + `MessagingMembershipQualificationTest`가 문서가 아니라 테스트로 붙들고 있다. +- 애플리케이션에 새로 배선하려면 registry의 `runtime_memberships`를 먼저 바꾸고 `verifyRuntimeModuleMembership`을 통과시켜야 한다. 코드만 추가하는 것으로는 런타임에 들어가지 않는다. - 지원 상태 문서(`docs/messaging/support-matrix.md`)의 tier는 이 사실과 별개다. tier는 "무엇이 증명되었는가", membership은 "무엇이 실행되는가"다. +## application이 이 family에 도달하는 경로 (MSG-015, 미해결) + +리뷰(`docs/reviews/2026-08-14-messaging-module-code-review.md` §6.1·§6.3)가 정한 흐름은 +`application-owned port → platform anti-corruption bridge → platform`이고, §6.2는 그 bridge의 자리를 +`adapter/outbound/messaging/platformbridge/`로 지정한다. §6.3 첫 규칙: **application은 신규 +broker/runtime 타입을 직접 보지 않고 자기 port만 소유한다.** + +**현재 그 bridge가 없다.** `adapter/outbound/messaging`은 platform에 의존하지 않고(registry의 +`allowed_dependencies` 참조), `app-bootstrap`이 starter를 직접 문다. 그래서 서로를 모르는 messaging +스택 두 개가 한 아티팩트에 있고, 그것이 `kafkaSeamProducer`/`messagingKafkaProducer` bean 이름 충돌로 +드러났다. + +이것이 리뷰의 **MSG-015 (P0)** 이며 닫히지 않았다. 지금 안전한 이유는 설계가 아니라 기본값이다 — +`app.messaging.enabled=false`. 이 스위치를 켜는 배포는 port를 거치지 않는 경로로 platform을 쓰게 된다. +bridge 구현은 런타임 배선 변경이라 리뷰가 말한 대로 별도 change set이다. + ## Stable 승격에 필요한 증거 adapter를 Experimental → Stable로 올릴 때 필요한 것 (`docs/messaging/support-matrix.md`가 현재 @@ -67,6 +100,20 @@ adapter를 Experimental → Stable로 올릴 때 필요한 것 (`docs/messaging/ Stable 문구는 evidence manifest가 존재할 때만 문서에 쓴다. +증거를 만드는 것은 인증 레인이고, manifest는 그 실행의 출력이다: + +```bash +./gradlew :messaging:messaging-kafka:verifyMessagingCertificationEvidence --console=plain +``` + +레인(`messagingCertificationTest`)은 Toxiproxy를 broker 앞에 두고 장애를 주입하며, +통과한 시나리오마다 `BrokerCertificationEvidence` 한 줄을 +`messaging-testkit/src/main/resources/messaging/broker-certification-evidence.jsonl`에 쓴다. +게이트는 커밋된 manifest와 이번 실행의 출력을 대조한다 — **manifest를 손으로 고치면 실패하고, +레인을 돌리면 manifest가 다시 쓰인다.** 이 레인만은 Docker 가드를 달지 않는다(없으면 실패한다). +skip하는 인증 레인은 아무도 켜지 않은 브로커에 대해 성공을 보고하기 때문이며, 그래서 `test`에서는 +`messaging-certification` 태그로 제외된다. + ## Focused test leaf의 `gradle_path`를 registry에서 읽어 파생한다. diff --git a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/CorrelationId.java b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/CorrelationId.java index 3d63b91..00b96bf 100644 --- a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/CorrelationId.java +++ b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/CorrelationId.java @@ -3,15 +3,22 @@ package dev.caskeleton.messaging.api; /** * Workflow-scoped correlation value shared by every message of one business flow. * - * @param value the correlation value, 1 to 160 characters + *

This value is copied onto every message of a flow and onto the header set of every broker the + * flow touches, so it is bounded in {@link WireSafeText UTF-8 bytes} rather than in Java + * characters: the previous 160-character bound admitted a 640-byte value, which a broker with a + * byte-counted header budget rejects at publish time, mid-flow, on a message the caller cannot + * resend under a different identity. + * + * @param value the correlation value, 1 to 160 UTF-8 bytes and free of control characters */ public record CorrelationId(String value) { - private static final int MAX_LENGTH = 160; + private static final int MAX_BYTES = 160; public CorrelationId { - if (value == null || value.isBlank() || value.length() > MAX_LENGTH) { - throw new IllegalArgumentException("correlationId must contain 1 to 160 characters"); + if (value == null || value.isBlank()) { + throw new IllegalArgumentException("correlationId must not be blank"); } + WireSafeText.require(value, "a correlation id", MAX_BYTES); } } diff --git a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageEnvelope.java b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageEnvelope.java index bc554b1..d5ecaa1 100644 --- a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageEnvelope.java +++ b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageEnvelope.java @@ -62,8 +62,18 @@ public record MessageEnvelope( Objects.requireNonNull(traceContext, "traceContext must not be null"); Objects.requireNonNull(headers, "headers must not be null"); Objects.requireNonNull(payload, "payload must not be null"); + // Both keys are routing input the caller chooses, and both used to be unbounded strings. A + // partition key is hashed by the broker, so an unbounded one is a cost rather than a + // correctness problem; an ordering key is carried on the wire by several bindings, so a + // control character in it is the same injection primitive as in a header. The bound is the + // smallest one every targeted broker accepts. + partitionKey.ifPresent(key -> WireSafeText.require(key, "a partition key", MAX_KEY_BYTES)); + orderingKey.ifPresent(key -> WireSafeText.require(key, "an ordering key", MAX_KEY_BYTES)); } + /** The routing-key bound every targeted broker accepts. */ + private static final int MAX_KEY_BYTES = 1024; + /** * Returns a copy of this envelope carrying a different payload representation. * @@ -119,6 +129,31 @@ public record MessageEnvelope( payload); } + /** + * Returns a copy of this envelope bound to a tenant. + * + * @param tenant the tenant this message belongs to + * @return an envelope with identical identity and payload + */ + public MessageEnvelope withTenant(TenantContext tenant) { + return new MessageEnvelope<>( + messageId, + messageType, + schemaVersion, + producedAt, + occurredAt, + producer, + correlationId, + causationId, + contentType, + partitionKey, + orderingKey, + Optional.of(Objects.requireNonNull(tenant, "tenant must not be null")), + traceContext, + headers, + payload); + } + /** * Returns a copy of this envelope carrying replacement headers. * diff --git a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageType.java b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageType.java index 47eef8c..953757b 100644 --- a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageType.java +++ b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/MessageType.java @@ -6,39 +6,16 @@ package dev.caskeleton.messaging.api; *

Java class names are deliberately not usable as a message type: renaming or repackaging a * class must never change the wire contract. * - * @param value the catalog name, 1 to 240 characters + * @param value the catalog name, 1 to 240 UTF-8 bytes and free of control characters */ public record MessageType(String value) { - private static final int MAX_LENGTH = 240; + private static final int MAX_BYTES = 240; public MessageType { - if (value == null || value.isBlank() || value.length() > MAX_LENGTH) { - throw new IllegalArgumentException("messageType must contain 1 to 240 characters"); - } - requireWireSafe(value, "a message type", MAX_LENGTH); - } - - /** - * Characters that must never reach a broker. - * - *

Length in Java {@code char}s bounded nothing useful: a 240-character string can be 960 UTF-8 - * bytes, and a control character passes a length check untouched. CR and LF split records on - * line-oriented protocols, NUL truncates them in C string handling, and both survive right up to - * the broker that mishandles them. - */ - private static void requireWireSafe(String value, String what, int maxBytes) { - for (int index = 0; index < value.length(); index++) { - char character = value.charAt(index); - if (character < 0x20 || character == 0x7F) { - throw new IllegalArgumentException( - what - + " must not contain control characters (found U+%04X)".formatted((int) character)); - } - } - int bytes = value.getBytes(java.nio.charset.StandardCharsets.UTF_8).length; - if (bytes > maxBytes) { - throw new IllegalArgumentException(what + " exceeds " + maxBytes + " UTF-8 bytes: " + bytes); + if (value == null || value.isBlank()) { + throw new IllegalArgumentException("messageType must not be blank"); } + WireSafeText.require(value, "a message type", MAX_BYTES); } } diff --git a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/ProducerId.java b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/ProducerId.java index 313bbc0..0bb858a 100644 --- a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/ProducerId.java +++ b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/ProducerId.java @@ -6,39 +6,16 @@ package dev.caskeleton.messaging.api; *

This is a deployment-independent service name, not a host, pod, or connection identity, so * that it stays a bounded value safe for metric tags. * - * @param value the service name, 1 to 120 characters + * @param value the service name, 1 to 120 UTF-8 bytes and free of control characters */ public record ProducerId(String value) { - private static final int MAX_LENGTH = 120; + private static final int MAX_BYTES = 120; public ProducerId { - if (value == null || value.isBlank() || value.length() > MAX_LENGTH) { - throw new IllegalArgumentException("producerId must contain 1 to 120 characters"); - } - requireWireSafe(value, "a producer id", MAX_LENGTH); - } - - /** - * Characters that must never reach a broker. - * - *

Length in Java {@code char}s bounded nothing useful: a 240-character string can be 960 UTF-8 - * bytes, and a control character passes a length check untouched. CR and LF split records on - * line-oriented protocols, NUL truncates them in C string handling, and both survive right up to - * the broker that mishandles them. - */ - private static void requireWireSafe(String value, String what, int maxBytes) { - for (int index = 0; index < value.length(); index++) { - char character = value.charAt(index); - if (character < 0x20 || character == 0x7F) { - throw new IllegalArgumentException( - what - + " must not contain control characters (found U+%04X)".formatted((int) character)); - } - } - int bytes = value.getBytes(java.nio.charset.StandardCharsets.UTF_8).length; - if (bytes > maxBytes) { - throw new IllegalArgumentException(what + " exceeds " + maxBytes + " UTF-8 bytes: " + bytes); + if (value == null || value.isBlank()) { + throw new IllegalArgumentException("producerId must not be blank"); } + WireSafeText.require(value, "a producer id", MAX_BYTES); } } diff --git a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/TraceContext.java b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/TraceContext.java index ee553f3..b71623f 100644 --- a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/TraceContext.java +++ b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/TraceContext.java @@ -1,7 +1,9 @@ package dev.caskeleton.messaging.api; +import java.nio.charset.StandardCharsets; import java.util.Objects; import java.util.Optional; +import java.util.regex.Pattern; /** * W3C trace propagation values carried with a message. @@ -10,6 +12,13 @@ import java.util.Optional; * envelope rather than only in headers means a trace survives an Outbox round trip through the * database, where broker headers do not exist yet. * + *

The three values used to be checked for non-nullness and nothing more, which made this record + * a general-purpose string carrier wearing the name of a standard. Anything accepted here is + * written to the {@code traceparent}, {@code tracestate} and {@code baggage} headers of every + * broker the message touches, so an unvalidated value is both a header-injection vector and a + * silent trace break: a collector that cannot parse {@code traceparent} drops the span rather than + * reporting a bad value, and the trace disappears at exactly the hop being investigated. + * * @param traceparent the {@code traceparent} value when a trace is active * @param tracestate the {@code tracestate} value when present * @param baggage the {@code baggage} value when present @@ -17,10 +26,112 @@ import java.util.Optional; public record TraceContext( Optional traceparent, Optional tracestate, Optional baggage) { + /** + * {@code version-traceid-spanid-flags}, all lowercase hex, per the W3C Trace Context grammar. + * + *

Uppercase hex is refused rather than folded: the standard defines the field as lowercase, + * and a receiver comparing trace IDs as strings — which collectors do — would treat the two cases + * as two different traces. + */ + private static final Pattern TRACEPARENT = + Pattern.compile("[0-9a-f]{2}-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}"); + + private static final String ALL_ZERO_TRACE_ID = "0".repeat(32); + + private static final String ALL_ZERO_SPAN_ID = "0".repeat(16); + + /** {@code key=value} or {@code tenant@vendor=value}, per the standard's list-member grammar. */ + private static final Pattern TRACESTATE_MEMBER = + Pattern.compile( + "[a-z0-9][_\\-*/a-z0-9]{0,255}(@[a-z0-9][_\\-*/a-z0-9]{0,13})?=[\\x20-\\x7E&&[^,=]]{0,255}"); + + /** + * The standard's bounds: at most 32 list members, and a field a receiver is allowed to truncate. + */ + private static final int MAX_TRACESTATE_MEMBERS = 32; + + private static final int MAX_TRACESTATE_BYTES = 512; + + /** W3C Baggage bounds: 8192 bytes total across at most 64 members. */ + private static final int MAX_BAGGAGE_BYTES = 8192; + + private static final int MAX_BAGGAGE_MEMBERS = 64; + public TraceContext { Objects.requireNonNull(traceparent, "traceparent must not be null"); Objects.requireNonNull(tracestate, "tracestate must not be null"); Objects.requireNonNull(baggage, "baggage must not be null"); + traceparent.ifPresent(TraceContext::requireValidTraceparent); + tracestate.ifPresent(TraceContext::requireValidTracestate); + baggage.ifPresent(TraceContext::requireValidBaggage); + if (tracestate.isPresent() && traceparent.isEmpty()) { + // A tracestate without a traceparent is vendor state belonging to no trace. Propagating it + // hands the next hop a key it will attribute to whatever trace that hop starts. + throw new IllegalArgumentException("tracestate must not be present without a traceparent"); + } + } + + private static void requireValidTraceparent(String value) { + if (!TRACEPARENT.matcher(value).matches()) { + throw new IllegalArgumentException( + "traceparent must be version-traceid-spanid-flags in lowercase hex: '" + value + '\''); + } + String version = value.substring(0, 2); + if ("ff".equals(version)) { + // The standard reserves ff and requires a receiver to reject it, rather than treat it as a + // future version to be forwarded. + throw new IllegalArgumentException("traceparent version ff is invalid"); + } + String traceId = value.substring(3, 35); + if (ALL_ZERO_TRACE_ID.equals(traceId)) { + throw new IllegalArgumentException("traceparent carries an all-zero trace id"); + } + String spanId = value.substring(36, 52); + if (ALL_ZERO_SPAN_ID.equals(spanId)) { + throw new IllegalArgumentException("traceparent carries an all-zero parent span id"); + } + } + + private static void requireValidTracestate(String value) { + int bytes = value.getBytes(StandardCharsets.UTF_8).length; + if (bytes > MAX_TRACESTATE_BYTES) { + throw new IllegalArgumentException("tracestate exceeds 512 bytes: " + bytes); + } + String[] members = value.split(",", -1); + if (members.length > MAX_TRACESTATE_MEMBERS) { + throw new IllegalArgumentException( + "tracestate carries more than 32 list members: " + members.length); + } + for (String member : members) { + String trimmed = member.strip(); + if (trimmed.isEmpty()) { + // The grammar permits empty members for forward compatibility; nothing is propagated for + // them, so they are dropped by the receiver rather than being an error here. + continue; + } + if (!TRACESTATE_MEMBER.matcher(trimmed).matches()) { + throw new IllegalArgumentException("tracestate member is not valid: '" + trimmed + '\''); + } + } + } + + private static void requireValidBaggage(String value) { + int bytes = value.getBytes(StandardCharsets.UTF_8).length; + if (bytes > MAX_BAGGAGE_BYTES) { + throw new IllegalArgumentException("baggage exceeds 8192 bytes: " + bytes); + } + WireSafeText.requireNoControls(value, "baggage"); + String[] members = value.split(",", -1); + if (members.length > MAX_BAGGAGE_MEMBERS) { + throw new IllegalArgumentException("baggage carries more than 64 members: " + members.length); + } + for (String member : members) { + String trimmed = member.strip(); + int equals = trimmed.indexOf('='); + if (equals <= 0 || equals == trimmed.length() - 1) { + throw new IllegalArgumentException("baggage member must be key=value: '" + trimmed + '\''); + } + } } /** diff --git a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/WireSafeText.java b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/WireSafeText.java new file mode 100644 index 0000000..f919c32 --- /dev/null +++ b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/WireSafeText.java @@ -0,0 +1,65 @@ +package dev.caskeleton.messaging.api; + +import java.nio.charset.StandardCharsets; + +/** + * The one place that decides which text may reach a broker. + * + *

Every wire-facing value used to bound itself in Java {@code char}s and check nothing else. A + * {@code char} count bounds nothing on a wire: a 240-character string is up to 960 UTF-8 bytes, so + * a "240 character" limit is a 960-byte limit against a broker that counts bytes. And a control + * character passes a length check untouched — CR and LF split records on line-oriented protocols, + * NUL truncates them in C string handling inside a broker, and DEL is stripped by some + * intermediaries and preserved by others. A value that survives one broker and corrupts the next is + * worse than a rejected one. + * + *

The rule is refused rather than sanitised on purpose. Silently stripping a CR turns a caller's + * two-line value into a one-line value that no longer means what they wrote, and the caller never + * learns; the broker downstream cannot tell the difference between that and a legitimate value. + * + *

Each copy of this check that lived in its own record was one more place for the rule to drift. + */ +public final class WireSafeText { + + private WireSafeText() {} + + /** + * Refuses text that no broker should be asked to carry. + * + *

Horizontal tab is refused along with the rest of the C0 range. It is legal in an HTTP field + * value, and that is exactly the problem: a header carried over a line-folding binding and the + * same header carried over a length-prefixed one disagree about whether a tab ends the value. + * Nothing in this platform needs a tab in an identifier or a header. + * + * @param value the text to check + * @param what a noun phrase naming the value, used in the failure message + * @param maxBytes the inclusive UTF-8 byte bound + * @throws IllegalArgumentException when the text carries a control character or exceeds the bound + */ + public static void require(String value, String what, int maxBytes) { + requireNoControls(value, what); + int bytes = value.getBytes(StandardCharsets.UTF_8).length; + if (bytes > maxBytes) { + throw new IllegalArgumentException( + what + " exceeds " + maxBytes + " UTF-8 bytes: " + bytes + " bytes"); + } + } + + /** + * Refuses control characters without applying a size bound. + * + * @param value the text to check + * @param what a noun phrase naming the value, used in the failure message + * @throws IllegalArgumentException when the text carries a control character + */ + public static void requireNoControls(String value, String what) { + for (int index = 0; index < value.length(); index++) { + char character = value.charAt(index); + if (character < 0x20 || character == 0x7F) { + throw new IllegalArgumentException( + "%s must not contain control characters (found U+%04X at index %d)" + .formatted(what, (int) character, index)); + } + } + } +} diff --git a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/CanonicalEnvelopeHeaders.java b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/CanonicalEnvelopeHeaders.java new file mode 100644 index 0000000..4e95526 --- /dev/null +++ b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/CanonicalEnvelopeHeaders.java @@ -0,0 +1,79 @@ +package dev.caskeleton.messaging.api.header; + +import java.util.Locale; +import java.util.Set; + +/** + * Splits the reserved namespace into the part an envelope field already owns and the part that only + * a header can carry. + * + *

{@link ReservedHeaders} answers one question — does the platform own this name — and every + * adapter that asked it treated the whole namespace as forgeable content. That is right for {@code + * msg.id} and wrong for {@code msg.retry-attempt}: the first restates a value the envelope already + * holds, so a header claiming it can only overwrite the canonical one, while the second has no + * envelope field at all and exists precisely to survive a hop through a broker. Refusing both is + * how a retry re-publish becomes an error, and dropping both on the way back is how an attempt + * counter silently restarts at one and a retry budget stops capping anything. + * + *

The split is derived from the envelope's own shape rather than listed by hand per adapter, so + * Kafka and RabbitMQ cannot disagree about which headers are identity and which are bookkeeping — a + * disagreement that would make the same message mean different things on two brokers. + */ +public final class CanonicalEnvelopeHeaders { + + /** + * Reserved names that restate a value the envelope already carries in a field. + * + *

Every one of these is written by the adapter from an envelope field on the way out and read + * back into that field on the way in. A copy of it inside the header map is therefore never the + * source of truth, only a way to disagree with it. + */ + private static final Set ENVELOPE_FIELDS = + Set.of( + ReservedHeaders.MESSAGE_ID, + ReservedHeaders.MESSAGE_TYPE, + ReservedHeaders.SCHEMA_VERSION, + ReservedHeaders.PRODUCER, + ReservedHeaders.PRODUCED_AT, + ReservedHeaders.OCCURRED_AT, + ReservedHeaders.CORRELATION_ID, + ReservedHeaders.CAUSATION_ID, + ReservedHeaders.CONTENT_TYPE, + ReservedHeaders.PARTITION_KEY, + ReservedHeaders.ORDERING_KEY, + ReservedHeaders.TENANT, + ReservedHeaders.TRACEPARENT, + ReservedHeaders.TRACESTATE, + ReservedHeaders.BAGGAGE); + + private CanonicalEnvelopeHeaders() {} + + /** + * Reports whether a header name restates an envelope field. + * + * @param name the header name in any case + * @return true when the envelope already carries this value in a field of its own + */ + public static boolean restatesEnvelopeField(String name) { + return name != null && ENVELOPE_FIELDS.contains(name.toLowerCase(Locale.ROOT)); + } + + /** + * Reports whether a header name is platform bookkeeping that must travel in the header map. + * + * @param name the header name in any case + * @return true for a reserved name with no envelope field behind it + */ + public static boolean isPlatformBookkeeping(String name) { + return ReservedHeaders.isReserved(name) && !restatesEnvelopeField(name); + } + + /** + * Returns the reserved names an envelope field owns. + * + * @return an immutable set of lowercase names + */ + public static Set envelopeFieldNames() { + return ENVELOPE_FIELDS; + } +} diff --git a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/HeaderValue.java b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/HeaderValue.java index 12c3199..c365fd1 100644 --- a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/HeaderValue.java +++ b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/HeaderValue.java @@ -1,11 +1,17 @@ package dev.caskeleton.messaging.api.header; -import java.nio.charset.StandardCharsets; +import dev.caskeleton.messaging.api.WireSafeText; /** * Value of a single message header. * - * @param value the header value, at most 4096 UTF-8 bytes + *

The bound used to be the only rule, so a value containing CR, LF or NUL reached the broker + * intact. On a line-oriented binding a CRLF inside a value ends the header and starts a new one the + * caller wrote — the same header-injection primitive as in HTTP, with the broker's own control + * headers as the target. A NUL truncates the value inside a broker that hands it to C string + * handling, so a subscriber receives a prefix of what was published and cannot tell. + * + * @param value the header value, at most 4096 UTF-8 bytes and free of control characters */ public record HeaderValue(String value) { @@ -15,9 +21,6 @@ public record HeaderValue(String value) { if (value == null) { throw new IllegalArgumentException("header value must not be null"); } - int bytes = value.getBytes(StandardCharsets.UTF_8).length; - if (bytes > MAX_BYTES) { - throw new IllegalArgumentException("header value exceeds 4096 bytes: " + bytes); - } + WireSafeText.require(value, "a header value", MAX_BYTES); } } diff --git a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/MessageHeaders.java b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/MessageHeaders.java index 328412a..21bbc76 100644 --- a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/MessageHeaders.java +++ b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/MessageHeaders.java @@ -34,6 +34,31 @@ public final class MessageHeaders { "password", "client_secret"); + /** + * Words that make a header name a credential regardless of what surrounds them. + * + *

Exact-name matching only refuses the spellings someone thought of. {@code x-api-key}, {@code + * auth-token} and {@code db_password} are the same credential leak as {@code authorization} and + * none of them was on the list. Matching whole segments rather than substrings is what keeps + * {@code tokenizer-version} or {@code secretariat-id} from being refused for containing the + * letters of a secret. + */ + private static final Set SECRET_SEGMENTS = + Set.of( + "password", + "passwd", + "secret", + "token", + "credential", + "credentials", + "apikey", + "authorization", + "cookie", + "signature"); + + private static final java.util.regex.Pattern SEGMENT_SEPARATOR = + java.util.regex.Pattern.compile("[._\\-]+"); + private static final MessageHeaders EMPTY = new MessageHeaders(Map.of()); private final Map values; @@ -96,7 +121,9 @@ public final class MessageHeaders { if (!reservedAllowed && ReservedHeaders.isReserved(normalized)) { throw new IllegalArgumentException("message header is not allowed: " + normalized); } - if (SECRET_NAMES.contains(normalized)) { + if (carriesACredential(normalized)) { + // The name is reported and the value never is: an error message is written to a log that + // is exactly as readable as the broker storage this check exists to keep the value out of. throw new IllegalArgumentException("message header is not allowed: " + normalized); } if (!seen.add(normalized)) { @@ -112,6 +139,26 @@ public final class MessageHeaders { return new MessageHeaders(Collections.unmodifiableMap(copy)); } + private static boolean carriesACredential(String normalized) { + if (SECRET_NAMES.contains(normalized)) { + return true; + } + String[] segments = SEGMENT_SEPARATOR.split(normalized, -1); + for (int index = 0; index < segments.length; index++) { + if (SECRET_SEGMENTS.contains(segments[index])) { + return true; + } + // Adjacent segments are also tested joined, because the same word is written both ways: + // `api_key` is one segment to a reader and two to a splitter, and `x-api-key` is two of + // three. Joining only neighbouring pairs is what keeps `routing-key` accepted. + if (index + 1 < segments.length + && SECRET_SEGMENTS.contains(segments[index] + segments[index + 1])) { + return true; + } + } + return false; + } + /** * Returns the headers as an immutable map. * diff --git a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/ReservedHeaders.java b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/ReservedHeaders.java index d06ee8c..50eb931 100644 --- a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/ReservedHeaders.java +++ b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/header/ReservedHeaders.java @@ -45,6 +45,16 @@ public final class ReservedHeaders { /** Ordering key. */ public static final String ORDERING_KEY = "msg.ordering-key"; + /** + * Bounded tenant identity. + * + *

The envelope carried a tenant and no header name existed for it, so the write side had + * nothing to write and the read side rebuilt every consumed message with an empty tenant. A + * message published by one tenant and consumed as belonging to none is not a lost label: it is + * the field every downstream authorization and data-partitioning decision reads. + */ + public static final String TENANT = "msg.tenant"; + /** Identity of the redrive operation that replayed this message. */ public static final String REDRIVE_ID = "msg.redrive-id"; @@ -91,6 +101,7 @@ public final class ReservedHeaders { CONTENT_TYPE, PARTITION_KEY, ORDERING_KEY, + TENANT, REDRIVE_ID, REDRIVE_COUNT, RETRY_ATTEMPT, @@ -105,6 +116,16 @@ public final class ReservedHeaders { private ReservedHeaders() {} + /** + * The namespace the platform owns, whether or not a name in it exists yet. + * + *

The check used to be exact membership of {@link #NAMES}, so {@code msg.anything} that this + * release has not defined was an ordinary application header — until a later release defined it, + * at which point every application already writing it silently started overwriting envelope + * metadata. Owning the prefix means a new platform header is a compatible change. + */ + private static final String PLATFORM_PREFIX = "msg."; + /** * Reports whether a header name belongs to the platform. * @@ -112,7 +133,11 @@ public final class ReservedHeaders { * @return true when the platform owns the name */ public static boolean isReserved(String name) { - return name != null && NAMES.contains(name.toLowerCase(Locale.ROOT)); + if (name == null) { + return false; + } + String normalized = name.toLowerCase(Locale.ROOT); + return NAMES.contains(normalized) || normalized.startsWith(PLATFORM_PREFIX); } /** diff --git a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishOptions.java b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishOptions.java index f89c75a..f3d07e5 100644 --- a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishOptions.java +++ b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishOptions.java @@ -2,27 +2,25 @@ package dev.caskeleton.messaging.api.publish; import dev.caskeleton.messaging.api.destination.ConfirmationRequirement; import java.time.Duration; -import java.util.Map; import java.util.Objects; import java.util.Optional; /** * Per-call publish options. * - *

{@code brokerHints} must be empty for M1 callers. Only an M3 native capability may populate - * it, because a hint map on the general API becomes an escape hatch that quietly bypasses - * destination policy. + *

There is no free-form hint map. One existed for a native surface that does not read it, so the + * only thing a caller could do with it was pass adapter settings that the publish path refused at + * runtime — an escape hatch around destination policy that never opened. A broker-specific option + * belongs in that broker's own typed API, where the adapter that honours it can be seen to. * * @param timeout the publish operation deadline * @param confirmation the confirmation this call requires * @param deduplication optional broker-side duplicate suppression - * @param brokerHints M3-only adapter hints */ public record PublishOptions( Duration timeout, ConfirmationRequirement confirmation, - Optional deduplication, - Map brokerHints) { + Optional deduplication) { private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(5); @@ -30,32 +28,18 @@ public record PublishOptions( Objects.requireNonNull(timeout, "timeout must not be null"); Objects.requireNonNull(confirmation, "confirmation must not be null"); Objects.requireNonNull(deduplication, "deduplication must not be null"); - Objects.requireNonNull(brokerHints, "brokerHints must not be null"); if (timeout.isNegative() || timeout.isZero()) { throw new IllegalArgumentException("publish timeout must be positive"); } - brokerHints = Map.copyOf(brokerHints); } /** - * Returns the M1 default: five second timeout, replication evidence, no hints. + * Returns the M1 default: five second timeout, replication evidence. * * @return the default options */ public static PublishOptions defaults() { return new PublishOptions( - DEFAULT_TIMEOUT, - ConfirmationRequirement.REPLICATION_OR_PERSISTENCE_ACK, - Optional.empty(), - Map.of()); - } - - /** - * Reports whether these options stay inside the M1 surface. - * - * @return true when no broker hints are present - */ - public boolean isM1Compatible() { - return brokerHints.isEmpty(); + DEFAULT_TIMEOUT, ConfirmationRequirement.REPLICATION_OR_PERSISTENCE_ACK, Optional.empty()); } } diff --git a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishResult.java b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishResult.java index 38c714f..7449c3a 100644 --- a/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishResult.java +++ b/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/publish/PublishResult.java @@ -8,10 +8,16 @@ import java.util.Optional; /** * The outcome of one publish call, with the evidence that justifies it. * - *

The constructor refuses combinations that would let a caller read a stronger guarantee than - * was actually obtained. An adapter that cannot decide must report {@link - * PublishCompletion#AMBIGUOUS} with no confirmation level, which is the conservative answer, rather - * than picking a completion that happens to make the call site simpler. + *

The constructor refuses every combination that would let a caller read a stronger guarantee + * than was actually obtained, and every one that contradicts itself: a rejection the broker + * acknowledged, a confirmation that does not know whether it routed, an ambiguous outcome the + * broker routed, a broker coordinate for bytes that never left. An adapter that cannot decide must + * report {@link PublishCompletion#AMBIGUOUS} with no confirmation level, which is the conservative + * answer, rather than picking a completion that happens to make the call site simpler. + * + *

Validating here rather than trusting call sites is deliberate: the record is public, every + * adapter builds one, and a contradiction assembled in an adapter is only ever read much later, by + * an operator deciding whether to re-publish. * * @param completion how the attempt ended * @param evidence the observable facts behind that outcome @@ -72,6 +78,26 @@ public record PublishResult( if (completion == PublishCompletion.CONFIRMED && failure.isPresent()) { throw new IllegalArgumentException("confirmed publish must not carry a failure descriptor"); } + if (completion == PublishCompletion.REJECTED && evidence.brokerAccepted()) { + // A broker that acknowledged the message did not reject it. Left representable, this is the + // combination that turns a delivered message into one the caller re-publishes as if it had + // never been sent. + throw new IllegalArgumentException("rejected publish cannot claim broker acceptance"); + } + if (completion == PublishCompletion.CONFIRMED && routingOutcome == RoutingOutcome.UNKNOWN) { + // The confirmation came from the broker, and the same response says whether it routed. A + // confirmed publish that does not know is an adapter reporting half of what it was told. + throw new IllegalArgumentException("confirmed publish knows whether it routed"); + } + if (completion == PublishCompletion.AMBIGUOUS && routingOutcome == RoutingOutcome.ROUTED) { + // Routing is something only the broker can report, and a broker that reported it answered. + throw new IllegalArgumentException("a routed publish is not ambiguous"); + } + if (position.isPresent() && evidence.transmission() == TransmissionEvidence.NOT_TRANSMITTED) { + // A partition and offset for a message that never left the process: whatever coordinate this + // is, it belongs to some other message, and an operator would chase it. + throw new IllegalArgumentException("an untransmitted publish has no broker position"); + } } /** diff --git a/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/WireBoundaryRejectionTest.java b/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/WireBoundaryRejectionTest.java new file mode 100644 index 0000000..cc73fd3 --- /dev/null +++ b/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/WireBoundaryRejectionTest.java @@ -0,0 +1,208 @@ +package dev.caskeleton.messaging.api; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.header.HeaderName; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * The rest of the wire boundary: values, correlation, trace propagation and the two denylists. + * + *

{@link WireSafeValueObjectTest} covers the name side. This covers what was still unbounded + * after it: a header value carrying CRLF straight to the broker, a correlation id bounded + * in Java characters rather than bytes, a {@code traceparent} that was any string at all, and two + * denylists that matched exact spellings only. + */ +class WireBoundaryRejectionTest { + + /** The C0 and DEL code points a broker disagrees with itself about. */ + @ParameterizedTest + @ValueSource(ints = {0x00, 0x09, 0x0A, 0x0D, 0x1B, 0x7F}) + @DisplayName("a header value carrying a control character is refused") + void aHeaderValueWithAControlCharacterIsRefused(int codePoint) { + // NUL truncates the value inside a broker that hands it to C string handling; LF and CR end + // the header on a line-oriented binding and start one the caller wrote; ESC and DEL are + // stripped by some intermediaries and preserved by others. Tab is refused with them because a + // line-folding binding and a length-prefixed one disagree about whether it ends the value. + String value = "order-" + (char) codePoint + "-created"; + + assertThatThrownBy(() -> new HeaderValue(value)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("control characters"); + } + + @Test + @DisplayName("an ordinary header value, including non-ASCII text, is still accepted") + void anOrdinaryHeaderValueIsAccepted() { + assertThatCode(() -> new HeaderValue("주문 생성됨 / order created")).doesNotThrowAnyException(); + } + + @Test + @DisplayName("a header value is bounded in bytes, not in characters") + void aHeaderValueIsBoundedInBytes() { + // 4096 characters of three bytes each: inside any character bound, three times the byte one. + assertThatThrownBy(() -> new HeaderValue("가".repeat(4096))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("UTF-8 bytes"); + } + + @Test + @DisplayName("a correlation id is bounded in bytes and refuses control characters") + void aCorrelationIdIsBoundedInBytes() { + assertThatThrownBy(() -> new CorrelationId("가".repeat(160))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("UTF-8 bytes"); + assertThatThrownBy(() -> new CorrelationId("flow-1\r\nmsg.id: forged")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("control characters"); + assertThatCode(() -> new CorrelationId("flow-0198e2b1")).doesNotThrowAnyException(); + } + + @ParameterizedTest + @ValueSource( + strings = { + "not-a-traceparent", + "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331", + "00-0AF7651916CD43DD8448EB211C80319C-b7ad6b7169203331-01", + "ff-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", + "00-00000000000000000000000000000000-b7ad6b7169203331-01", + "00-0af7651916cd43dd8448eb211c80319c-0000000000000000-01", + "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01\r\nx: y" + }) + @DisplayName("an invalid traceparent is refused rather than propagated") + void anInvalidTraceparentIsRefused(String traceparent) { + // In order: not the grammar at all; no flags field; uppercase hex, which a collector comparing + // trace ids as strings reads as a different trace; the reserved ff version; an all-zero trace + // id and an all-zero parent span id, both of which the standard declares invalid; and a value + // with a CRLF, which is a header injection once it is written to the traceparent header. + assertThatThrownBy(() -> TraceContext.of(traceparent)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + @DisplayName("the W3C example traceparent is accepted") + void theStandardExampleIsAccepted() { + assertThatCode(() -> TraceContext.of("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01")) + .doesNotThrowAnyException(); + } + + @Test + @DisplayName("tracestate is bounded by the standard's member count and size") + void tracestateIsBounded() { + String parent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; + + assertThatCode( + () -> + new TraceContext( + Optional.of(parent), + Optional.of("congo=t61rcWkgMzE,rojo=00f067aa0ba902b7"), + Optional.empty())) + .doesNotThrowAnyException(); + + String tooManyMembers = + java.util.stream.IntStream.range(0, 33) + .mapToObj(index -> "k" + index + "=v") + .collect(java.util.stream.Collectors.joining(",")); + assertThatThrownBy( + () -> + new TraceContext( + Optional.of(parent), Optional.of(tooManyMembers), Optional.empty())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("32 list members"); + + assertThatThrownBy( + () -> + new TraceContext( + Optional.of(parent), Optional.of("congo=a\r\nx=y"), Optional.empty())) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + @DisplayName("tracestate without a traceparent is vendor state belonging to no trace") + void tracestateWithoutATraceparentIsRefused() { + assertThatThrownBy( + () -> + new TraceContext( + Optional.empty(), Optional.of("congo=t61rcWkgMzE"), Optional.empty())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("without a traceparent"); + } + + @Test + @DisplayName("baggage is bounded in total bytes and in member count") + void baggageIsBounded() { + assertThatCode( + () -> + new TraceContext( + Optional.empty(), Optional.empty(), Optional.of("tenant=acme,region=eu"))) + .doesNotThrowAnyException(); + + String oversize = "k=" + "v".repeat(8200); + assertThatThrownBy( + () -> new TraceContext(Optional.empty(), Optional.empty(), Optional.of(oversize))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("8192 bytes"); + + String tooManyMembers = + java.util.stream.IntStream.range(0, 65) + .mapToObj(index -> "k" + index + "=v") + .collect(java.util.stream.Collectors.joining(",")); + assertThatThrownBy( + () -> new TraceContext(Optional.empty(), Optional.empty(), Optional.of(tooManyMembers))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("64 members"); + } + + @ParameterizedTest + @ValueSource(strings = {"x-api-key", "auth-token", "db_password", "request.signature", "Cookie"}) + @DisplayName("a credential-shaped header name is refused however it is spelled") + void aCredentialShapedNameIsRefused(String name) { + assertThatThrownBy( + () -> + MessageHeaders.application(Map.of(new HeaderName(name), new HeaderValue("opaque")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("not allowed"); + } + + @Test + @DisplayName("a name that merely contains the letters of a secret is still accepted") + void aNameThatMerelyContainsTheLettersIsAccepted() { + assertThatCode( + () -> + MessageHeaders.application( + Map.of( + new HeaderName("tokenizer-version"), new HeaderValue("3"), + new HeaderName("secretariat-id"), new HeaderValue("s-9")))) + .as("segment matching, not substring matching") + .doesNotThrowAnyException(); + } + + @Test + @DisplayName("the platform namespace is reserved whether or not the name exists yet") + void thePlatformNamespaceIsReserved() { + assertThat(ReservedHeaders.isReserved("msg.not-defined-in-this-release")).isTrue(); + assertThatThrownBy( + () -> + MessageHeaders.application( + Map.of(new HeaderName("msg.not-defined-yet"), new HeaderValue("mine")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("not allowed"); + + // The platform factory still writes its own namespace; that is the whole distinction. + assertThatCode( + () -> + MessageHeaders.platform( + Map.of(new HeaderName(ReservedHeaders.MESSAGE_ID), new HeaderValue("m-1")))) + .doesNotThrowAnyException(); + } +} diff --git a/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/publish/PublishResultTest.java b/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/publish/PublishResultTest.java index dc34219..2e1add5 100644 --- a/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/publish/PublishResultTest.java +++ b/messaging/messaging-core-api/src/test/java/dev/caskeleton/messaging/api/publish/PublishResultTest.java @@ -11,9 +11,90 @@ import org.junit.jupiter.api.Test; class PublishResultTest { + /** A coordinate no adapter would have produced for a message that never left. */ + private static final BrokerPosition POSITION = + new BrokerPosition() { + @Override + public String broker() { + return "kafka"; + } + + @Override + public java.util.Map diagnosticAttributes() { + return java.util.Map.of("topic", "order-events", "partition", "0", "offset", "17"); + } + }; + private static final FailureDescriptor AMBIGUOUS_FAILURE = FailureDescriptor.of(FailureCategory.AMBIGUOUS, "CONFIRM_TIMEOUT", "confirm timed out"); + @Test + void aRejectionCannotClaimTheBrokerAcceptedIt() { + PublishEvidence accepted = + PublishEvidence.confirmed(ConfirmationLevel.REPLICATION_OR_PERSISTENCE_ACK); + + assertThatThrownBy( + () -> + new PublishResult( + PublishCompletion.REJECTED, + accepted, + RoutingOutcome.ROUTED, + Optional.empty(), + 1, + Duration.ofMillis(10), + Optional.of(AMBIGUOUS_FAILURE))) + .as("re-publishing a message the broker acknowledged is how one order becomes two") + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void aConfirmationCannotBeUnsureWhetherItRouted() { + assertThatThrownBy( + () -> + new PublishResult( + PublishCompletion.CONFIRMED, + PublishEvidence.confirmed(ConfirmationLevel.BROKER_ACK), + RoutingOutcome.UNKNOWN, + Optional.empty(), + 1, + Duration.ofMillis(10), + Optional.empty())) + .as("the response that confirmed the publish is the one that says whether it routed") + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void aRoutedPublishIsNotAmbiguous() { + assertThatThrownBy( + () -> + new PublishResult( + PublishCompletion.AMBIGUOUS, + PublishEvidence.ambiguous(), + RoutingOutcome.ROUTED, + Optional.empty(), + 1, + Duration.ofMillis(10), + Optional.of(AMBIGUOUS_FAILURE))) + .as("only the broker can report routing, and a broker that reported it answered") + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void anUntransmittedPublishHasNoBrokerPosition() { + assertThatThrownBy( + () -> + new PublishResult( + PublishCompletion.REJECTED, + PublishEvidence.notTransmitted(), + RoutingOutcome.NOT_APPLICABLE, + Optional.of(POSITION), + 1, + Duration.ofMillis(10), + Optional.of(AMBIGUOUS_FAILURE))) + .as("a coordinate for bytes that never left belongs to some other message") + .isInstanceOf(IllegalArgumentException.class); + } + @Test void confirmedResultRequiresBrokerAcceptance() { PublishEvidence evidence = @@ -158,9 +239,4 @@ class PublishResultTest { Optional.empty())) .isInstanceOf(IllegalArgumentException.class); } - - @Test - void m1OptionsCarryNoBrokerHints() { - assertThat(PublishOptions.defaults().isM1Compatible()).isTrue(); - } } diff --git a/messaging/messaging-kafka/build.gradle b/messaging/messaging-kafka/build.gradle index a15369d..0d1618e 100644 --- a/messaging/messaging-kafka/build.gradle +++ b/messaging/messaging-kafka/build.gradle @@ -30,3 +30,100 @@ dependencies { testImplementation 'org.testcontainers:testcontainers-toxiproxy' testImplementation 'eu.rekawek.toxiproxy:toxiproxy-java:2.1.7' } + +// The certification lane, and the gate that makes its output the only source of a Stable claim. +// +// `CertifiedEvidence` used to be a hand-written list of three scenarios, and every consumer of it — +// the compatibility matrix, the cross-broker release suite, the support matrix document — believed +// it without a container ever starting. The lane below produces that list, and +// `verifyMessagingCertificationEvidence` refuses to pass when the committed manifest claims a +// scenario this run did not prove. Editing the manifest by hand therefore fails the build; running +// the lane rewrites it. +strictTestLanes { + lane('messagingCertificationTest') { + tag = 'messaging-certification' + description = 'Runs the broker fault scenarios against a real Kafka and writes the ' + + 'certification evidence the compatibility matrix reads.' + customize = { test -> + // Pinned images. A certification claim names the build it was made against, so a + // floating tag would make a red run unattributable and a green one unrepeatable. + // `-PmessagingKafkaImage=` overrides for a one-off run against another version. + test.systemProperty 'messaging.kafka.image', + (project.findProperty('messagingKafkaImage') ?: 'apache/kafka:4.1.0').toString() + test.systemProperty 'messaging.toxiproxy.image', + (project.findProperty('messagingToxiproxyImage') + ?: 'ghcr.io/shopify/toxiproxy:2.12.0').toString() + test.systemProperty 'messaging.certification.manifest', + project.layout.buildDirectory + .file('messaging-certification/broker-certification-evidence.jsonl') + .get().asFile.absolutePath + // The commit is part of the evidence: "certified" is a claim about one source tree. + // Read from the environment rather than a `-P` flag so the CI job's command line stays + // the literal grammar the gate matrix lint accepts. + test.systemProperty 'messaging.certification.commit', + (project.findProperty('certificationCommit') + ?: providers.environmentVariable('GITHUB_SHA').getOrElse('local')) + .toString() + } + } +} + +// The certification tag is excluded from `test` because the lane deliberately carries no Docker +// guard. Every other container suite here skips with a stated reason when Docker is absent; a lane +// that skipped would report success for a broker nobody started, which is the whole failure the +// evidence exists to rule out. Keeping it out of `test` is what lets it fail closed without +// breaking a laptop build. +tasks.named('test', Test) { + useJUnitPlatform { + excludeTags 'quarantine', 'messaging-certification' + } +} + +tasks.register('verifyMessagingCertificationEvidence') { + group = 'verification' + description = 'Fails when the committed broker certification manifest claims a scenario the ' + + 'certification lane did not produce.' + dependsOn 'messagingCertificationTest' + + def produced = layout.buildDirectory + .file('messaging-certification/broker-certification-evidence.jsonl') + def committed = rootProject.file( + 'messaging/messaging-testkit/src/main/resources/messaging/' + + 'broker-certification-evidence.jsonl') + inputs.file(produced) + inputs.file(committed) + outputs.file(layout.buildDirectory.file('reports/messaging-certification-evidence.txt')) + // A gate whose result can be served from an earlier run is evidence about that run. + outputs.upToDateWhen { false } + + doLast { + // The commit and the observation instant differ on every run by design, so they are not + // part of the comparison — what has to match is which adapter proved which scenario against + // which image, and which test produced it. + Closure> claims = { File file -> + file.readLines('UTF-8') + .findAll { !it.trim().isEmpty() } + .collect { line -> + line.replaceAll(/,"gitCommit":"[^"]*"/, '') + .replaceAll(/,"observedAt":"[^"]*"/, '') + } + .toSet() + } + Set ran = claims(produced.get().asFile) + Set shipped = claims(committed) + + if (ran != shipped) { + def unproven = shipped - ran + def unrecorded = ran - shipped + throw new GradleException( + "the committed certification manifest does not match this run.\n" + + " claimed but not produced: ${unproven.isEmpty() ? 'none' : unproven}\n" + + " produced but not claimed: ${unrecorded.isEmpty() ? 'none' : unrecorded}\n" + + "Copy ${produced.get().asFile} over ${committed} — the manifest is a " + + "record of a run, not a statement about one.") + } + def report = outputs.files.singleFile + report.parentFile.mkdirs() + report.text = "scenarios=${ran.size()} manifest=${committed}\n" + } +} diff --git a/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaDeliveryMapper.java b/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaDeliveryMapper.java index 0d70ff8..239ba8e 100644 --- a/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaDeliveryMapper.java +++ b/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaDeliveryMapper.java @@ -13,7 +13,6 @@ import dev.caskeleton.messaging.api.TraceContext; import dev.caskeleton.messaging.api.delivery.DeliveryMetadata; import dev.caskeleton.messaging.api.destination.DestinationName; import dev.caskeleton.messaging.api.error.MessageValidationException; -import dev.caskeleton.messaging.api.header.MessageHeaders; import dev.caskeleton.messaging.api.header.ReservedHeaders; import dev.caskeleton.messaging.schema.EncodedMessage; import java.time.Instant; @@ -86,9 +85,10 @@ public final class KafkaDeliveryMapper { contentType, Optional.ofNullable(headerMapper.value(headers, ReservedHeaders.PARTITION_KEY)), Optional.ofNullable(headerMapper.value(headers, ReservedHeaders.ORDERING_KEY)), - Optional.empty(), + Optional.ofNullable(headerMapper.value(headers, ReservedHeaders.TENANT)) + .map(TenantContext::new), traceContext(headers), - MessageHeaders.empty(), + headerMapper.carriedForward(headers), new EncodedMessage(payload, contentType, Optional.empty())); } diff --git a/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaHeaderMapper.java b/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaHeaderMapper.java index f72e3d1..b9624dd 100644 --- a/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaHeaderMapper.java +++ b/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaHeaderMapper.java @@ -1,6 +1,7 @@ package dev.caskeleton.messaging.kafka; import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.header.CanonicalEnvelopeHeaders; import dev.caskeleton.messaging.api.header.HeaderName; import dev.caskeleton.messaging.api.header.HeaderValue; import dev.caskeleton.messaging.api.header.MessageHeaders; @@ -54,6 +55,12 @@ public final class KafkaHeaderMapper { value -> put(headers, ReservedHeaders.CAUSATION_ID, value.value().value().toString())); envelope.partitionKey().ifPresent(value -> put(headers, ReservedHeaders.PARTITION_KEY, value)); envelope.orderingKey().ifPresent(value -> put(headers, ReservedHeaders.ORDERING_KEY, value)); + // Written because the consumer has no other way to learn it: a broker record carries no + // ambient tenant, so a tenant left off the wire is a tenant the consumed envelope does not + // have — and every authorization and partitioning decision downstream reads that field. + envelope + .tenantContext() + .ifPresent(value -> put(headers, ReservedHeaders.TENANT, value.tenantId())); envelope .traceContext() .traceparent() @@ -67,24 +74,26 @@ public final class KafkaHeaderMapper { .baggage() .ifPresent(value -> put(headers, ReservedHeaders.BAGGAGE, value)); - // Application headers are written last and must not be able to reach a reserved name. + // The envelope's own header map is written last and the consumer reads `lastHeader`, so a name + // written here wins over the canonical value above. A row in the outbox carrying `msg.id` + // therefore overwrote the identity that inbox deduplication and the audit trail key on, + // corrupting a different message's dedup state rather than merely its own. // - // They used to be appended after the canonical ones, and the consumer reads `lastHeader` — so a - // row in the outbox carrying `msg.id` overwrote the canonical message identity on the wire. - // That identity is what inbox deduplication and the audit trail key on, so a forged one - // corrupts another message's dedup state rather than merely its own. + // Only names that restate an envelope field are refused. Retry attempt, failure and redrive + // bookkeeping have no envelope field to be read back from, so refusing them would make every + // retry-topic and dead-letter re-publish fail on the guard meant to protect identity. envelope .headers() .asMap() .forEach( (name, value) -> { - if (ReservedHeaders.isReserved(name.value())) { + if (CanonicalEnvelopeHeaders.restatesEnvelopeField(name.value())) { throw new dev.caskeleton.messaging.api.error.MessagingConfigurationException( "RESERVED_HEADER_FORGED", - "application header '" + "header '" + name.value() - + "' uses the reserved namespace; the canonical value is written by the" - + " platform and must not be overridable by message content"); + + "' restates a value the envelope already carries; the canonical value is" + + " written by the platform and must not be overridable by message content"); } put(headers, name.value(), value.value()); }); @@ -110,6 +119,36 @@ public final class KafkaHeaderMapper { return MessageHeaders.platform(values); } + /** + * Reads back the headers a rebuilt envelope keeps. + * + *

Everything except the names that were hoisted into envelope fields: the application's own + * headers, which were dropped entirely and so never survived a broker hop, and the platform's + * retry and redrive bookkeeping, whose loss restarted the attempt counter at one on every + * redelivery and left the retry budget capping nothing. + * + *

The hoisted names are excluded rather than kept alongside their fields, because a duplicate + * that can drift from the field is how the same message ends up with two different ids depending + * on which one the reader trusts. + * + * @param headers the record headers + * @return the headers to carry on the rebuilt envelope + */ + public MessageHeaders carriedForward(Headers headers) { + Objects.requireNonNull(headers, "headers must not be null"); + Map values = new LinkedHashMap<>(); + for (Header header : headers) { + if (CanonicalEnvelopeHeaders.restatesEnvelopeField(header.key())) { + continue; + } + byte[] value = header.value(); + values.put( + new HeaderName(header.key()), + new HeaderValue(value == null ? "" : new String(value, StandardCharsets.UTF_8))); + } + return MessageHeaders.platform(values); + } + /** * Reads one header value. * diff --git a/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaBrokerCertificationIT.java b/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaBrokerCertificationIT.java new file mode 100644 index 0000000..97ef840 --- /dev/null +++ b/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaBrokerCertificationIT.java @@ -0,0 +1,463 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.TransmissionEvidence; +import dev.caskeleton.messaging.testkit.BrokerCertificationEvidence; +import dev.caskeleton.messaging.testkit.NetworkFaultScenario; +import dev.caskeleton.messaging.transport.GracefulShutdownCoordinator; +import dev.caskeleton.messaging.transport.TransportConsumerSpec; +import dev.caskeleton.messaging.transport.TransportDelivery; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import eu.rekawek.toxiproxy.Proxy; +import eu.rekawek.toxiproxy.ToxiproxyClient; +import eu.rekawek.toxiproxy.model.Toxic; +import eu.rekawek.toxiproxy.model.ToxicDirection; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Network; +import org.testcontainers.kafka.KafkaContainer; +import org.testcontainers.toxiproxy.ToxiproxyContainer; + +/** + * The certification lane: every fault scenario this adapter claims, run against a real broker, with + * the run's own output as the evidence. + * + *

The claim used to be a list. {@code CertifiedEvidence} named three scenarios as + * live-broker-covered, the compatibility matrix derived "Stable" from that list, and the support + * matrix repeated it — none of which required a container to have started. This suite is where the + * list comes from now: each test proves one scenario and appends one {@link + * BrokerCertificationEvidence} line, and the manifest the release gate compares is what these tests + * wrote. + * + *

No {@code @EnabledIf} on Docker, deliberately, and that is the difference between this suite + * and the other integration suites in this module. A developer's {@code test} run may skip a + * container suite with a stated reason; a certification lane that skips reports success for a + * broker nobody started, which is the exact failure the evidence exists to rule out. The tag keeps + * it out of {@code test} so that skipping is never necessary. + * + *

Faults are injected with Toxiproxy in front of the broker rather than by pausing the + * container. A pause stalls every scenario the same way; the distinction the scenarios draw — a cut + * connection, a response that never arrives, a response that arrives too late — only exists at the + * socket. The broker's advertised listener points at the proxy, so the client's data path really + * does run through it instead of reconnecting straight to the container. + * + *

{@code connection-refused} is absent, and its absence is the honest answer rather than an + * oversight: a Kafka producer buffers a record before it learns whether a connection exists, so a + * refused connection surfaces as a delivery timeout that proves nothing about transmission. + * Reporting it as {@code REJECTED} would be the guess this platform refuses to make, so the + * scenario stays an uncovered gap that {@code CertifiedEvidence.knownGaps} names. + */ +@Tag("messaging-certification") +class KafkaBrokerCertificationIT { + + private static final String ADAPTER = "messaging-kafka"; + + /** + * The certified image, pinned rather than floating. + * + *

A mutable tag makes a red run unattributable and a green one unrepeatable: the evidence + * records the image it ran against, so the image has to be a fact and not "whatever {@code + * latest} resolved to that morning". + */ + private static final String IMAGE = + System.getProperty("messaging.kafka.image", "apache/kafka:4.1.0"); + + private static final String TOXIPROXY_IMAGE = + System.getProperty("messaging.toxiproxy.image", "ghcr.io/shopify/toxiproxy:2.12.0"); + + /** The in-network listener the proxy forwards to. */ + private static final int BROKER_PORT = 19092; + + /** + * The first port the Toxiproxy container publishes, used for the single proxy this lane needs. + */ + private static final int PROXY_PORT = 8666; + + /** + * The publish deadline every fault is measured against. + * + *

Short on purpose: each scenario resolves inside its own test rather than inside the suite + * timeout, and the injected latency below only has to exceed this rather than a production + * default. + */ + private static final Duration DELIVERY_TIMEOUT = Duration.ofSeconds(4); + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + private static final List OBSERVED = new CopyOnWriteArrayList<>(); + + private static Network network; + private static ToxiproxyContainer toxiproxy; + private static KafkaContainer kafka; + private static Proxy brokerProxy; + private static String bootstrapServers; + + @BeforeAll + static void startBrokerBehindAProxy() throws IOException { + network = Network.newNetwork(); + toxiproxy = new ToxiproxyContainer(TOXIPROXY_IMAGE).withNetwork(network); + toxiproxy.start(); + + ToxiproxyClient toxiproxyClient = + new ToxiproxyClient(toxiproxy.getHost(), toxiproxy.getControlPort()); + brokerProxy = + toxiproxyClient.createProxy( + "messaging-kafka", "0.0.0.0:" + PROXY_PORT, "kafka:" + BROKER_PORT); + bootstrapServers = toxiproxy.getHost() + ":" + toxiproxy.getMappedPort(PROXY_PORT); + + kafka = + new KafkaContainer(IMAGE) + .withNetwork(network) + .withNetworkAliases("kafka") + // The advertised listener is the proxy, so a client that follows the broker's own + // metadata still arrives through the fault injector. Advertising the container instead + // would let every connection after the bootstrap bypass the proxy, and the suite would + // certify a data path it never faulted. + .withListener("kafka:" + BROKER_PORT, () -> bootstrapServers); + kafka.start(); + } + + @AfterEach + void clearInjectedFaults() throws IOException { + for (Toxic toxic : brokerProxy.toxics().getAll()) { + toxic.remove(); + } + brokerProxy.enable(); + } + + @AfterAll + static void writeEvidence() throws IOException { + Path manifest = + Path.of( + System.getProperty( + "messaging.certification.manifest", + "build/messaging-certification/broker-certification-evidence.jsonl")); + Files.createDirectories(manifest.toAbsolutePath().getParent()); + List lines = + OBSERVED.stream() + .sorted(java.util.Comparator.comparing(evidence -> evidence.scenario().name())) + .map(BrokerCertificationEvidence::toJson) + .toList(); + Files.write(manifest, lines, StandardCharsets.UTF_8); + + if (kafka != null) { + kafka.stop(); + } + if (toxiproxy != null) { + toxiproxy.stop(); + } + if (network != null) { + network.close(); + } + } + + @Test + void aConnectionCutAfterTheWriteIsReportedAsAmbiguous() throws IOException { + String topic = createTopic("certification.cut"); + try (Producer producer = producer()) { + KafkaMessagingTransport transport = new KafkaMessagingTransport("kafka-primary", 1, producer); + // A warm-up publish so metadata and the broker connection already exist. Without it the + // failure under test would be a metadata lookup that never started, which is a different + // scenario with a different honest answer. + assertThat(publish(transport, topic, "warm-up").completion()) + .isEqualTo(PublishCompletion.CONFIRMED); + + brokerProxy.disable(); + PublishResult result = publish(transport, topic, "cut"); + + assertAmbiguous(result); + record( + NetworkFaultScenario.CONNECTION_CUT_AFTER_WRITE, + "KafkaBrokerCertificationIT#aConnectionCutAfterTheWriteIsReportedAsAmbiguous", + result); + } + } + + @Test + void aConfirmThatNeverArrivesIsReportedAsAmbiguousAndTheRecordIsThere() throws IOException { + String topic = createTopic("certification.confirm"); + try (Producer producer = producer()) { + KafkaMessagingTransport transport = new KafkaMessagingTransport("kafka-primary", 1, producer); + assertThat(publish(transport, topic, "warm-up").completion()) + .isEqualTo(PublishCompletion.CONFIRMED); + + // Only the broker's answers are stopped. The produce request still lands, so this is the + // case where the record really is stored and only the acknowledgement is missing — the exact + // state in which reporting a rejection would lose a message the broker is holding. + brokerProxy.toxics().timeout("confirm-lost", ToxicDirection.DOWNSTREAM, 0); + PublishResult result = publish(transport, topic, "confirm-lost"); + assertAmbiguous(result); + + clearInjectedFaults(); + assertThat(recordCount(topic)) + .as("the broker kept both records, so AMBIGUOUS was the true answer and REJECTED a lie") + .isEqualTo(2); + + record( + NetworkFaultScenario.CONFIRM_TIMEOUT, + "KafkaBrokerCertificationIT#aConfirmThatNeverArrivesIsReportedAsAmbiguousAndTheRecordIsThere", + result); + } + } + + @Test + void latencyBeyondThePublishDeadlineIsReportedAsAmbiguous() throws IOException { + String topic = createTopic("certification.latency"); + try (Producer producer = producer()) { + KafkaMessagingTransport transport = new KafkaMessagingTransport("kafka-primary", 1, producer); + assertThat(publish(transport, topic, "warm-up").completion()) + .isEqualTo(PublishCompletion.CONFIRMED); + + // Slow, not broken: nothing is dropped and no connection closes. At the moment the deadline + // expires the caller cannot tell this apart from a lost confirm, and the platform's answer + // must be the same for both. + brokerProxy + .toxics() + .latency("high-latency", ToxicDirection.DOWNSTREAM, DELIVERY_TIMEOUT.toMillis() * 2); + PublishResult result = publish(transport, topic, "slow"); + + assertAmbiguous(result); + record( + NetworkFaultScenario.HIGH_LATENCY, + "KafkaBrokerCertificationIT#latencyBeyondThePublishDeadlineIsReportedAsAmbiguous", + result); + } + } + + @Test + void aDeliveryLostBeforeSettlementIsRedelivered() throws IOException { + String topic = createTopic("certification.settlement"); + String group = "certification-" + System.nanoTime(); + try (Producer producer = producer()) { + KafkaMessagingTransport transport = new KafkaMessagingTransport("kafka-primary", 1, producer); + assertThat(publish(transport, topic, "settle-me").completion()) + .isEqualTo(PublishCompletion.CONFIRMED); + } + + List firstAttempt = new ArrayList<>(); + try (Consumer first = consumer(group)) { + KafkaConsumerRegistrar registrar = registrar(topic, first, firstAttempt, false); + registrar.subscribe(topic); + drain(registrar, firstAttempt); + // The broker disappears while the delivery is unsettled. That is what makes the settlement + // lost rather than merely withheld: the consumer had the record and no acknowledgement could + // reach the broker even if the handler had produced one. + brokerProxy.disable(); + registrar.pollOnce(NOW); + brokerProxy.enable(); + + assertThat(registrar.committedOffset(new org.apache.kafka.common.TopicPartition(topic, 0))) + .as("nothing was committed, so the record is still owed to this group") + .isEmpty(); + registrar.close(); + } + + assertThat(firstAttempt).as("the first consumer received the record").hasSize(1); + + List secondAttempt = new ArrayList<>(); + try (Consumer second = consumer(group)) { + KafkaConsumerRegistrar registrar = registrar(topic, second, secondAttempt, true); + registrar.subscribe(topic); + // Longer than the other drains: this consumer is joining a group whose previous member left + // under a network fault, so the first poll may sit through a rebalance rather than a fetch. + drain(registrar, secondAttempt, 80); + registrar.close(); + } + + assertThat(secondAttempt) + .as("an unsettled delivery is redelivered; the handler is idempotent, not the platform") + .hasSize(1); + record( + NetworkFaultScenario.SETTLEMENT_LOST, + "KafkaBrokerCertificationIT#aDeliveryLostBeforeSettlementIsRedelivered", + NetworkFaultScenario.Expectation.REDELIVERED); + } + + private static void assertAmbiguous(PublishResult result) { + assertThat(result.completion()).isEqualTo(PublishCompletion.AMBIGUOUS); + assertThat(result.evidence().transmission()) + .isEqualTo(TransmissionEvidence.MAY_HAVE_BEEN_TRANSMITTED); + assertThat(result.evidence().brokerAccepted()) + .as("nothing proves the broker took it") + .isFalse(); + assertThat(result.evidence().confirmationLevel()).isEqualTo(ConfirmationLevel.NONE); + assertThat(result.mayHaveBeenStored()).isTrue(); + assertThat(result.failure().orElseThrow().retryable()) + .as("an ambiguous publish is not silently retried; the caller decides, under the same id") + .isFalse(); + } + + /** + * Records what the run observed, translated from the platform's own reported completion. + * + *

Translated rather than asserted equal to the scenario's expectation, because the two are + * compared later by {@code BrokerFailureMatrix.from}: evidence whose outcome contradicts the + * scenario it claims is refused there, and hard-coding the expected value here would remove the + * only place that check has anything to check. + */ + private static void record(NetworkFaultScenario scenario, String testId, PublishResult result) { + NetworkFaultScenario.Expectation observed = + switch (result.completion()) { + case AMBIGUOUS -> NetworkFaultScenario.Expectation.AMBIGUOUS; + case REJECTED -> NetworkFaultScenario.Expectation.REJECTED; + case CONFIRMED -> + throw new AssertionError( + "a confirmed publish is not evidence of a fault scenario: " + scenario.name()); + }; + record(scenario, testId, observed); + } + + private static void record( + NetworkFaultScenario scenario, String testId, NetworkFaultScenario.Expectation observed) { + OBSERVED.add( + new BrokerCertificationEvidence( + ADAPTER, + IMAGE, + scenario, + testId, + observed.name(), + System.getProperty("messaging.certification.commit", "local"), + Instant.now())); + } + + private static PublishResult publish( + KafkaMessagingTransport transport, String topic, String key) { + return transport + .publish( + new TransportPublishRequest( + KafkaFixtureProfiles.withTopic(topic), + KafkaTransactionFixtures.delivery(topic, key).outputs().get(0).envelope(), + PublishOptions.defaults())) + .toCompletableFuture() + .join() + .result(); + } + + private static String createTopic(String prefix) { + String topic = prefix + '.' + System.nanoTime(); + try (Admin admin = + Admin.create(Map.of(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers))) { + admin + .createTopics(List.of(new NewTopic(topic, 1, (short) 1))) + .all() + .get(30, TimeUnit.SECONDS); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted creating " + topic, interrupted); + } catch (java.util.concurrent.ExecutionException | java.util.concurrent.TimeoutException e) { + throw new IllegalStateException("could not create " + topic, e); + } + return topic; + } + + private static int recordCount(String topic) { + try (Consumer consumer = consumer("count-" + System.nanoTime())) { + consumer.subscribe(List.of(topic)); + int seen = 0; + for (int attempt = 0; attempt < 10; attempt++) { + seen += consumer.poll(Duration.ofMillis(500)).count(); + } + return seen; + } + } + + private static Producer producer() { + Properties config = new Properties(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + // The Stable profile's durability. A lane that quietly weakened these would certify a + // configuration the platform refuses to run. + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true); + config.put(ProducerConfig.ACKS_CONFIG, "all"); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, (int) DELIVERY_TIMEOUT.toMillis()); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 2_000); + config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, (int) DELIVERY_TIMEOUT.toMillis()); + return new KafkaProducer<>(config); + } + + private static Consumer consumer(String group) { + Properties config = new Properties(); + config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + config.put(ConsumerConfig.GROUP_ID_CONFIG, group); + config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + config.put( + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); + // Short group timeouts. The settlement scenario kills the connection under a live member, so + // its departure may reach the broker as a session expiry rather than as a clean leave; with the + // defaults the replacement consumer would then wait three quarters of a minute for the group to + // rebalance and the suite would time out before the redelivery it is certifying ever happened. + config.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 6_000); + config.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 2_000); + config.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 20_000); + return new KafkaConsumer<>(config); + } + + private static KafkaConsumerRegistrar registrar( + String topic, + Consumer consumer, + List handled, + boolean settle) { + return new KafkaConsumerRegistrar( + new TransportConsumerSpec( + KafkaFixtureProfiles.withTopic(topic), + delivery -> { + handled.add(delivery); + if (settle) { + delivery.settlement().acknowledge().toCompletableFuture().join(); + } + return CompletableFuture.completedFuture(null); + }), + consumer, + Runnable::run, + new GracefulShutdownCoordinator(Duration.ofSeconds(30)), + Duration.ofMillis(500)); + } + + private static void drain(KafkaConsumerRegistrar registrar, List handled) { + drain(registrar, handled, 20); + } + + private static void drain( + KafkaConsumerRegistrar registrar, List handled, int attempts) { + for (int attempt = 0; attempt < attempts && handled.isEmpty(); attempt++) { + registrar.pollOnce(NOW); + } + // One more cycle so the settlements the handler queued are applied and committed. + registrar.pollOnce(NOW); + } +} diff --git a/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaEnvelopeRoundTripTest.java b/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaEnvelopeRoundTripTest.java new file mode 100644 index 0000000..b765d3e --- /dev/null +++ b/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaEnvelopeRoundTripTest.java @@ -0,0 +1,154 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.error.FailureCategory; +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.api.header.HeaderName; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.common.record.TimestampType; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * What an envelope still holds after it has been through Kafka. + * + *

Publishing and consuming were written against each other's headers but never checked as one + * trip, so what the producer wrote and what the consumer rebuilt were free to disagree — and they + * did, in the two ways that matter operationally: the application's own headers vanished, and the + * retry bookkeeping vanished with them, which restarts an attempt counter that is supposed to cap + * redeliveries. + */ +class KafkaEnvelopeRoundTripTest { + + private static final Instant FAILED_AT = Instant.parse("2026-08-15T10:00:00Z"); + + private final KafkaHeaderMapper headerMapper = new KafkaHeaderMapper(); + + private final KafkaDeliveryMapper deliveryMapper = new KafkaDeliveryMapper(headerMapper); + + private final KafkaRetryMetadataMapper retryMapper = new KafkaRetryMetadataMapper(); + + @Test + @DisplayName("an application header survives the trip through the broker") + void anApplicationHeaderSurvivesTheTrip() { + MessageEnvelope published = + KafkaFixtures.orderCreatedEnvelope() + .withHeaders( + MessageHeaders.application( + Map.of(new HeaderName("x-source-system"), new HeaderValue("erp")))); + + MessageEnvelope delivered = roundTrip(published); + + assertThat(delivered.headers().find("x-source-system")) + .map(HeaderValue::value) + .as("a header the producer wrote and the consumer cannot read is a header nobody has") + .hasValue("erp"); + } + + @Test + @DisplayName("the tenant survives the trip instead of arriving as no tenant at all") + void theTenantSurvivesTheTrip() { + MessageEnvelope published = + KafkaFixtures.orderCreatedEnvelope() + .withTenant(new dev.caskeleton.messaging.api.TenantContext("acme")); + + MessageEnvelope delivered = roundTrip(published); + + assertThat(delivered.tenantContext()) + .as("a broker record carries no ambient tenant, so an unwritten tenant is an absent one") + .contains(new dev.caskeleton.messaging.api.TenantContext("acme")); + } + + @Test + @DisplayName("the retry attempt survives the trip instead of restarting at one") + void theRetryAttemptSurvivesTheTrip() { + MessageEnvelope retried = + retryMapper.stamp(KafkaFixtures.orderCreatedEnvelope(), 3, failure(), FAILED_AT); + + MessageEnvelope delivered = roundTrip(retried); + + assertThat(retryMapper.attemptOf(delivered)) + .as("an attempt counter that resets on every hop caps nothing") + .isEqualTo(3); + } + + @Test + @DisplayName("a retry-stamped envelope can be published to its retry topic") + void aRetryStampedEnvelopeCanBePublished() { + MessageEnvelope retried = + retryMapper.stamp(KafkaFixtures.orderCreatedEnvelope(), 2, failure(), FAILED_AT); + + assertThatCode(() -> headerMapper.toKafkaHeaders(retried)) + .as("the platform's own bookkeeping is not message content forging an identity") + .doesNotThrowAnyException(); + } + + @Test + @DisplayName("a header restating the message id is still refused") + void aForgedMessageIdIsStillRefused() { + MessageEnvelope forged = + KafkaFixtures.orderCreatedEnvelope() + .withHeaders( + MessageHeaders.platform( + Map.of(new HeaderName(ReservedHeaders.MESSAGE_ID), new HeaderValue("forged")))); + + assertThatThrownBy(() -> headerMapper.toKafkaHeaders(forged)) + .isInstanceOf(MessagingConfigurationException.class); + } + + @Test + @DisplayName("identity is read from its field, not duplicated into the header map") + void identityIsNotDuplicatedIntoTheHeaderMap() { + MessageEnvelope delivered = roundTrip(KafkaFixtures.orderCreatedEnvelope()); + + assertThat(delivered.headers().find(ReservedHeaders.MESSAGE_ID)) + .as("a copy that can drift from the field gives the same message two identities") + .isEmpty(); + assertThat(delivered.messageId()).isNotNull(); + } + + private MessageEnvelope roundTrip(MessageEnvelope published) { + Headers wire = headerMapper.toKafkaHeaders(published); + return deliveryMapper.toEnvelope(record(wire, published.payload().bytes())); + } + + private static ConsumerRecord record(Headers wire, byte[] payload) { + RecordHeaders headers = new RecordHeaders(); + wire.forEach(header -> headers.add(header.key(), header.value())); + return new ConsumerRecord<>( + "order-events", + 0, + 7L, + FAILED_AT.toEpochMilli(), + TimestampType.CREATE_TIME, + 0, + payload.length, + null, + payload, + headers, + Optional.empty()); + } + + private static FailureDescriptor failure() { + return new FailureDescriptor( + FailureCategory.PROCESSING_TRANSIENT, + "HANDLER_FAILED", + true, + "IllegalStateException", + Optional.empty()); + } +} diff --git a/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/ReservedHeaderForgeryTest.java b/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/ReservedHeaderForgeryTest.java index b60045b..8d19484 100644 --- a/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/ReservedHeaderForgeryTest.java +++ b/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/ReservedHeaderForgeryTest.java @@ -41,7 +41,7 @@ class ReservedHeaderForgeryTest { assertThatThrownBy( () -> mapper.toKafkaHeaders(envelopeWithHeader(ReservedHeaders.MESSAGE_ID, "forged"))) .isInstanceOf(MessagingConfigurationException.class) - .hasMessageContaining("reserved namespace"); + .hasMessageContaining("restates a value the envelope already carries"); } @Test diff --git a/messaging/messaging-nats-experimental/src/test/java/dev/caskeleton/messaging/nats/NatsAdapterContractTest.java b/messaging/messaging-nats-experimental/src/test/java/dev/caskeleton/messaging/nats/NatsAdapterContractTest.java index 1c0edc5..f2f909d 100644 --- a/messaging/messaging-nats-experimental/src/test/java/dev/caskeleton/messaging/nats/NatsAdapterContractTest.java +++ b/messaging/messaging-nats-experimental/src/test/java/dev/caskeleton/messaging/nats/NatsAdapterContractTest.java @@ -285,8 +285,7 @@ class NatsAdapterContractTest { return new TransportPublishRequest( destinationProfile(), envelope(64), - new PublishOptions( - timeout, defaults.confirmation(), defaults.deduplication(), defaults.brokerHints())); + new PublishOptions(timeout, defaults.confirmation(), defaults.deduplication())); } private static DestinationProfile destinationProfile() { diff --git a/messaging/messaging-observability/build.gradle b/messaging/messaging-observability/build.gradle index 8e5b04d..a7ec46f 100644 --- a/messaging/messaging-observability/build.gradle +++ b/messaging/messaging-observability/build.gradle @@ -7,22 +7,3 @@ dependencies { // naming the type. api 'io.micrometer:micrometer-core' } - -// Wave 0 — a report, not a gate. The secret-scanner characterization lives in this module. -tasks.register('wave0Red', Test) { - description = 'Reports which Wave 0 baseline characterizations are still red in this module.' - group = 'verification' - testClassesDirs = sourceSets.test.output.classesDirs - classpath = sourceSets.test.runtimeClasspath - useJUnitPlatform { includeTags 'wave0-red' } - ignoreFailures = true - outputs.upToDateWhen { false } - reports.junitXml.required = true - reports.junitXml.outputLocation = layout.buildDirectory.dir('test-results/wave0Red') -} - -tasks.named('test') { - // See app-bootstrap: the Wave 0 characterizations are reported by wave0RedReport rather than - // failing every build until the wave that closes them lands. - useJUnitPlatform { excludeTags 'wave0-red' } -} diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxEventRouter.java b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxEventRouter.java index efd16f6..dbb64a8 100644 --- a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxEventRouter.java +++ b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxEventRouter.java @@ -1,6 +1,7 @@ package dev.caskeleton.messaging.outbox; import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata; import dev.caskeleton.messaging.reliability.OutboxRecord; import java.util.LinkedHashMap; import java.util.Map; @@ -16,12 +17,29 @@ import java.util.Objects; *

The platform contract does not change with the transport. The router emits the same reserved * headers the polling relay does, so a consumer cannot tell which one published a message, and * switching between them is a deployment decision rather than a contract change. + * + *

The message key and the topic are two different questions and used to be answered by one + * column. {@code destination} routed the topic and became the key, so every message on a + * topic carried the same key, landed on one partition, and made keyed ordering a claim with no + * mechanism behind it. The key now comes from {@code routing_key}, the stored column that states + * the partition-key-or-message-id fallback once for both relays, and {@code destination} stays what + * it always was: routing metadata. + * + *

One canonical field has no header here: the schema subject. There is no reserved header name + * for it, and inventing one that only this path wrote would replace a symmetric gap with an + * asymmetric one — the polling relay would carry a schema reference on the envelope and the CDC + * relay a header no adapter reads. It stays a column that the polling relay resolves, and closing + * it means adding the name to {@code ReservedHeaders} and teaching both broker header mappers to + * write and read it. */ public final class DebeziumOutboxEventRouter { - /** The outbox column Debezium uses as the message key. */ + /** The outbox column Debezium routes the topic by. */ public static final String ROUTE_BY_COLUMN = "destination"; + /** The outbox column Debezium uses as the message key. */ + public static final String KEY_COLUMN = "routing_key"; + /** The outbox column carrying the logical message id. */ public static final String ID_COLUMN = "message_id"; @@ -39,7 +57,7 @@ public final class DebeziumOutboxEventRouter { Map configuration = new LinkedHashMap<>(); configuration.put("transforms.outbox.type", "io.debezium.transforms.outbox.EventRouter"); configuration.put("transforms.outbox.table.field.event.id", ID_COLUMN); - configuration.put("transforms.outbox.table.field.event.key", ROUTE_BY_COLUMN); + configuration.put("transforms.outbox.table.field.event.key", KEY_COLUMN); configuration.put("transforms.outbox.table.field.event.payload", PAYLOAD_COLUMN); configuration.put("transforms.outbox.route.by.field", ROUTE_BY_COLUMN); configuration.put( @@ -51,7 +69,18 @@ public final class DebeziumOutboxEventRouter { "message_type:header:" + ReservedHeaders.MESSAGE_TYPE, "schema_version:header:" + ReservedHeaders.SCHEMA_VERSION, "content_type:header:" + ReservedHeaders.CONTENT_TYPE, - "created_at:header:" + ReservedHeaders.PRODUCED_AT)); + "created_at:header:" + ReservedHeaders.PRODUCED_AT, + "destination:header:" + ReservedHeaders.ORIGIN_DESTINATION, + "producer:header:" + ReservedHeaders.PRODUCER, + "occurred_at:header:" + ReservedHeaders.OCCURRED_AT, + "correlation_id:header:" + ReservedHeaders.CORRELATION_ID, + "causation_id:header:" + ReservedHeaders.CAUSATION_ID, + "tenant:header:" + ReservedHeaders.TENANT, + "partition_key:header:" + ReservedHeaders.PARTITION_KEY, + "ordering_key:header:" + ReservedHeaders.ORDERING_KEY, + "traceparent:header:" + ReservedHeaders.TRACEPARENT, + "tracestate:header:" + ReservedHeaders.TRACESTATE, + "baggage:header:" + ReservedHeaders.BAGGAGE)); return Map.copyOf(configuration); } @@ -71,6 +100,52 @@ public final class DebeziumOutboxEventRouter { headers.put(ReservedHeaders.SCHEMA_VERSION, Integer.toString(record.schemaVersion().value())); headers.put(ReservedHeaders.CONTENT_TYPE, record.contentType().value()); headers.put(ReservedHeaders.PRODUCED_AT, record.createdAt().toString()); + headers.put(ReservedHeaders.ORIGIN_DESTINATION, record.destination().value()); + + // Absent stays absent. Writing an empty string for a column the row does not have would make a + // message that carries no tenant indistinguishable from one whose tenant is the empty string, + // and the consumer would have to know which of the two this platform meant. + OutboxCanonicalMetadata metadata = record.metadata(); + metadata.producer().ifPresent(value -> headers.put(ReservedHeaders.PRODUCER, value.value())); + metadata + .occurredAt() + .ifPresent(value -> headers.put(ReservedHeaders.OCCURRED_AT, value.toString())); + metadata + .correlationId() + .ifPresent(value -> headers.put(ReservedHeaders.CORRELATION_ID, value.value())); + metadata + .causationId() + .ifPresent( + value -> headers.put(ReservedHeaders.CAUSATION_ID, value.value().value().toString())); + metadata.tenant().ifPresent(value -> headers.put(ReservedHeaders.TENANT, value.tenantId())); + metadata.partitionKey().ifPresent(value -> headers.put(ReservedHeaders.PARTITION_KEY, value)); + metadata.orderingKey().ifPresent(value -> headers.put(ReservedHeaders.ORDERING_KEY, value)); + metadata + .traceContext() + .traceparent() + .ifPresent(value -> headers.put(ReservedHeaders.TRACEPARENT, value)); + metadata + .traceContext() + .tracestate() + .ifPresent(value -> headers.put(ReservedHeaders.TRACESTATE, value)); + metadata + .traceContext() + .baggage() + .ifPresent(value -> headers.put(ReservedHeaders.BAGGAGE, value)); return Map.copyOf(headers); } + + /** + * Returns the key a routed record carries. + * + *

The same value the {@code routing_key} column computes, stated here so a test can compare + * the two rather than trusting that the SQL and the Java agree. + * + * @param record the outbox row + * @return the broker message key + */ + public String routingKey(OutboxRecord record) { + Objects.requireNonNull(record, "record must not be null"); + return record.metadata().partitionKey().orElseGet(() -> record.messageId().value().toString()); + } } diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapper.java b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapper.java index a1480fb..513f246 100644 --- a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapper.java +++ b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapper.java @@ -19,11 +19,17 @@ import java.util.Optional; *

The message id comes from the row's {@code message_id} column, never from Debezium's own event * id. Debezium mints a fresh event id per change record, so using it would break identity across a * CDC re-snapshot — exactly the scenario where identity matters most. + * + *

The reserved headers come from {@link DebeziumOutboxEventRouter} rather than being rebuilt + * here. Two lists of the same headers is how the connector's configuration and the expectation this + * class encodes come to disagree, and the disagreement would be invisible: the test compares this + * output against the polling relay's, so it would pass while the connector emitted something else. */ public final class DebeziumOutboxRecordMapper { private final String topicPrefix; private final boolean aggregateIdAsPartitionKey; + private final DebeziumOutboxEventRouter router = new DebeziumOutboxEventRouter(); /** * Creates a mapper from a CDC profile. @@ -45,27 +51,29 @@ public final class DebeziumOutboxRecordMapper { public DebeziumMappedRecord map(OutboxRecord record) { Objects.requireNonNull(record, "record must not be null"); + // Application headers first, canonical headers second, and the canonical ones win. The row's + // header map is application data; a row that wrote `msg.id` into it would otherwise overwrite + // the identity the platform put there, and the consumer trusts the last such header it sees. + // The write path refuses reserved names outright — this order is the second lock on the same + // door, because a CDC connector reading the table directly never passes through that check. Map headers = new LinkedHashMap<>(record.headers()); - headers.put(ReservedHeaders.MESSAGE_ID, record.messageId().value().toString()); - headers.put(ReservedHeaders.MESSAGE_TYPE, record.messageType().value()); - headers.put(ReservedHeaders.SCHEMA_VERSION, Integer.toString(record.schemaVersion().value())); - headers.put(ReservedHeaders.CONTENT_TYPE, record.contentType().value()); - headers.put(ReservedHeaders.PRODUCED_AT, record.createdAt().toString()); + headers.keySet().removeIf(ReservedHeaders::isReserved); + headers.putAll(router.reservedHeaders(record)); return new DebeziumMappedRecord( topicPrefix + record.destination().value(), - partitionKey(record, headers), + partitionKey(record), record.payload(), headers); } - private Optional partitionKey(OutboxRecord record, Map headers) { + private Optional partitionKey(OutboxRecord record) { if (!aggregateIdAsPartitionKey) { return Optional.empty(); } - // The partition key header is the one the polling relay would also have set, so both relays - // land the same message on the same partition and neither reorders a keyed stream. - return Optional.ofNullable(headers.get(ReservedHeaders.PARTITION_KEY)) - .or(() -> Optional.of(record.messageId().value().toString())); + // The same value the polling relay would use and the same value the `routing_key` column + // computes, so both relays land the same message on the same partition and neither reorders a + // keyed stream. + return Optional.of(router.routingKey(record)); } } diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/JdbcOutboxRepository.java b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/JdbcOutboxRepository.java index f4351c7..ec88cba 100644 --- a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/JdbcOutboxRepository.java +++ b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/JdbcOutboxRepository.java @@ -1,11 +1,17 @@ package dev.caskeleton.messaging.outbox; +import dev.caskeleton.messaging.api.CausationId; import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.CorrelationId; import dev.caskeleton.messaging.api.MessageId; import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TenantContext; +import dev.caskeleton.messaging.api.TraceContext; import dev.caskeleton.messaging.api.destination.DestinationName; import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata; import dev.caskeleton.messaging.reliability.OutboxLease; import dev.caskeleton.messaging.reliability.OutboxRecord; import dev.caskeleton.messaging.reliability.OutboxRepository; @@ -45,10 +51,32 @@ public final class JdbcOutboxRepository implements OutboxRepository { """ INSERT INTO messaging_outbox ( message_id, destination, message_type, schema_version, content_type, - payload, headers, created_at, status, attempts) - VALUES (?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?, ?) + payload, headers, created_at, status, attempts, + producer, occurred_at, correlation_id, causation_id, tenant, + partition_key, ordering_key, schema_subject, schema_uri, + traceparent, tracestate, baggage) + VALUES (?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?, ?, + ?, ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?) """; + /** + * The columns every read of a row selects. + * + *

One constant rather than four copies of the list. The lease, claim, find and reclaim + * statements each spelled it out, and a column added to one of them and not the others is a + * message that arrives with its tenant on the claim path and without it on the lease path — a + * difference that shows up as a data bug in whichever consumer happened to be reading. + */ + private static final String COLUMNS = + """ + message_id, destination, message_type, schema_version, content_type, + payload, headers, created_at, status, attempts, lease_expires_at, last_failure_code, + producer, occurred_at, correlation_id, causation_id, tenant, + partition_key, ordering_key, schema_subject, schema_uri, + traceparent, tracestate, baggage"""; + private static final String LEASE = """ WITH claimable AS ( @@ -69,7 +97,10 @@ public final class JdbcOutboxRepository implements OutboxRepository { WHERE o.message_id = c.message_id RETURNING o.message_id, o.destination, o.message_type, o.schema_version, o.content_type, o.payload, o.headers, o.created_at, o.status, o.attempts, - o.lease_expires_at, o.last_failure_code + o.lease_expires_at, o.last_failure_code, + o.producer, o.occurred_at, o.correlation_id, o.causation_id, o.tenant, + o.partition_key, o.ordering_key, o.schema_subject, o.schema_uri, + o.traceparent, o.tracestate, o.baggage """; /** @@ -104,7 +135,10 @@ public final class JdbcOutboxRepository implements OutboxRepository { WHERE o.message_id = c.message_id RETURNING o.message_id, o.destination, o.message_type, o.schema_version, o.content_type, o.payload, o.headers, o.created_at, o.status, o.attempts, - o.lease_expires_at, o.last_failure_code, o.lease_owner, o.lease_token + o.lease_expires_at, o.last_failure_code, o.lease_owner, o.lease_token, + o.producer, o.occurred_at, o.correlation_id, o.causation_id, o.tenant, + o.partition_key, o.ordering_key, o.schema_subject, o.schema_uri, + o.traceparent, o.tracestate, o.baggage """; private final DataSource dataSource; @@ -144,6 +178,20 @@ public final class JdbcOutboxRepository implements OutboxRepository { statement.setTimestamp(8, Timestamp.from(record.createdAt())); statement.setString(9, record.status().name()); statement.setInt(10, record.attempts()); + OutboxCanonicalMetadata metadata = record.metadata(); + statement.setString(11, metadata.producer().map(ProducerId::value).orElse(null)); + statement.setTimestamp(12, metadata.occurredAt().map(Timestamp::from).orElse(null)); + statement.setString(13, metadata.correlationId().map(CorrelationId::value).orElse(null)); + statement.setObject( + 14, metadata.causationId().map(causation -> causation.value().value()).orElse(null)); + statement.setString(15, metadata.tenant().map(TenantContext::tenantId).orElse(null)); + statement.setString(16, metadata.partitionKey().orElse(null)); + statement.setString(17, metadata.orderingKey().orElse(null)); + statement.setString(18, metadata.schemaSubject().orElse(null)); + statement.setString(19, metadata.schemaUri().orElse(null)); + statement.setString(20, metadata.traceContext().traceparent().orElse(null)); + statement.setString(21, metadata.traceContext().tracestate().orElse(null)); + statement.setString(22, metadata.traceContext().baggage().orElse(null)); statement.executeUpdate(); } catch (SQLException exception) { throw new MessagingConfigurationException( @@ -425,9 +473,7 @@ public final class JdbcOutboxRepository implements OutboxRepository { connection -> { try (PreparedStatement statement = connection.prepareStatement( - "SELECT message_id, destination, message_type, schema_version, content_type, " - + "payload, headers, created_at, status, attempts, lease_expires_at, " - + "last_failure_code FROM messaging_outbox WHERE message_id = ?")) { + "SELECT " + COLUMNS + " FROM messaging_outbox WHERE message_id = ?")) { statement.setObject(1, messageId.value()); try (ResultSet results = statement.executeQuery()) { return results.next() ? Optional.of(read(results)) : Optional.empty(); @@ -509,6 +555,7 @@ public final class JdbcOutboxRepository implements OutboxRepository { new ContentType(results.getString("content_type")), results.getBytes("payload"), fromJson(results.getString("headers")), + readMetadata(results), results.getTimestamp("created_at").toInstant(), OutboxStatus.valueOf(results.getString("status")), results.getInt("attempts"), @@ -516,6 +563,33 @@ public final class JdbcOutboxRepository implements OutboxRepository { Optional.ofNullable(results.getString("last_failure_code"))); } + /** + * Rebuilds the canonical metadata from the row's own columns. + * + *

Absent columns stay absent. A row written before the columns existed rebuilds with empty + * metadata, which is the same thing a row written by a producer that had none rebuilds with — and + * the reader's behaviour has to be identical in both cases, because guessing a tenant or a + * correlation is worse than carrying none. + */ + private static OutboxCanonicalMetadata readMetadata(ResultSet results) throws SQLException { + Timestamp occurredAt = results.getTimestamp("occurred_at"); + java.util.UUID causation = results.getObject("causation_id", java.util.UUID.class); + return new OutboxCanonicalMetadata( + Optional.ofNullable(results.getString("producer")).map(ProducerId::new), + Optional.ofNullable(occurredAt).map(Timestamp::toInstant), + Optional.ofNullable(results.getString("correlation_id")).map(CorrelationId::new), + Optional.ofNullable(causation).map(value -> new CausationId(new MessageId(value))), + Optional.ofNullable(results.getString("tenant")).map(TenantContext::new), + Optional.ofNullable(results.getString("partition_key")), + Optional.ofNullable(results.getString("ordering_key")), + Optional.ofNullable(results.getString("schema_subject")), + Optional.ofNullable(results.getString("schema_uri")), + new TraceContext( + Optional.ofNullable(results.getString("traceparent")), + Optional.ofNullable(results.getString("tracestate")), + Optional.ofNullable(results.getString("baggage")))); + } + private T withConnection(ConnectionCallback callback) { try (Connection connection = dataSource.getConnection()) { return callback.apply(connection); diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxEnvelopeFactory.java b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxEnvelopeFactory.java index b3b219a..1e3750c 100644 --- a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxEnvelopeFactory.java +++ b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxEnvelopeFactory.java @@ -2,13 +2,16 @@ package dev.caskeleton.messaging.outbox; import dev.caskeleton.messaging.api.MessageEnvelope; import dev.caskeleton.messaging.api.ProducerId; -import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.SchemaVersion; import dev.caskeleton.messaging.api.header.HeaderName; import dev.caskeleton.messaging.api.header.HeaderValue; import dev.caskeleton.messaging.api.header.MessageHeaders; import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata; import dev.caskeleton.messaging.reliability.OutboxRecord; import dev.caskeleton.messaging.schema.EncodedMessage; +import dev.caskeleton.messaging.schema.SchemaReference; +import java.net.URI; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -20,6 +23,18 @@ import java.util.Optional; *

The identity comes from the row, never from a fresh mint. That is what makes a retry after an * ambiguous publish safe: the broker's deduplication, or the consumer's Inbox, sees the same id and * collapses the duplicate. + * + *

So does everything else the envelope carries. This used to rebuild correlation, causation, + * tenant, trace and the schema reference as empty, and read the routing keys out of the row's + * header map — so a message that travelled through the outbox reached its consumer with less + * provenance than one published directly, and the publish path became part of the message's + * meaning. The row now has columns for those facts and this reads them. + * + *

Reserved header names in the row are refused outright, with no exception for the routing keys. + * They had one, because the keys had nowhere else to live; a row could therefore write {@code + * msg.partition-key} and the same code path would have accepted {@code msg.id} if the check had + * been one line looser. Now that the keys are columns, the rule is the simple one: an outbox row + * cannot write into the platform's namespace at all. */ public final class OutboxEnvelopeFactory { @@ -28,7 +43,7 @@ public final class OutboxEnvelopeFactory { /** * Creates a factory for one service. * - * @param producer the logical producing service + * @param producer the logical producing service, used for rows that name none */ public OutboxEnvelopeFactory(ProducerId producer) { this.producer = Objects.requireNonNull(producer, "producer must not be null"); @@ -52,13 +67,7 @@ public final class OutboxEnvelopeFactory { // MessageHeaders.platform let a row carrying `msg.id` or `msg.type` reach the wire // as if the platform had written it — and the consumer trusts the last such header, // so a forged identity lands in another message's inbox deduplication. - // - // Partition and ordering keys are the exception: they are reserved names the outbox - // legitimately carries as routing metadata, and they are read below rather than - // written as headers. - if (ReservedHeaders.isReserved(name) - && !ReservedHeaders.PARTITION_KEY.equals(name) - && !ReservedHeaders.ORDERING_KEY.equals(name)) { + if (ReservedHeaders.isReserved(name)) { throw new dev.caskeleton.messaging.api.error.MessagingConfigurationException( "RESERVED_HEADER_IN_OUTBOX_ROW", "outbox row header '" @@ -66,26 +75,50 @@ public final class OutboxEnvelopeFactory { + "' uses the reserved namespace; canonical identity and metadata come from" + " the record's own columns, not from its header map"); } - if (!ReservedHeaders.isReserved(name)) { - headers.put(new HeaderName(name), new HeaderValue(value)); - } + headers.put(new HeaderName(name), new HeaderValue(value)); }); + OutboxCanonicalMetadata metadata = record.metadata(); return new MessageEnvelope<>( record.messageId(), record.messageType(), record.schemaVersion(), record.createdAt(), - Optional.of(record.createdAt()), - producer, - Optional.empty(), - Optional.empty(), + // The write instant stands in when the row records no separate occurrence. It is the + // closest true statement available: the business transaction that wrote the row is the one + // the fact occurred in. + Optional.of(metadata.occurredAt().orElse(record.createdAt())), + // A row written before the producer column existed names none, and the service that owns + // this relay is the only producer it can honestly be attributed to. + metadata.producer().orElse(producer), + metadata.correlationId(), + metadata.causationId(), record.contentType(), - Optional.ofNullable(record.headers().get(ReservedHeaders.PARTITION_KEY)), - Optional.ofNullable(record.headers().get(ReservedHeaders.ORDERING_KEY)), - Optional.empty(), - TraceContext.none(), + metadata.partitionKey(), + metadata.orderingKey(), + metadata.tenant(), + metadata.traceContext(), MessageHeaders.platform(headers), - new EncodedMessage(record.payload(), record.contentType(), Optional.empty())); + new EncodedMessage( + record.payload(), record.contentType(), schemaReference(record, metadata))); + } + + /** + * Rebuilds the schema reference from the row's subject, the row's version, and its location. + * + *

The version is the row's {@code schema_version} column rather than a third copy of the same + * number: a reference whose version could disagree with the column the relay filters on is a + * reference nobody can trust. + */ + private static Optional schemaReference( + OutboxRecord record, OutboxCanonicalMetadata metadata) { + return metadata + .schemaSubject() + .map( + subject -> + new SchemaReference( + subject, + new SchemaVersion(record.schemaVersion().value()), + metadata.schemaUri().map(URI::create))); } } diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelay.java b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelay.java index 69246ac..64decb8 100644 --- a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelay.java +++ b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelay.java @@ -87,6 +87,33 @@ public final class OutboxRelay { OutboxRetryScheduler.standard()); } + /** + * A relay with an explicit retry policy and a process-derived owner. + * + * @param repository the outbox storage + * @param publisher the platform publisher + * @param envelopeFactory rebuilds envelopes from rows + * @param batchSize how many rows to claim per pass + * @param leaseDuration how long a claim holds + * @param scheduler the retry policy + */ + public OutboxRelay( + OutboxRepository repository, + MessagePublisher publisher, + OutboxEnvelopeFactory envelopeFactory, + int batchSize, + Duration leaseDuration, + OutboxRetryScheduler scheduler) { + this( + repository, + publisher, + envelopeFactory, + batchSize, + leaseDuration, + defaultOwner(), + scheduler); + } + /** * A relay with an explicit retry policy. * @@ -134,6 +161,7 @@ public final class OutboxRelay { int ambiguous = 0; int failed = 0; int stale = 0; + int exhausted = 0; for (OutboxLease lease : batch) { OutboxRecord record = lease.record(); @@ -147,10 +175,28 @@ public final class OutboxRelay { } } case AMBIGUOUS -> { - if (repository.markAmbiguous( - lease, result.failure().map(failure -> failure.code()).orElse("AMBIGUOUS"), now) - == OutboxTransitionResult.APPLIED) { - ambiguous++; + String code = result.failure().map(failure -> failure.code()).orElse("AMBIGUOUS"); + // The attempt this pass just spent. The claim predicate and the row both count attempts + // after the transition, so the budget has to be judged on the same number the next claim + // will read, or the last attempt is spent twice. + int spent = record.attempts() + 1; + OutboxTransitionResult transition = + scheduler + .parkReason(spent) + // A row that has spent its budget without an answer is parked under its own + // status. Leaving it AMBIGUOUS makes it a row the claim predicate silently skips + // forever, which looks identical to a healthy backlog on every dashboard. + .map(reason -> repository.markExhausted(lease, reason, now)) + .orElseGet( + () -> + repository.markAmbiguous( + lease, code, now, scheduler.nextAttemptAt(now, spent))); + if (transition == OutboxTransitionResult.APPLIED) { + if (scheduler.isExhausted(spent)) { + exhausted++; + } else { + ambiguous++; + } } else { stale++; } @@ -171,7 +217,7 @@ public final class OutboxRelay { throw new IllegalStateException("unhandled publish completion: " + result.completion()); } } - return new OutboxRelayReport(batch.size(), published, ambiguous, failed, stale); + return new OutboxRelayReport(batch.size(), published, ambiguous, failed, stale, exhausted); } private PublishResult publish(OutboxRecord record) { diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelayReport.java b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelayReport.java index 493576c..d91d3f7 100644 --- a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelayReport.java +++ b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelayReport.java @@ -12,23 +12,39 @@ package dev.caskeleton.messaging.outbox; * published something whose outcome another worker has already recorded, so a non-zero value is the * visible form of a duplicate publication. * + *

{@code exhausted} is counted apart from {@code failed} for the same reason: a rejected row is + * a message the broker refused and an exhausted one is a message nobody ever answered for. The + * first needs a fix, the second needs a redrive, and a single counter for both tells an operator + * neither. + * * @param leased how many rows were claimed * @param published how many were confirmed * @param ambiguous how many had an unknown outcome and stay retryable * @param failed how many were definitively rejected * @param staleLeases how many terminal writes were refused because the lease had been superseded + * @param exhausted how many spent their attempt budget without a confirmation and were parked */ public record OutboxRelayReport( - int leased, int published, int ambiguous, int failed, int staleLeases) { + int leased, int published, int ambiguous, int failed, int staleLeases, int exhausted) { public OutboxRelayReport { - if (leased < 0 || published < 0 || ambiguous < 0 || failed < 0 || staleLeases < 0) { + if (leased < 0 + || published < 0 + || ambiguous < 0 + || failed < 0 + || staleLeases < 0 + || exhausted < 0) { throw new IllegalArgumentException("relay counters must not be negative"); } } + /** A report from a pass that parked nothing. */ + public OutboxRelayReport(int leased, int published, int ambiguous, int failed, int staleLeases) { + this(leased, published, ambiguous, failed, staleLeases, 0); + } + /** A report from a pass that observed no stale leases. */ public OutboxRelayReport(int leased, int published, int ambiguous, int failed) { - this(leased, published, ambiguous, failed, 0); + this(leased, published, ambiguous, failed, 0, 0); } } diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelayWorker.java b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelayWorker.java new file mode 100644 index 0000000..5a3ac4c --- /dev/null +++ b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRelayWorker.java @@ -0,0 +1,199 @@ +package dev.caskeleton.messaging.outbox; + +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +/** + * Runs relay passes on the schedule the retry policy decides. + * + *

The relay, its retry scheduler and the attempt budget all existed and nothing ever called + * {@code runOnce}. An outbox whose relay is never driven is the worst shape of all: the business + * transaction commits its row and reports success, and the message is never published — a silent, + * unbounded backlog rather than a visible failure. + * + *

Passes are self-scheduled rather than run at a fixed rate. A fixed rate keeps firing while the + * broker is down, and the pass after an empty or failed one is exactly where the policy's backoff + * belongs; a pass that published something resets to the base interval because work is flowing. + * + *

One thread, and the next pass is scheduled only after the previous returns. Overlapping passes + * from one process would have two claims racing on the same rows, which the fencing predicate would + * then reject as stale — correct, but it is duplicate publishing work bought at the cost of a + * duplicate publish. + */ +public final class OutboxRelayWorker implements AutoCloseable { + + /** How long {@link #close()} waits for a running pass when no deadline is given. */ + public static final Duration DEFAULT_DRAIN_DEADLINE = Duration.ofSeconds(30); + + private final OutboxRelay relay; + + private final OutboxRetryScheduler scheduler; + + private final Supplier clock; + + private final ScheduledExecutorService passes; + + private final AtomicBoolean running = new AtomicBoolean(); + + private final AtomicInteger unproductivePasses = new AtomicInteger(); + + private final AtomicLong completedPasses = new AtomicLong(); + + private final AtomicLong failedPasses = new AtomicLong(); + + /** + * Creates a worker driving one relay. + * + * @param relay the relay to run + * @param scheduler the policy deciding when the next pass runs + */ + public OutboxRelayWorker(OutboxRelay relay, OutboxRetryScheduler scheduler) { + this(relay, scheduler, Instant::now); + } + + /** + * Creates a worker with an explicit clock, so a pass can be driven without waiting on wall time. + * + * @param relay the relay to run + * @param scheduler the policy deciding when the next pass runs + * @param clock the source of the current instant + */ + public OutboxRelayWorker( + OutboxRelay relay, OutboxRetryScheduler scheduler, Supplier clock) { + this.relay = Objects.requireNonNull(relay, "relay must not be null"); + this.scheduler = Objects.requireNonNull(scheduler, "scheduler must not be null"); + this.clock = Objects.requireNonNull(clock, "clock must not be null"); + ScheduledThreadPoolExecutor executor = + new ScheduledThreadPoolExecutor( + 1, + runnable -> { + // Daemon: a relay thread that outlives the application would keep publishing from a + // process that has already released its other resources. + Thread thread = new Thread(runnable, "messaging-outbox-relay"); + thread.setDaemon(true); + return thread; + }); + // A pass waiting out its backoff is dropped on shutdown rather than run afterwards. Left on, + // the default policy makes stop() wait for a pass that has not started and cannot do anything + // useful, so an orderly shutdown pays the whole poll interval before it can close. + executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + this.passes = executor; + } + + /** Starts driving passes. Idempotent: a second call does not add a second schedule. */ + public void start() { + if (running.compareAndSet(false, true)) { + schedule(Duration.ZERO); + } + } + + /** + * Stops scheduling and lets the pass that is already running finish. + * + *

Draining rather than interrupting is the whole point. A pass killed between its claim and + * its terminal write leaves rows {@code IN_FLIGHT} holding a lease, and nothing may touch them + * until that lease expires — so an orderly shutdown would produce exactly the stall that a crash + * produces. + * + * @param deadline how long to wait for the running pass + * @return true when the running pass finished inside the deadline + */ + public boolean stop(Duration deadline) { + Objects.requireNonNull(deadline, "deadline must not be null"); + running.set(false); + passes.shutdown(); + try { + if (passes.awaitTermination(deadline.toMillis(), TimeUnit.MILLISECONDS)) { + return true; + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + // The deadline passed with a pass still running. Its rows stay leased until the lease expires, + // which is the outcome the deadline exists to bound rather than to prevent. + passes.shutdownNow(); + return false; + } + + @Override + public void close() { + stop(DEFAULT_DRAIN_DEADLINE); + } + + /** + * Reports whether the worker is scheduling passes. + * + * @return true between {@link #start()} and {@link #stop(Duration)} + */ + public boolean isRunning() { + return running.get(); + } + + /** + * Returns how many passes have finished, successfully or not. + * + * @return the completed pass count + */ + public long completedPasses() { + return completedPasses.get(); + } + + /** + * Returns how many passes ended in an exception. + * + * @return the failed pass count + */ + public long failedPasses() { + return failedPasses.get(); + } + + // The scheduled future is not kept: a pass reports nothing to a caller and handles its own + // failure below, so the only thing holding it would achieve is a reference that has to be + // cleaned up. + @SuppressWarnings("FutureReturnValueIgnored") + private void schedule(Duration delay) { + if (!running.get()) { + return; + } + try { + passes.schedule(this::runPass, Math.max(0, delay.toMillis()), TimeUnit.MILLISECONDS); + } catch (RejectedExecutionException stopping) { + // stop() shut the executor between the running check and the submission. There is nothing to + // schedule onto and nothing to report: the worker is stopping, which is what was asked for. + running.set(false); + } + } + + private void runPass() { + if (!running.get()) { + return; + } + int unproductive; + try { + OutboxRelayReport report = relay.runOnce(clock.get()); + unproductive = report.published() > 0 ? reset() : unproductivePasses.incrementAndGet(); + } catch (RuntimeException passFailed) { + // A failed pass must not stop the loop: the scheduled task's own exception would cancel every + // future pass, turning one broker error into a relay that never runs again. The failure is + // counted and the next pass backs off as if nothing was published, which is true. + failedPasses.incrementAndGet(); + unproductive = unproductivePasses.incrementAndGet(); + } + completedPasses.incrementAndGet(); + schedule(scheduler.backoff(unproductive)); + } + + private int reset() { + unproductivePasses.set(0); + return 0; + } +} diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRetryScheduler.java b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRetryScheduler.java index 093933d..65f234c 100644 --- a/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRetryScheduler.java +++ b/messaging/messaging-outbox-jdbc-postgresql/src/main/java/dev/caskeleton/messaging/outbox/OutboxRetryScheduler.java @@ -8,10 +8,12 @@ import java.util.Optional; /** * Decides when a relay's next pass should run, and when a record has retried enough. * - *

The backoff is on the relay, not on the individual record. A broker outage fails - * every record in the batch for the same reason, so backing off per record would hammer the broker - * with the whole backlog on a schedule while achieving nothing; backing off the pass lets the - * broker recover. + *

The same curve serves two different jobs and neither replaces the other. Backing off the + * pass is what lets a broker recover: an outage fails every record in the batch for the + * same reason, so retrying the whole backlog on the poll interval achieves nothing but load. But a + * pass-level wait is process memory — it is forgotten on restart and each replica keeps its own — + * so eligibility also has to be written on the row, or the backlog is retried as fast as the + * unluckiest replica polls. * *

Jitter is applied deterministically from the attempt count rather than randomly. Several relay * instances that all started at deployment time would otherwise synchronise their retries into a @@ -58,6 +60,25 @@ public final class OutboxRetryScheduler { return now.plus(backoff(consecutiveEmptyOrFailedPasses)); } + /** + * Returns when a row that has spent this many attempts may be claimed again. + * + *

Written onto the row rather than held in the relay, because a deadline that lives in process + * memory is forgotten on restart and computed independently by every replica — which turns a + * backoff into every worker re-publishing the same backlog as fast as it polls. + * + * @param now the current instant + * @param attempts how many attempts the row has spent + * @return the earliest instant the row is eligible again + */ + public Instant nextAttemptAt(Instant now, int attempts) { + Objects.requireNonNull(now, "now must not be null"); + if (attempts < 1) { + throw new IllegalArgumentException("a row that has been published once has spent an attempt"); + } + return now.plus(backoff(attempts)); + } + /** The platform's default policy, for a relay whose caller did not supply one. */ public static OutboxRetryScheduler standard() { return new OutboxRetryScheduler(OutboxProperties.defaults(), java.time.Duration.ofMinutes(5)); diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/main/resources/db/migration/messaging/V4__messaging_outbox_canonical_metadata.sql b/messaging/messaging-outbox-jdbc-postgresql/src/main/resources/db/migration/messaging/V4__messaging_outbox_canonical_metadata.sql new file mode 100644 index 0000000..920ed04 --- /dev/null +++ b/messaging/messaging-outbox-jdbc-postgresql/src/main/resources/db/migration/messaging/V4__messaging_outbox_canonical_metadata.sql @@ -0,0 +1,66 @@ +-- The envelope's canonical facts, as columns. +-- +-- V1 stored identity, type, version, content type, payload and a free-form header map. Everything +-- else the envelope carries — producer, tenant, correlation, causation, the routing keys, the +-- schema the payload was written against, the active trace — had nowhere to go, so it was either +-- lost when the relay rebuilt the envelope or smuggled through the header map under the reserved +-- `msg.*` names the platform is supposed to own. Smuggling is the worse of the two: a row whose +-- header map contains `msg.id` overwrites another message's identity on the wire, and the consumer +-- trusts the last such header it sees. +-- +-- Columns rather than a versioned envelope blob. Both round-trip the values faithfully; only one of +-- them lets the relay answer an operator's questions. "Which tenant is the backlog for", "which +-- correlation is stuck", "which rows carry a schema this consumer cannot read" are SELECTs against +-- this table if the fields are columns, and payload decoding of the whole backlog if they are not. +-- +-- Every column is nullable. A row written before this migration has none of them, and a producer +-- that genuinely has no correlation writes none either; the reader treats both the same way, which +-- is to carry what is there and invent nothing. +ALTER TABLE messaging_outbox + ADD COLUMN producer VARCHAR(120), + ADD COLUMN occurred_at TIMESTAMPTZ, + ADD COLUMN correlation_id VARCHAR(160), + ADD COLUMN causation_id UUID, + ADD COLUMN tenant VARCHAR(64), + ADD COLUMN partition_key VARCHAR(1024), + ADD COLUMN ordering_key VARCHAR(1024), + ADD COLUMN schema_subject VARCHAR(240), + ADD COLUMN schema_uri VARCHAR(1024), + ADD COLUMN traceparent VARCHAR(64), + ADD COLUMN tracestate VARCHAR(512), + ADD COLUMN baggage VARCHAR(8192); + +-- The same bound TenantContext enforces in Java. Stated here as well because the relay, the CDC +-- connector and any operator query read this table directly: a tenant slug that only the +-- application validates is a tenant slug that an INSERT from anywhere else can violate, and a +-- tenant id is the field every downstream authorization decision reads. +ALTER TABLE messaging_outbox + ADD CONSTRAINT ck_messaging_outbox_tenant + CHECK (tenant IS NULL OR tenant ~ '^[a-z0-9][a-z0-9._-]{0,63}$'); + +-- A schema location with no subject cannot be resolved: the reader would hold a URI and no way to +-- know what it is a schema for. +ALTER TABLE messaging_outbox + ADD CONSTRAINT ck_messaging_outbox_schema_reference + CHECK (schema_uri IS NULL OR schema_subject IS NOT NULL); + +-- "Which tenant is the backlog for" is the first question asked when an outbox stops draining, and +-- before this column it could only be answered by decoding every payload. Partial, like the other +-- outbox indexes, because PUBLISHED rows accumulate until retention removes them and no operator +-- asks that question about work that already left. +CREATE INDEX ix_messaging_outbox_tenant_backlog + ON messaging_outbox (tenant, created_at) + WHERE status IN ('PENDING', 'AMBIGUOUS', 'IN_FLIGHT'); + +-- The key a relay puts on the wire, computed once and stored, so both relays agree. +-- +-- Debezium's Event Router takes the message key from a column. It was pointed at `destination`, +-- which made the key the topic name — every message on a topic sharing one key, so every message +-- landing on one partition, and keyed ordering meaning nothing. The polling relay meanwhile used +-- the partition key when the row had one and the message id when it did not. +-- +-- A generated column states that fallback once, in the place both relays read, instead of leaving +-- it as a rule each of them implements separately and one of them gets wrong. +ALTER TABLE messaging_outbox + ADD COLUMN routing_key TEXT + GENERATED ALWAYS AS (COALESCE(partition_key, message_id::TEXT)) STORED; diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapperTest.java b/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapperTest.java index c5b5117..e49e162 100644 --- a/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapperTest.java +++ b/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/DebeziumOutboxRecordMapperTest.java @@ -12,6 +12,7 @@ import dev.caskeleton.messaging.api.destination.DestinationName; import dev.caskeleton.messaging.api.error.MessagingConfigurationException; import dev.caskeleton.messaging.api.header.ReservedHeaders; import dev.caskeleton.messaging.outbox.DebeziumOutboxProfile.RelayMode; +import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata; import dev.caskeleton.messaging.reliability.OutboxRecord; import dev.caskeleton.messaging.reliability.OutboxStatus; import java.nio.charset.StandardCharsets; @@ -26,6 +27,11 @@ class DebeziumOutboxRecordMapperTest { private static final MessageId MESSAGE_ID = MessageId.newId(); private static OutboxRecord record(Map headers) { + return record(headers, OutboxCanonicalMetadata.none()); + } + + private static OutboxRecord record( + Map headers, OutboxCanonicalMetadata metadata) { return new OutboxRecord( MESSAGE_ID, new DestinationName("orders.v1"), @@ -34,6 +40,7 @@ class DebeziumOutboxRecordMapperTest { ContentType.JSON, "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), headers, + metadata, NOW, OutboxStatus.PENDING, 0, @@ -89,13 +96,71 @@ class DebeziumOutboxRecordMapperTest { @Test void aKeyedProfileUsesTheSamePartitionKeyThePollingRelayWouldHave() { DebeziumMappedRecord mapped = - mapper(true).map(record(Map.of(ReservedHeaders.PARTITION_KEY, "customer-42"))); + mapper(true) + .map( + record( + Map.of(), + OutboxCanonicalMetadata.none().withRoutingKeys("customer-42", "customer-42"))); assertThat(mapped.key()) .as("a different key would send the two relays' messages to different partitions") .hasValue("customer-42"); } + @Test + void aKeyedProfileFallsBackToTheMessageIdWhenTheRowHasNoPartitionKey() { + assertThat(mapper(true).map(record(Map.of())).key()) + .as("the fallback the routing_key column computes, stated in one place for both relays") + .hasValue(MESSAGE_ID.value().toString()); + } + + @Test + void theCanonicalColumnsBecomeReservedHeaders() { + OutboxCanonicalMetadata metadata = + OutboxCanonicalMetadata.none() + .withProducer(new dev.caskeleton.messaging.api.ProducerId("order-api")) + .withTenant(new dev.caskeleton.messaging.api.TenantContext("acme")) + .withRoutingKeys("customer-42", "customer-42"); + + Map headers = mapper(false).map(record(Map.of(), metadata)).headers(); + + assertThat(headers) + .as("a relay that drops these makes the publish path part of the message's meaning") + .containsEntry(ReservedHeaders.PRODUCER, "order-api") + .containsEntry(ReservedHeaders.TENANT, "acme") + .containsEntry(ReservedHeaders.PARTITION_KEY, "customer-42") + .containsEntry(ReservedHeaders.ORDERING_KEY, "customer-42") + .containsEntry(ReservedHeaders.ORIGIN_DESTINATION, "orders.v1"); + } + + @Test + void aColumnTheRowDoesNotCarryProducesNoHeaderAtAll() { + assertThat(mapper(false).map(record(Map.of())).headers()) + .as("an empty string would be indistinguishable from a tenant literally named ''") + .doesNotContainKeys( + ReservedHeaders.TENANT, ReservedHeaders.CORRELATION_ID, ReservedHeaders.TRACEPARENT); + } + + @Test + void aForgedReservedHeaderOnTheRowCannotOverwriteTheCanonicalOne() { + Map forged = + Map.of(ReservedHeaders.MESSAGE_ID, "00000000-0000-7000-8000-000000000000"); + + assertThat(mapper(false).map(record(forged)).headers()) + .as("a CDC connector reads the table directly and never passes the write-path check") + .containsEntry(ReservedHeaders.MESSAGE_ID, MESSAGE_ID.value().toString()); + } + + @Test + void theRoutedKeyIsNotTheTopicName() { + // Literals, not the class's own constants: comparing a configuration value against the constant + // that produced it asserts that the router agrees with itself, which it always will. + assertThat(new DebeziumOutboxEventRouter().connectorConfiguration("prod.")) + .as("keying by destination puts every message on a topic onto one partition") + .containsEntry("transforms.outbox.table.field.event.key", "routing_key") + .containsEntry("transforms.outbox.route.by.field", "destination"); + } + @Test void thePayloadIsUnchangedByTheTransform() { assertThat(mapper(false).map(record(Map.of())).payload()) diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/JdbcOutboxTransactionRequirementTest.java b/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/JdbcOutboxTransactionRequirementTest.java index 0c05890..1b33e1a 100644 --- a/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/JdbcOutboxTransactionRequirementTest.java +++ b/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/JdbcOutboxTransactionRequirementTest.java @@ -95,6 +95,7 @@ class JdbcOutboxTransactionRequirementTest { ContentType.JSON, "{}".getBytes(java.nio.charset.StandardCharsets.UTF_8), Map.of(), + dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata.none(), Instant.parse("2026-08-14T00:00:00Z"), OutboxStatus.PENDING, 0, diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxEnvelopeFactoryTest.java b/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxEnvelopeFactoryTest.java new file mode 100644 index 0000000..76745f4 --- /dev/null +++ b/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxEnvelopeFactoryTest.java @@ -0,0 +1,158 @@ +package dev.caskeleton.messaging.outbox; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.CausationId; +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.CorrelationId; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TenantContext; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata; +import dev.caskeleton.messaging.reliability.OutboxRecord; +import dev.caskeleton.messaging.reliability.OutboxStatus; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +/** + * What survives a trip through the database. + * + *

The envelope this factory rebuilds used to be poorer than the one that was written: producer + * came from the relay's own configuration, and correlation, causation, tenant, trace and the schema + * reference were all rebuilt as empty. A message published directly and the same message published + * through the outbox therefore arrived at the consumer as two different messages, and which one a + * service got depended on a deployment decision it could not see. + */ +class OutboxEnvelopeFactoryTest { + + private static final Instant WRITTEN_AT = Instant.parse("2026-08-10T09:15:00Z"); + private static final Instant OCCURRED_AT = Instant.parse("2026-08-10T09:14:00Z"); + private static final MessageId CAUSE = MessageId.newId(); + private static final String TRACEPARENT = + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; + + private final OutboxEnvelopeFactory factory = + new OutboxEnvelopeFactory(new ProducerId("relay-owner")); + + @Test + void everyCanonicalFieldOnTheRowReachesTheEnvelope() { + MessageEnvelope envelope = factory.toEnvelope(record(fullMetadata(), Map.of())); + + assertThat(envelope.producer()).isEqualTo(new ProducerId("order-api")); + assertThat(envelope.occurredAt()).contains(OCCURRED_AT); + assertThat(envelope.correlationId()).contains(new CorrelationId("checkout-7")); + assertThat(envelope.causationId()).contains(new CausationId(CAUSE)); + assertThat(envelope.tenantContext()).contains(new TenantContext("acme")); + assertThat(envelope.partitionKey()).contains("customer-42"); + assertThat(envelope.orderingKey()).contains("customer-42"); + assertThat(envelope.traceContext().traceparent()).contains(TRACEPARENT); + assertThat(envelope.payload().schemaReference()) + .hasValueSatisfying( + reference -> { + assertThat(reference.subject()).isEqualTo("order.created-value"); + assertThat(reference.version()).isEqualTo(new SchemaVersion(1)); + assertThat(reference.schemaUri()).contains(java.net.URI.create("urn:schema:1")); + }); + } + + @Test + void aTenantIsNeverReconstructedAsAbsent() { + assertThat(factory.toEnvelope(record(fullMetadata(), Map.of())).tenantContext()) + .as("a message published by one tenant and consumed as belonging to none is an escalation") + .contains(new TenantContext("acme")); + } + + @Test + void aRowWithoutAProducerIsAttributedToTheServiceThatOwnsTheRelay() { + MessageEnvelope envelope = + factory.toEnvelope(record(OutboxCanonicalMetadata.none(), Map.of())); + + assertThat(envelope.producer()) + .as("a row written before the column existed has no honest producer but this one") + .isEqualTo(new ProducerId("relay-owner")); + assertThat(envelope.occurredAt()) + .as("the transaction that wrote the row is the one the fact occurred in") + .contains(WRITTEN_AT); + assertThat(envelope.tenantContext()).isEmpty(); + assertThat(envelope.payload().schemaReference()).isEmpty(); + } + + @Test + void applicationHeadersOnTheRowStillTravel() { + assertThat( + factory + .toEnvelope(record(OutboxCanonicalMetadata.none(), Map.of("x-tenant-tier", "gold"))) + .headers() + .find("x-tenant-tier")) + .hasValueSatisfying(value -> assertThat(value.value()).isEqualTo("gold")); + } + + @Test + void aReservedHeaderOnTheRowIsRefusedEvenWhenItIsARoutingKey() { + assertThatThrownBy( + () -> + factory.toEnvelope( + record( + OutboxCanonicalMetadata.none(), + Map.of(ReservedHeaders.PARTITION_KEY, "customer-42")))) + .as("the routing keys were the exception that kept the reserved-name door ajar") + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining("reserved namespace"); + } + + @Test + void aForgedMessageIdOnTheRowCannotOverwriteTheCanonicalOne() { + assertThatThrownBy( + () -> + factory.toEnvelope( + record( + OutboxCanonicalMetadata.none(), + Map.of(ReservedHeaders.MESSAGE_ID, MessageId.newId().value().toString())))) + .isInstanceOf(MessagingConfigurationException.class) + .hasMessageContaining(ReservedHeaders.MESSAGE_ID); + } + + private static OutboxCanonicalMetadata fullMetadata() { + return new OutboxCanonicalMetadata( + Optional.of(new ProducerId("order-api")), + Optional.of(OCCURRED_AT), + Optional.of(new CorrelationId("checkout-7")), + Optional.of(new CausationId(CAUSE)), + Optional.of(new TenantContext("acme")), + Optional.of("customer-42"), + Optional.of("customer-42"), + Optional.of("order.created-value"), + Optional.of("urn:schema:1"), + new TraceContext(Optional.of(TRACEPARENT), Optional.empty(), Optional.empty())); + } + + private static OutboxRecord record( + OutboxCanonicalMetadata metadata, Map headers) { + return new OutboxRecord( + MessageId.newId(), + new DestinationName("order-events"), + new MessageType("order.created"), + new SchemaVersion(1), + ContentType.JSON, + "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), + headers, + metadata, + WRITTEN_AT, + OutboxStatus.PENDING, + 0, + Optional.empty(), + Optional.empty()); + } +} diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxPostgresIT.java b/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxPostgresIT.java index 2222e9f..b8d9a5b 100644 --- a/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxPostgresIT.java +++ b/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxPostgresIT.java @@ -7,6 +7,7 @@ import dev.caskeleton.messaging.api.MessageId; import dev.caskeleton.messaging.api.MessageType; import dev.caskeleton.messaging.api.SchemaVersion; import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata; import dev.caskeleton.messaging.reliability.OutboxRecord; import dev.caskeleton.messaging.reliability.OutboxStatus; import dev.caskeleton.messaging.testkit.DockerAvailability; @@ -94,7 +95,9 @@ class OutboxPostgresIT { assertThat(leasedRecord.messageId()).isEqualTo(record.messageId()); assertThat(leasedRecord.messageType()).isEqualTo(record.messageType()); assertThat(leasedRecord.payload()).isEqualTo(record.payload()); - assertThat(leasedRecord.headers()).containsEntry("msg.partition-key", "acct-1"); + assertThat(leasedRecord.metadata().partitionKey()) + .as("the routing key is a column the relay can read without decoding the payload") + .contains("acct-1"); }); } @@ -200,6 +203,12 @@ class OutboxPostgresIT { } private static OutboxRecord record() { + return record( + // The routing key is a column now, not a reserved header smuggled through the header map. + OutboxCanonicalMetadata.none().withRoutingKeys("acct-1", "acct-1")); + } + + private static OutboxRecord record(OutboxCanonicalMetadata metadata) { return new OutboxRecord( MessageId.newId(), new DestinationName("order-events"), @@ -207,7 +216,8 @@ class OutboxPostgresIT { new SchemaVersion(1), ContentType.JSON, "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), - Map.of("msg.partition-key", "acct-1"), + Map.of(), + metadata, NOW.minusSeconds(60), OutboxStatus.PENDING, 0, @@ -223,9 +233,67 @@ class OutboxPostgresIT { return source; } + /** + * Every migration in the module, in version order. + * + *

Discovered rather than listed. The list said V1 and V2 above a comment claiming it was every + * migration, so V3 and V4 were never applied here — and the first symptom of that was not a + * missing check but this suite failing against a column the production schema has. A test that + * applies part of the schema certifies a database nobody deploys. + */ private static java.util.List migrations() throws IOException { - return java.util.List.of( - migration("V1__messaging_outbox.sql"), migration("V2__messaging_outbox_lease_fencing.sql")); + java.net.URL directory = + OutboxPostgresIT.class.getClassLoader().getResource("db/migration/messaging"); + if (directory == null) { + throw new IOException("the messaging migrations are missing from the classpath"); + } + java.io.File[] files = + new java.io.File(java.net.URI.create(directory.toString())) + .listFiles((dir, name) -> name.endsWith(".sql")); + if (files == null || files.length == 0) { + throw new IOException("no migration applied; the schema under test would be empty"); + } + java.util.List names = + java.util.Arrays.stream(files).map(java.io.File::getName).sorted().toList(); + java.util.List statements = new java.util.ArrayList<>(); + for (String name : names) { + statements.add(migration(name)); + } + return statements; + } + + private int countWhere(String predicate) { + try (Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement(); + java.sql.ResultSet results = + statement.executeQuery("SELECT count(*) FROM messaging_outbox WHERE " + predicate)) { + results.next(); + return results.getInt(1); + } catch (SQLException failure) { + throw new IllegalStateException("could not count outbox rows", failure); + } + } + + private String routingKeyOf(MessageId messageId) { + try (Connection connection = dataSource.getConnection(); + java.sql.PreparedStatement statement = + connection.prepareStatement( + "SELECT routing_key FROM messaging_outbox WHERE message_id = ?")) { + statement.setObject(1, messageId.value()); + try (java.sql.ResultSet results = statement.executeQuery()) { + results.next(); + return results.getString(1); + } + } catch (SQLException failure) { + throw new IllegalStateException("could not read the routing key", failure); + } + } + + private void execute(String sql) throws SQLException { + try (Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.execute(sql); + } } private static String migration(String fileName) throws IOException { @@ -385,6 +453,84 @@ class OutboxPostgresIT { .isEqualTo(OutboxStatus.PUBLISHED); } + @Test + void everyCanonicalColumnRoundTripsThroughTheDatabase() { + MessageId cause = MessageId.newId(); + OutboxCanonicalMetadata written = + new OutboxCanonicalMetadata( + Optional.of(new dev.caskeleton.messaging.api.ProducerId("order-api")), + Optional.of(NOW.minusSeconds(120)), + Optional.of(new dev.caskeleton.messaging.api.CorrelationId("checkout-7")), + Optional.of(new dev.caskeleton.messaging.api.CausationId(cause)), + Optional.of(new dev.caskeleton.messaging.api.TenantContext("acme")), + Optional.of("customer-42"), + Optional.of("customer-42"), + Optional.of("order.created-value"), + Optional.of("urn:schema:1"), + new dev.caskeleton.messaging.api.TraceContext( + Optional.of("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"), + Optional.empty(), + Optional.empty())); + OutboxRecord record = record(written); + + appendInTransaction(record); + + assertThat(repository.find(record.messageId()).orElseThrow().metadata()) + .as("what the relay reads back has to be what the business transaction wrote") + .isEqualTo(written); + } + + @Test + void theRelayCanSelectOneTenantsBacklogWithoutDecodingAPayload() { + appendInTransaction( + record( + OutboxCanonicalMetadata.none() + .withTenant(new dev.caskeleton.messaging.api.TenantContext("acme")))); + appendInTransaction( + record( + OutboxCanonicalMetadata.none() + .withTenant(new dev.caskeleton.messaging.api.TenantContext("globex")))); + + assertThat(countWhere("tenant = 'acme'")) + .as("the operational question an outbox backlog always raises, answerable as a SELECT") + .isEqualTo(1); + } + + @Test + void theStoredRoutingKeyIsTheOneBothRelaysWouldUse() { + OutboxRecord keyed = + record(OutboxCanonicalMetadata.none().withRoutingKeys("customer-42", "customer-42")); + OutboxRecord unkeyed = record(OutboxCanonicalMetadata.none()); + appendInTransaction(keyed); + appendInTransaction(unkeyed); + + DebeziumOutboxEventRouter router = new DebeziumOutboxEventRouter(); + assertThat(routingKeyOf(keyed.messageId())) + .as("the CDC key used to be the destination, so every message on a topic shared a key") + .isEqualTo(router.routingKey(keyed)) + .isEqualTo("customer-42"); + assertThat(routingKeyOf(unkeyed.messageId())) + .as("the fallback is stated once, in the column, and the Java agrees with it") + .isEqualTo(router.routingKey(unkeyed)) + .isEqualTo(unkeyed.messageId().value().toString()); + } + + @Test + void aTenantThatBreaksTheSlugBoundIsRefusedByTheDatabase() { + org.assertj.core.api.Assertions.assertThatThrownBy( + () -> + execute( + "INSERT INTO messaging_outbox (message_id, destination, message_type, " + + "schema_version, content_type, payload, created_at, status, attempts, " + + "tenant) VALUES ('" + + MessageId.newId().value() + + "', 'order-events', 'order.created', 1, 'application/json', '\\x7b7d', " + + "now(), 'PENDING', 0, 'Tenant With Spaces')")) + .as("a CDC connector and an operator both write this table without passing through Java") + .isInstanceOf(SQLException.class) + .hasMessageContaining("ck_messaging_outbox_tenant"); + } + @Test void aHeaderValueWithControlCharactersRoundTrips() { OutboxRecord record = @@ -398,7 +544,8 @@ class OutboxPostgresIT { // A tab and a newline in a header value. Only backslash and quote were escaped, so // this produced a document PostgreSQL refuses as invalid JSONB and the append failed // with an error naming neither the header nor the row. - Map.of("msg.note", "line one\nline\ttwo"), + Map.of("x-note", "line one\nline\ttwo"), + OutboxCanonicalMetadata.none(), NOW.minusSeconds(60), OutboxStatus.PENDING, 0, @@ -408,7 +555,7 @@ class OutboxPostgresIT { appendInTransaction(record); assertThat(repository.find(record.messageId()).orElseThrow().headers()) - .containsEntry("msg.note", "line one\nline\ttwo"); + .containsEntry("x-note", "line one\nline\ttwo"); } private void appendInTransaction(OutboxRecord record) { diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxRelayTest.java b/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxRelayTest.java index fc00e4d..62bc74f 100644 --- a/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxRelayTest.java +++ b/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxRelayTest.java @@ -80,6 +80,41 @@ class OutboxRelayTest { assertThat(publisher.publishedIds.get(1)).isEqualTo(record.messageId()); } + @Test + void anAmbiguousRowWaitsForItsBackoffBeforeItIsClaimedAgain() { + OutboxRecord record = OutboxFixtures.pending(); + repository.append(record); + RecordingOutboxPublisher publisher = RecordingOutboxPublisher.ambiguous(); + + relay(publisher).runOnce(NOW); + OutboxRelayReport tooSoon = relay(publisher).runOnce(NOW.plusMillis(100)); + + assertThat(tooSoon.leased()) + .as("a backoff nothing writes on the row is a backoff the next pass cannot see") + .isZero(); + assertThat(publisher.publishedIds).hasSize(1); + } + + @Test + void aRowThatSpendsItsAttemptBudgetIsParkedRatherThanLeftRetryable() { + OutboxRecord record = OutboxFixtures.pending(); + repository.append(record); + RecordingOutboxPublisher publisher = RecordingOutboxPublisher.ambiguous(); + OutboxRelay relay = relay(publisher, budgetOf(2)); + + relay.runOnce(NOW); + OutboxRelayReport last = relay.runOnce(NOW.plus(Duration.ofHours(1))); + + assertThat(last.exhausted()) + .as("a rejected row needs a fix and a spent one needs a redrive") + .isEqualTo(1); + assertThat(repository.find(record.messageId()).orElseThrow().status()) + .isEqualTo(OutboxStatus.EXHAUSTED); + assertThat(relay.runOnce(NOW.plus(Duration.ofDays(1))).leased()) + .as("only an operator redrive brings a parked row back") + .isZero(); + } + @Test void aRejectedPublishIsMarkedFailedAndNotRetried() { OutboxRecord record = OutboxFixtures.pending(); @@ -135,6 +170,30 @@ class OutboxRelayTest { 100, Duration.ofSeconds(30)); } + + private OutboxRelay relay(MessagePublisher publisher, OutboxRetryScheduler scheduler) { + return new OutboxRelay( + repository, + publisher, + new OutboxEnvelopeFactory(new ProducerId("order-api")), + 100, + Duration.ofSeconds(30), + "relay-under-test", + scheduler); + } + + private static OutboxRetryScheduler budgetOf(int maxAttempts) { + OutboxProperties defaults = OutboxProperties.defaults(); + return new OutboxRetryScheduler( + new OutboxProperties( + defaults.batchSize(), + defaults.leaseDuration(), + defaults.publishTimeout(), + defaults.pollInterval(), + defaults.retentionAfterPublish(), + maxAttempts), + Duration.ofMinutes(5)); + } } /** Builds outbox rows. */ @@ -151,6 +210,7 @@ final class OutboxFixtures { ContentType.JSON, "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), Map.of(), + dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata.none(), Instant.parse("2026-08-10T09:14:00Z"), OutboxStatus.PENDING, 0, diff --git a/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxRelayWorkerTest.java b/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxRelayWorkerTest.java new file mode 100644 index 0000000..cadc679 --- /dev/null +++ b/messaging/messaging-outbox-jdbc-postgresql/src/test/java/dev/caskeleton/messaging/outbox/OutboxRelayWorkerTest.java @@ -0,0 +1,163 @@ +package dev.caskeleton.messaging.outbox; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.publish.ConfirmationLevel; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishEvidence; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.api.publish.RoutingOutcome; +import dev.caskeleton.messaging.reliability.OutboxRecord; +import dev.caskeleton.messaging.reliability.OutboxStatus; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Whether anything actually drives the relay. + * + *

The relay, its scheduler and the attempt budget were all constructed by the auto-configuration + * and {@code runOnce} was called by no production code. A business transaction would commit its + * outbox row, report success, and the message would never be published — a silent backlog rather + * than a visible failure. + */ +class OutboxRelayWorkerTest { + + private static final Duration PATIENCE = Duration.ofSeconds(5); + + private final InMemoryOutboxRepository repository = new InMemoryOutboxRepository(); + + @Test + @DisplayName("a started worker publishes the backlog without anyone calling a pass") + void aStartedWorkerPublishesTheBacklog() throws InterruptedException { + OutboxRecord record = OutboxFixtures.pending(); + repository.append(record); + OutboxRelayWorker worker = worker(RecordingOutboxPublisher.confirming()); + + worker.start(); + try { + awaitStatus(record, OutboxStatus.PUBLISHED); + } finally { + worker.stop(PATIENCE); + } + + assertThat(repository.find(record.messageId()).orElseThrow().status()) + .isEqualTo(OutboxStatus.PUBLISHED); + assertThat(worker.completedPasses()).isPositive(); + } + + @Test + @DisplayName("stopping waits for the running pass instead of stranding its claimed rows") + void stoppingWaitsForTheRunningPass() throws InterruptedException { + OutboxRecord record = OutboxFixtures.pending(); + repository.append(record); + // The publish is still in progress when stop() is called, so the row is claimed and IN_FLIGHT + // at that moment: the pass has to be allowed to finish or the lease strands the row. + SlowPublisher publisher = new SlowPublisher(Duration.ofMillis(300)); + OutboxRelayWorker worker = worker(publisher); + + worker.start(); + assertThat(publisher.started.await(PATIENCE.toMillis(), TimeUnit.MILLISECONDS)).isTrue(); + boolean drained = worker.stop(PATIENCE); + + assertThat(drained).isTrue(); + assertThat(repository.find(record.messageId()).orElseThrow().status()) + .as("a pass killed between its claim and its terminal write leaves the row leased") + .isEqualTo(OutboxStatus.PUBLISHED); + } + + @Test + @DisplayName("a pass that throws does not stop every pass after it") + void aFailedPassDoesNotEndTheLoop() throws InterruptedException { + repository.append(OutboxFixtures.pending()); + OutboxRelayWorker worker = worker(new ThrowingPublisher()); + + worker.start(); + try { + long deadline = System.nanoTime() + PATIENCE.toNanos(); + while (worker.completedPasses() < 2 && System.nanoTime() < deadline) { + Thread.sleep(10); + } + } finally { + worker.stop(PATIENCE); + } + + assertThat(worker.completedPasses()) + .as("a scheduled task's own exception cancels every future run of it") + .isGreaterThanOrEqualTo(2); + assertThat(worker.failedPasses()).isPositive(); + } + + private OutboxRelayWorker worker(MessagePublisher publisher) { + OutboxRelay relay = + new OutboxRelay( + repository, + publisher, + new OutboxEnvelopeFactory(new ProducerId("order-api")), + 100, + Duration.ofSeconds(30)); + return new OutboxRelayWorker(relay, OutboxRetryScheduler.standard()); + } + + private void awaitStatus(OutboxRecord record, OutboxStatus expected) throws InterruptedException { + long deadline = System.nanoTime() + PATIENCE.toNanos(); + while (System.nanoTime() < deadline) { + if (repository.find(record.messageId()).map(OutboxRecord::status).orElse(null) == expected) { + return; + } + Thread.sleep(10); + } + } + + /** A publisher that is still publishing when shutdown begins. */ + private static final class SlowPublisher implements MessagePublisher { + + private final CountDownLatch started = new CountDownLatch(1); + + private final Duration duration; + + private SlowPublisher(Duration duration) { + this.duration = duration; + } + + @Override + public CompletionStage publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options) { + started.countDown(); + try { + Thread.sleep(duration.toMillis()); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + return CompletableFuture.completedFuture( + new PublishResult( + PublishCompletion.CONFIRMED, + PublishEvidence.confirmed(ConfirmationLevel.BROKER_ACK), + RoutingOutcome.NOT_APPLICABLE, + Optional.empty(), + 1, + Duration.ofMillis(1), + Optional.empty())); + } + } + + /** A publisher whose failure reaches the worker as an exception rather than a result. */ + private static final class ThrowingPublisher implements MessagePublisher { + + @Override + public CompletionStage publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options) { + throw new IllegalStateException("the broker connection is gone"); + } + } +} diff --git a/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/MessagingAdmissionController.java b/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/MessagingAdmissionController.java index c426ba2..108e743 100644 --- a/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/MessagingAdmissionController.java +++ b/messaging/messaging-policy/src/main/java/dev/caskeleton/messaging/policy/MessagingAdmissionController.java @@ -2,7 +2,10 @@ package dev.caskeleton.messaging.policy; import dev.caskeleton.messaging.api.error.MessageBackpressureException; import dev.caskeleton.messaging.api.error.MessageTooLargeException; +import java.util.Map; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; /** * The single gate every publish passes before any transport work begins. @@ -16,22 +19,49 @@ import java.util.Objects; * under the same message id without risking a duplicate. That is the reason admission lives at the * front of the pipeline instead of being folded into the adapter's error handling, where a partial * write would already have made the outcome unknowable. + * + *

Two ceilings, because one is not enough. The per-destination ceiling stops a single slow + * downstream from consuming every permit in the process, and the process-wide ceiling stops the sum + * of well-behaved destinations from exhausting memory — without it, adding a destination silently + * raises what the process can be holding at once. */ public final class MessagingAdmissionController { private final PayloadLimitGuard payloadGuard; private final InFlightLimiter limiter; + private final int perDestinationLimit; + private final Map perDestination = new ConcurrentHashMap<>(); private volatile boolean acceptingNewWork = true; /** - * Creates an admission controller. + * Creates an admission controller whose only ceiling is the process-wide one. * * @param payloadGuard the payload size guard * @param limiter the in-flight permit pool */ public MessagingAdmissionController(PayloadLimitGuard payloadGuard, InFlightLimiter limiter) { + this(payloadGuard, limiter, limiter.limit()); + } + + /** + * Creates an admission controller with a per-destination ceiling. + * + * @param payloadGuard the payload size guard + * @param limiter the in-flight permit pool + * @param perDestinationLimit how much in-flight work one destination may hold + */ + public MessagingAdmissionController( + PayloadLimitGuard payloadGuard, InFlightLimiter limiter, int perDestinationLimit) { this.payloadGuard = Objects.requireNonNull(payloadGuard, "payloadGuard must not be null"); this.limiter = Objects.requireNonNull(limiter, "limiter must not be null"); + if (perDestinationLimit < 1) { + throw new IllegalArgumentException("perDestinationLimit must be at least 1"); + } + if (perDestinationLimit > limiter.limit()) { + throw new IllegalArgumentException( + "perDestinationLimit cannot exceed the process-wide limit, or it is not a limit"); + } + this.perDestinationLimit = perDestinationLimit; } /** @@ -51,15 +81,27 @@ public final class MessagingAdmissionController { "SHUTTING_DOWN", "the messaging runtime stopped accepting new publishes"); } + if (!reserve(destination)) { + throw new MessageBackpressureException( + "DESTINATION_IN_FLIGHT_LIMIT_EXCEEDED", + "%s already holds its %d in-flight publishes; one slow destination does not get to" + .formatted(destination, perDestinationLimit) + + " consume the whole process"); + } + boolean acquired; try { acquired = limiter.tryAcquire(); } catch (InterruptedException interrupted) { + // The destination slot was taken a moment ago and no publish will use it, so it goes back + // here: a slot leaked per interruption shrinks the destination's ceiling until it is zero. + release(destination); Thread.currentThread().interrupt(); throw new MessageBackpressureException( "ADMISSION_INTERRUPTED", "interrupted while waiting for a publish permit"); } if (!acquired) { + release(destination); throw new MessageBackpressureException( "IN_FLIGHT_LIMIT_EXCEEDED", "no publish permit became available for %s within the bounded wait" @@ -67,8 +109,24 @@ public final class MessagingAdmissionController { } } - /** Releases the permit taken by a successful {@link #admit}. */ - public void complete() { + /** + * Releases the permit taken by a successful {@link #admit}. + * + *

The destination is named because the caller cannot release a permit it never took: a release + * carrying a different destination would return a slot to a queue that has nothing outstanding + * and leave the real one occupied forever. + * + * @param destination the destination this publish was admitted for + */ + public void complete(String destination) { + Objects.requireNonNull(destination, "destination must not be null"); + if (!release(destination)) { + // A completion for a destination that holds nothing: either it names the wrong destination or + // it is a second completion for the same publish. Returning the process permit anyway frees + // one nobody took, and the process-wide ceiling then reads below what is really in flight and + // admits more work than the process can carry. + return; + } limiter.release(); } @@ -98,4 +156,45 @@ public final class MessagingAdmissionController { public int inFlight() { return limiter.inFlight(); } + + /** + * Returns how much in-flight work one destination holds. + * + * @param destination the logical destination name + * @return the destination's in-flight count + */ + public int inFlight(String destination) { + AtomicInteger counter = perDestination.get(destination); + return counter == null ? 0 : counter.get(); + } + + private boolean reserve(String destination) { + AtomicInteger counter = perDestination.computeIfAbsent(destination, key -> new AtomicInteger()); + while (true) { + int current = counter.get(); + if (current >= perDestinationLimit) { + return false; + } + if (counter.compareAndSet(current, current + 1)) { + return true; + } + } + } + + /** + * Gives one slot back to a destination. + * + * @return true when this call actually released a slot the destination was holding + */ + private boolean release(String destination) { + AtomicInteger counter = perDestination.get(destination); + if (counter == null) { + return false; + } + int before = counter.getAndUpdate(current -> current > 0 ? current - 1 : current); + // Drop the entry at zero, atomically, so the map does not accumulate one counter per + // destination ever published to for the life of the process. + perDestination.computeIfPresent(destination, (key, value) -> value.get() == 0 ? null : value); + return before > 0; + } } diff --git a/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/MessagingAdmissionControllerTest.java b/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/MessagingAdmissionControllerTest.java index 4768378..93c7820 100644 --- a/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/MessagingAdmissionControllerTest.java +++ b/messaging/messaging-policy/src/test/java/dev/caskeleton/messaging/policy/MessagingAdmissionControllerTest.java @@ -26,7 +26,7 @@ class MessagingAdmissionControllerTest { controller.admit(DESTINATION, 100); assertThat(controller.inFlight()).isEqualTo(1); - controller.complete(); + controller.complete(DESTINATION); assertThat(controller.inFlight()).isZero(); } @@ -81,8 +81,8 @@ class MessagingAdmissionControllerTest { void aReleaseWithoutAnAdmitCannotRaiseTheCeiling() { MessagingAdmissionController controller = controller(1); - controller.complete(); - controller.complete(); + controller.complete(DESTINATION); + controller.complete(DESTINATION); controller.admit(DESTINATION, 100); assertThatThrownBy(() -> controller.admit(DESTINATION, 100)) @@ -90,6 +90,81 @@ class MessagingAdmissionControllerTest { .isInstanceOf(MessageBackpressureException.class); } + @Test + void oneSlowDestinationCannotOccupyEveryPermit() { + MessagingAdmissionController controller = + new MessagingAdmissionController( + new PayloadLimitGuard(new PayloadPolicy(1024, 512)), + new InFlightLimiter(4, Duration.ZERO), + 2); + + controller.admit(DESTINATION, 100); + controller.admit(DESTINATION, 100); + + assertThatThrownBy(() -> controller.admit(DESTINATION, 100)) + .as("a single slow downstream taking every permit is how one destination stops the others") + .isInstanceOf(MessageBackpressureException.class); + assertThat(controller.inFlight("other.destination")).isZero(); + controller.admit("other.destination", 100); + assertThat(controller.inFlight()).isEqualTo(3); + } + + @Test + void aRefusedAdmissionLeavesNoDestinationSlotBehind() { + MessagingAdmissionController controller = + new MessagingAdmissionController( + new PayloadLimitGuard(new PayloadPolicy(1024, 512)), + new InFlightLimiter(1, Duration.ZERO), + 1); + controller.admit(DESTINATION, 100); + + assertThatThrownBy(() -> controller.admit("other.destination", 100)) + .isInstanceOf(MessageBackpressureException.class); + + assertThat(controller.inFlight("other.destination")) + .as("a slot leaked per refusal shrinks the destination's ceiling until it is zero") + .isZero(); + } + + @Test + void completingReturnsBothTheProcessPermitAndTheDestinationSlot() { + MessagingAdmissionController controller = + new MessagingAdmissionController( + new PayloadLimitGuard(new PayloadPolicy(1024, 512)), + new InFlightLimiter(4, Duration.ZERO), + 2); + controller.admit(DESTINATION, 100); + + controller.complete(DESTINATION); + + assertThat(controller.inFlight()).isZero(); + assertThat(controller.inFlight(DESTINATION)).isZero(); + } + + @Test + void completingAnUnknownDestinationFreesNoProcessPermit() { + MessagingAdmissionController controller = controller(2); + controller.admit(DESTINATION, 100); + + controller.complete("something-else"); + + assertThat(controller.inFlight()) + .as("a permit nobody took cannot be given back; doing so makes the ceiling fiction") + .isEqualTo(1); + } + + @Test + void aSecondCompletionForOnePublishFreesNoSecondPermit() { + MessagingAdmissionController controller = controller(4); + controller.admit(DESTINATION, 100); + + controller.complete(DESTINATION); + controller.complete(DESTINATION); + + assertThat(controller.inFlight()).isZero(); + assertThat(controller.inFlight(DESTINATION)).isZero(); + } + @Test void aBatchIsBoundedByCountAndByAggregateBytes() { PayloadLimitGuard guard = new PayloadLimitGuard(new PayloadPolicy(1024, 512)); @@ -113,7 +188,7 @@ class MessagingAdmissionControllerTest { Thread waiter = new Thread(() -> controller.admit(DESTINATION, 100)); waiter.start(); - controller.complete(); + controller.complete(DESTINATION); waiter.join(Duration.ofSeconds(5).toMillis()); assertThat(waiter.isAlive()).isFalse(); diff --git a/messaging/messaging-pulsar-experimental/src/test/java/dev/caskeleton/messaging/pulsar/PulsarAdapterContractTest.java b/messaging/messaging-pulsar-experimental/src/test/java/dev/caskeleton/messaging/pulsar/PulsarAdapterContractTest.java index 0d3c130..0ec8223 100644 --- a/messaging/messaging-pulsar-experimental/src/test/java/dev/caskeleton/messaging/pulsar/PulsarAdapterContractTest.java +++ b/messaging/messaging-pulsar-experimental/src/test/java/dev/caskeleton/messaging/pulsar/PulsarAdapterContractTest.java @@ -234,8 +234,7 @@ class PulsarAdapterContractTest { return new TransportPublishRequest( profile(), envelope(64), - new PublishOptions( - timeout, defaults.confirmation(), defaults.deduplication(), defaults.brokerHints())); + new PublishOptions(timeout, defaults.confirmation(), defaults.deduplication())); } private static TransportPublishRequest request(int payloadBytes) { diff --git a/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitDeliveryMapper.java b/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitDeliveryMapper.java index f78d932..0572c18 100644 --- a/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitDeliveryMapper.java +++ b/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitDeliveryMapper.java @@ -13,7 +13,6 @@ import dev.caskeleton.messaging.api.TraceContext; import dev.caskeleton.messaging.api.delivery.DeliveryMetadata; import dev.caskeleton.messaging.api.destination.DestinationName; import dev.caskeleton.messaging.api.error.MessageValidationException; -import dev.caskeleton.messaging.api.header.MessageHeaders; import dev.caskeleton.messaging.api.header.ReservedHeaders; import dev.caskeleton.messaging.schema.EncodedMessage; import java.time.Instant; @@ -87,12 +86,13 @@ public final class RabbitDeliveryMapper { contentType, Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.PARTITION_KEY)), Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.ORDERING_KEY)), - Optional.empty(), + Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.TENANT)) + .map(TenantContext::new), new TraceContext( Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.TRACEPARENT)), Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.TRACESTATE)), Optional.ofNullable(headerMapper.value(properties, ReservedHeaders.BAGGAGE))), - MessageHeaders.empty(), + headerMapper.carriedForward(properties), new EncodedMessage(payload, contentType, Optional.empty())); } diff --git a/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitHeaderMapper.java b/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitHeaderMapper.java index 85ad458..dc5124c 100644 --- a/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitHeaderMapper.java +++ b/messaging/messaging-rabbit/src/main/java/dev/caskeleton/messaging/rabbit/RabbitHeaderMapper.java @@ -1,6 +1,7 @@ package dev.caskeleton.messaging.rabbit; import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.header.CanonicalEnvelopeHeaders; import dev.caskeleton.messaging.api.header.HeaderName; import dev.caskeleton.messaging.api.header.HeaderValue; import dev.caskeleton.messaging.api.header.MessageHeaders; @@ -68,6 +69,11 @@ public final class RabbitHeaderMapper { envelope .orderingKey() .ifPresent(value -> setHeader(properties, ReservedHeaders.ORDERING_KEY, value)); + // Written for the same reason as on the other broker: an AMQP message carries no ambient + // tenant, so a tenant left off the wire is a tenant the consumed envelope does not have. + envelope + .tenantContext() + .ifPresent(value -> setHeader(properties, ReservedHeaders.TENANT, value.tenantId())); envelope .traceContext() .traceparent() @@ -81,21 +87,25 @@ public final class RabbitHeaderMapper { .baggage() .ifPresent(value -> setHeader(properties, ReservedHeaders.BAGGAGE, value)); - // Application headers are written last, so an unchecked one would overwrite the canonical + // The envelope's header map is written last, so an unchecked name would overwrite the canonical // reserved value the platform just wrote. A forged `msg.id` corrupts another message's inbox // deduplication, not merely its own delivery. + // + // Only names that restate an envelope field are refused. Retry attempt, failure and redrive + // bookkeeping have no envelope field to be read back from, so refusing them would make every + // retry-queue and dead-letter re-publish fail on the guard meant to protect identity. envelope .headers() .asMap() .forEach( (name, value) -> { - if (dev.caskeleton.messaging.api.header.ReservedHeaders.isReserved(name.value())) { + if (CanonicalEnvelopeHeaders.restatesEnvelopeField(name.value())) { throw new dev.caskeleton.messaging.api.error.MessagingConfigurationException( "RESERVED_HEADER_FORGED", - "application header '" + "header '" + name.value() - + "' uses the reserved namespace; the canonical value is written by the" - + " platform and must not be overridable by message content"); + + "' restates a value the envelope already carries; the canonical value is" + + " written by the platform and must not be overridable by message content"); } setHeader(properties, name.value(), value.value()); }); @@ -123,6 +133,35 @@ public final class RabbitHeaderMapper { return MessageHeaders.platform(values); } + /** + * Reads back the headers a rebuilt envelope keeps. + * + *

Everything except the names that were hoisted into envelope fields: the application's own + * headers, which were dropped entirely and so never survived a broker hop, and the platform's + * retry and redrive bookkeeping, whose loss restarted the attempt counter at one on every + * redelivery and left the retry budget capping nothing. + * + *

The hoisted names are excluded rather than kept alongside their fields, because a duplicate + * that can drift from the field is how the same message ends up with two different ids depending + * on which one the reader trusts. + * + * @param properties the message properties + * @return the headers to carry on the rebuilt envelope + */ + public MessageHeaders carriedForward(MessageProperties properties) { + Objects.requireNonNull(properties, "properties must not be null"); + Map values = new LinkedHashMap<>(); + properties + .getHeaders() + .forEach( + (name, value) -> { + if (value != null && !CanonicalEnvelopeHeaders.restatesEnvelopeField(name)) { + values.put(new HeaderName(name), new HeaderValue(String.valueOf(value))); + } + }); + return MessageHeaders.platform(values); + } + /** * Reads one header value. * diff --git a/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitEnvelopeRoundTripTest.java b/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitEnvelopeRoundTripTest.java new file mode 100644 index 0000000..4a6f105 --- /dev/null +++ b/messaging/messaging-rabbit/src/test/java/dev/caskeleton/messaging/rabbit/RabbitEnvelopeRoundTripTest.java @@ -0,0 +1,152 @@ +package dev.caskeleton.messaging.rabbit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.CorrelationId; +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.TraceContext; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.api.header.HeaderName; +import dev.caskeleton.messaging.api.header.HeaderValue; +import dev.caskeleton.messaging.api.header.MessageHeaders; +import dev.caskeleton.messaging.api.header.ReservedHeaders; +import dev.caskeleton.messaging.schema.EncodedMessage; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.core.MessageProperties; + +/** + * What an envelope still holds after it has been through RabbitMQ. + * + *

The publish and delivery mappers were written against each other's headers but never checked + * as one trip, so the application's own headers were dropped on the way back and the retry + * bookkeeping went with them — an attempt counter that restarts on every hop caps no redeliveries + * at all. + */ +class RabbitEnvelopeRoundTripTest { + + private static final Instant NOW = Instant.parse("2026-08-15T10:00:00Z"); + + private final RabbitHeaderMapper headerMapper = new RabbitHeaderMapper(); + + private final RabbitDeliveryMapper deliveryMapper = new RabbitDeliveryMapper(headerMapper); + + @Test + @DisplayName("an application header survives the trip through the broker") + void anApplicationHeaderSurvivesTheTrip() { + MessageEnvelope published = + envelope( + MessageHeaders.application( + Map.of(new HeaderName("x-source-system"), new HeaderValue("erp")))); + + MessageEnvelope delivered = roundTrip(published); + + assertThat(delivered.headers().find("x-source-system")) + .map(HeaderValue::value) + .as("a header the producer wrote and the consumer cannot read is a header nobody has") + .hasValue("erp"); + } + + @Test + @DisplayName("the tenant survives the trip instead of arriving as no tenant at all") + void theTenantSurvivesTheTrip() { + MessageEnvelope published = + envelope(MessageHeaders.empty()) + .withTenant(new dev.caskeleton.messaging.api.TenantContext("acme")); + + MessageEnvelope delivered = roundTrip(published); + + assertThat(delivered.tenantContext()) + .as("an AMQP message carries no ambient tenant, so an unwritten tenant is an absent one") + .contains(new dev.caskeleton.messaging.api.TenantContext("acme")); + } + + @Test + @DisplayName("the retry attempt survives the trip instead of restarting at one") + void theRetryAttemptSurvivesTheTrip() { + MessageEnvelope delivered = roundTrip(retryStamped(3)); + + assertThat(delivered.headers().find(ReservedHeaders.RETRY_ATTEMPT)) + .map(HeaderValue::value) + .as("an attempt counter that resets on every hop caps nothing") + .hasValue("3"); + } + + @Test + @DisplayName("a retry-stamped envelope can be published to its retry queue") + void aRetryStampedEnvelopeCanBePublished() { + assertThatCode(() -> headerMapper.toProperties(retryStamped(2))) + .as("the platform's own bookkeeping is not message content forging an identity") + .doesNotThrowAnyException(); + } + + @Test + @DisplayName("a header restating the message id is still refused") + void aForgedMessageIdIsStillRefused() { + MessageEnvelope forged = + envelope( + MessageHeaders.platform( + Map.of(new HeaderName(ReservedHeaders.MESSAGE_ID), new HeaderValue("forged")))); + + assertThatThrownBy(() -> headerMapper.toProperties(forged)) + .isInstanceOf(MessagingConfigurationException.class); + } + + @Test + @DisplayName("identity is read from its field, not duplicated into the header map") + void identityIsNotDuplicatedIntoTheHeaderMap() { + MessageEnvelope delivered = roundTrip(envelope(MessageHeaders.empty())); + + assertThat(delivered.headers().find(ReservedHeaders.MESSAGE_ID)) + .as("a copy that can drift from the field gives the same message two identities") + .isEmpty(); + } + + private MessageEnvelope roundTrip(MessageEnvelope published) { + MessageProperties properties = headerMapper.toProperties(published); + return deliveryMapper.toEnvelope(new Message(published.payload().bytes(), properties)); + } + + private static MessageEnvelope retryStamped(int attempt) { + Map headers = new LinkedHashMap<>(); + headers.put( + new HeaderName(ReservedHeaders.RETRY_ATTEMPT), new HeaderValue(Integer.toString(attempt))); + headers.put(new HeaderName(ReservedHeaders.LAST_FAILURE_AT), new HeaderValue(NOW.toString())); + return envelope(MessageHeaders.platform(headers)); + } + + private static MessageEnvelope envelope(MessageHeaders headers) { + return new MessageEnvelope<>( + MessageId.newId(), + new MessageType("order.created"), + new SchemaVersion(1), + NOW, + Optional.of(NOW), + new ProducerId("order-api"), + Optional.of(new CorrelationId("wf-1")), + Optional.empty(), + ContentType.JSON, + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none(), + headers, + new EncodedMessage( + "{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8), + ContentType.JSON, + Optional.empty())); + } +} diff --git a/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxCanonicalMetadata.java b/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxCanonicalMetadata.java new file mode 100644 index 0000000..6747d56 --- /dev/null +++ b/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxCanonicalMetadata.java @@ -0,0 +1,158 @@ +package dev.caskeleton.messaging.reliability; + +import dev.caskeleton.messaging.api.CausationId; +import dev.caskeleton.messaging.api.CorrelationId; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.TenantContext; +import dev.caskeleton.messaging.api.TraceContext; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * The envelope facts an outbox row carries in its own columns. + * + *

They used to live nowhere. A row held identity, type, version, content type, payload and an + * arbitrary header map, so producer, tenant, correlation, causation, trace and schema were either + * invented when the envelope was rebuilt — {@code Optional.empty()} for every one of them — or + * smuggled through the header map under reserved names the platform was supposed to own. + * + *

Both routes fail in the same direction. A relay cannot filter, route or diagnose by tenant + * without decoding the payload, so the operational question "which tenant is backed up" has no + * answer; and a message that crossed the outbox arrived at its consumer with a different tenant, + * trace and correlation than the one that was published, which makes the publish path — direct, + * polling or CDC — part of the message's meaning. + * + *

Columns rather than a blob, because the point is that the database can answer questions about + * them. A versioned envelope encoding would round-trip just as faithfully and would still leave the + * relay unable to select rows for one tenant. + * + *

Every field is optional except the trace context, which has its own empty form. A row written + * before these columns existed has none of them, and a row written by a producer that genuinely has + * no correlation is not distinguishable from it — nor does it need to be, because the reader's + * behaviour is the same: carry what is there and invent nothing. + * + * @param producer the logical producing service that wrote the row + * @param occurredAt the instant the business fact occurred, when it differs from the write + * @param correlationId the workflow correlation + * @param causationId the directly causing message + * @param tenant the tenant this message belongs to + * @param partitionKey the broker distribution key + * @param orderingKey the ordering key + * @param schemaSubject the registry subject the payload was written against + * @param schemaUri the resolvable schema location, when one exists + * @param traceContext the W3C trace values active when the row was written + */ +public record OutboxCanonicalMetadata( + Optional producer, + Optional occurredAt, + Optional correlationId, + Optional causationId, + Optional tenant, + Optional partitionKey, + Optional orderingKey, + Optional schemaSubject, + Optional schemaUri, + TraceContext traceContext) { + + public OutboxCanonicalMetadata { + Objects.requireNonNull(producer, "producer must not be null"); + Objects.requireNonNull(occurredAt, "occurredAt must not be null"); + Objects.requireNonNull(correlationId, "correlationId must not be null"); + Objects.requireNonNull(causationId, "causationId must not be null"); + Objects.requireNonNull(tenant, "tenant must not be null"); + Objects.requireNonNull(partitionKey, "partitionKey must not be null"); + Objects.requireNonNull(orderingKey, "orderingKey must not be null"); + Objects.requireNonNull(schemaSubject, "schemaSubject must not be null"); + Objects.requireNonNull(schemaUri, "schemaUri must not be null"); + Objects.requireNonNull(traceContext, "traceContext must not be null"); + if (schemaUri.isPresent() && schemaSubject.isEmpty()) { + throw new IllegalArgumentException( + "a schema location without a subject cannot be resolved; a reader would have a URI and " + + "no way to know what it is a schema for"); + } + } + + /** + * Returns the metadata of a row that carries none. + * + *

Used by callers that write an outbox row from a message with no provenance of its own, and + * by the reader for rows written before these columns existed. + * + * @return empty canonical metadata + */ + public static OutboxCanonicalMetadata none() { + return new OutboxCanonicalMetadata( + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + TraceContext.none()); + } + + /** + * Returns a copy naming the producing service. + * + * @param producingService the logical producing service + * @return the metadata with its producer set + */ + public OutboxCanonicalMetadata withProducer(ProducerId producingService) { + return new OutboxCanonicalMetadata( + Optional.of(Objects.requireNonNull(producingService, "producer must not be null")), + occurredAt, + correlationId, + causationId, + tenant, + partitionKey, + orderingKey, + schemaSubject, + schemaUri, + traceContext); + } + + /** + * Returns a copy bound to a tenant. + * + * @param owner the tenant this message belongs to + * @return the metadata with its tenant set + */ + public OutboxCanonicalMetadata withTenant(TenantContext owner) { + return new OutboxCanonicalMetadata( + producer, + occurredAt, + correlationId, + causationId, + Optional.of(Objects.requireNonNull(owner, "tenant must not be null")), + partitionKey, + orderingKey, + schemaSubject, + schemaUri, + traceContext); + } + + /** + * Returns a copy carrying the broker routing keys. + * + * @param distributionKey the partition key, or null for none + * @param ordering the ordering key, or null for none + * @return the metadata with its routing keys set + */ + public OutboxCanonicalMetadata withRoutingKeys(String distributionKey, String ordering) { + return new OutboxCanonicalMetadata( + producer, + occurredAt, + correlationId, + causationId, + tenant, + Optional.ofNullable(distributionKey), + Optional.ofNullable(ordering), + schemaSubject, + schemaUri, + traceContext); + } +} diff --git a/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxRecord.java b/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxRecord.java index 058c4bb..d694995 100644 --- a/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxRecord.java +++ b/messaging/messaging-reliability-api/src/main/java/dev/caskeleton/messaging/reliability/OutboxRecord.java @@ -23,13 +23,19 @@ import java.util.Optional; * this row carrying a stable {@code messageId} and the consumer having an Inbox — not from the * outbox alone. * + *

{@link OutboxCanonicalMetadata} is a separate component rather than more fields here because + * the two halves answer to different owners. Identity, payload, status, attempts and lease are the + * relay's bookkeeping; the metadata is the message's own provenance, and it is the half that has to + * survive the round trip through the database unchanged. + * * @param messageId the stable logical identity, preserved across every retry * @param destination the logical destination * @param messageType the catalog message type * @param schemaVersion the payload schema revision * @param contentType the codec media type * @param payload the encoded payload - * @param headers the reserved and application headers to publish + * @param headers the application headers to publish; reserved names are refused + * @param metadata the canonical envelope facts this row carries in its own columns * @param createdAt when the business transaction wrote this row * @param status the current lifecycle state * @param attempts how many publish attempts have been made @@ -45,6 +51,7 @@ public record OutboxRecord( ContentType contentType, byte[] payload, Map headers, + OutboxCanonicalMetadata metadata, Instant createdAt, OutboxStatus status, int attempts, @@ -59,6 +66,7 @@ public record OutboxRecord( Objects.requireNonNull(contentType, "contentType must not be null"); Objects.requireNonNull(payload, "payload must not be null"); Objects.requireNonNull(headers, "headers must not be null"); + Objects.requireNonNull(metadata, "metadata must not be null"); Objects.requireNonNull(createdAt, "createdAt must not be null"); Objects.requireNonNull(status, "status must not be null"); Objects.requireNonNull(leaseExpiresAt, "leaseExpiresAt must not be null"); @@ -95,6 +103,7 @@ public record OutboxRecord( contentType, payload, headers, + metadata, createdAt, newStatus, newAttempts, diff --git a/messaging/messaging-runtime-core/src/main/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisher.java b/messaging/messaging-runtime-core/src/main/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisher.java index 645e2d6..7298578 100644 --- a/messaging/messaging-runtime-core/src/main/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisher.java +++ b/messaging/messaging-runtime-core/src/main/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisher.java @@ -182,23 +182,45 @@ public final class DefaultMessagePublisher implements MessagePublisher { return rejected("PUBLISH_PREPARATION_FAILED", sanitized(beforeTheWire), startedAt); } + Duration remaining = remainingBudget(options.timeout(), startedAt); + if (remaining.isZero() || remaining.isNegative()) { + // The budget was spent resolving and encoding, so nothing has been transmitted and the + // outcome is still definite. Sending anyway would start a message the caller has already + // stopped waiting for. + return rejected( + "PUBLISH_DEADLINE_EXCEEDED", + "the " + + options.timeout().toMillis() + + "ms publish deadline passed before the message reached the transport", + startedAt); + } + admission.admit(destination.name().value(), encoded.payload().bytes().length); MessagingRuntimeLease lease = null; try { lease = runtimes.acquire(profile.broker()); MessagingRuntimeLease held = lease; - return transport - .publish(new TransportPublishRequest(profile, encoded, options)) + return withDeadline( + transport.publish(new TransportPublishRequest(profile, encoded, options)), remaining) .handle( (result, failure) -> { // One release per acquisition, whatever happened. A permit that leaks on the // failure path is a limiter that shrinks by one per failure until it accepts // nothing. held.close(); - admission.complete(); + admission.complete(destination.name().value()); if (failure != null) { // The request was on the wire when this failed, so the broker may hold it. - PublishResult ambiguous = ambiguousResult(sanitized(failure), startedAt); + PublishResult ambiguous = + isDeadline(failure) + ? ambiguousResult( + "PUBLISH_DEADLINE_EXCEEDED", + "no broker outcome within the " + + options.timeout().toMillis() + + "ms publish deadline", + startedAt) + : ambiguousResult( + "PUBLISH_OUTCOME_UNKNOWN", sanitized(failure), startedAt); observe(profile, ambiguous, startedAt); return ambiguous; } @@ -209,7 +231,7 @@ public final class DefaultMessagePublisher implements MessagePublisher { if (lease != null) { lease.close(); } - admission.complete(); + admission.complete(destination.name().value()); return rejected("PUBLISH_RUNTIME_UNAVAILABLE", sanitized(beforeTheSend), startedAt); } } @@ -217,22 +239,13 @@ public final class DefaultMessagePublisher implements MessagePublisher { /** * Refuses options the destination cannot honour, before anything is encoded or sent. * - *

The transports read {@code request.options()} for nothing: the timeout, the confirmation - * requirement, the deduplication request and the broker hints were all accepted and ignored. A - * caller asking for broker-side deduplication got a publish with no deduplication and no error, - * and then skipped the idempotency it would otherwise have written — which is exactly the case - * {@code PublishDeduplication}'s own javadoc says must be a startup failure rather than a silent - * no-op. + *

The transports accept {@code request.options()} and read nothing from it, so an option this + * destination cannot honour has to be refused here or it is honoured nowhere. A caller asking for + * broker-side deduplication got a publish with no deduplication and no error, and then skipped + * the idempotency it would otherwise have written — which is exactly the case {@code + * PublishDeduplication}'s own javadoc says must be a startup failure rather than a silent no-op. */ private void requireSupportedOptions(DestinationProfile profile, PublishOptions options) { - if (!options.isM1Compatible()) { - // isM1Compatible existed and nothing called it, so an ordinary caller could attach M3 broker - // hints to an M1 publish and have them dropped. - throw new dev.caskeleton.messaging.api.error.MessagingConfigurationException( - "PUBLISH_BROKER_HINTS_UNSUPPORTED", - "broker hints are an M3 surface and this publish path does not forward them; passing" - + " them here would drop them silently"); - } if (options.deduplication().isPresent() && !transport.capabilities(profile.name()).capabilities().deduplicatedPublish()) { throw new dev.caskeleton.messaging.api.error.MessagingConfigurationException( @@ -276,7 +289,7 @@ public final class DefaultMessagePublisher implements MessagePublisher { FailureCategory.CONFIGURATION, code, false, reason, Optional.empty())))); } - private PublishResult ambiguousResult(String reason, long startedAt) { + private PublishResult ambiguousResult(String code, String reason, long startedAt) { return new PublishResult( PublishCompletion.AMBIGUOUS, PublishEvidence.ambiguous(), @@ -286,11 +299,51 @@ public final class DefaultMessagePublisher implements MessagePublisher { elapsedSince(startedAt), Optional.of( new FailureDescriptor( - FailureCategory.AMBIGUOUS, - "PUBLISH_OUTCOME_UNKNOWN", - true, - reason, - Optional.empty()))); + FailureCategory.AMBIGUOUS, code, true, reason, Optional.empty()))); + } + + /** + * Returns what is left of the caller's deadline. + * + *

Measured from the call, not from the send. {@code PublishOptions.timeout()} is documented as + * the publish operation's deadline, so a slow destination lookup or a large encode spends the + * same budget the broker wait does; timing only the transport call would let the total exceed the + * deadline by however long preparation took. + */ + private Duration remainingBudget(Duration timeout, long startedAt) { + return timeout.minus(elapsedSince(startedAt)); + } + + /** + * Bounds the wait for a broker outcome. + * + *

The transports accepted {@code request.options()} and read nothing from it, so the deadline + * belonged to whatever the broker client happened to be configured with — a Rabbit publish + * waiting on a confirm that never arrives had no deadline at all, and the caller's thread was + * pinned to a stage that could not complete. + * + *

The bound is applied to a copy so that expiry never completes the transport's own stage: the + * adapter still owns its in-flight publish and its own bookkeeping. The permit and the runtime + * lease are released when the copy completes, which is deliberate — holding them until a stalled + * broker answers is how a rotation waits forever on a generation nobody is using. + */ + private static CompletableFuture + withDeadline( + CompletionStage inFlight, + Duration remaining) { + return inFlight + .toCompletableFuture() + .copy() + .orTimeout(remaining.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS); + } + + /** Reports whether a stage failed because the deadline passed rather than because of a broker. */ + private static boolean isDeadline(Throwable failure) { + Throwable cause = + failure instanceof java.util.concurrent.CompletionException && failure.getCause() != null + ? failure.getCause() + : failure; + return cause instanceof java.util.concurrent.TimeoutException; } /** diff --git a/messaging/messaging-runtime-core/src/test/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisherTest.java b/messaging/messaging-runtime-core/src/test/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisherTest.java index 98c0a3d..1caad12 100644 --- a/messaging/messaging-runtime-core/src/test/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisherTest.java +++ b/messaging/messaging-runtime-core/src/test/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisherTest.java @@ -162,6 +162,66 @@ class DefaultMessagePublisherTest { assertThat(admission.inFlight()).isZero(); } + @Test + @DisplayName("a broker that never answers is ambiguous at the caller's deadline") + void aStalledBrokerIsAmbiguousAtTheDeadline() { + transport.stall(); + + CompletableFuture pending = + publisher(allowOrders()) + .publish( + new MessageDestination<>(ORDERS, new MessageType("order.created"), String.class), + envelope(), + withTimeout(Duration.ofMillis(120))) + .toCompletableFuture(); + + // Bounded rather than joined: with no deadline of its own this stage never completes, and a + // test that waits forever reports a hang instead of the missing deadline. + assertThat(pending).succeedsWithin(Duration.ofSeconds(5)); + PublishResult result = pending.join(); + + assertThat(result.completion()) + .as("the message reached the transport, so the broker may hold it") + .isEqualTo(PublishCompletion.AMBIGUOUS); + assertThat(result.failure().orElseThrow().code()).isEqualTo("PUBLISH_DEADLINE_EXCEEDED"); + assertThat(admission.inFlight()) + .as("a permit held until a stalled broker answers is a permit never returned") + .isZero(); + } + + @Test + @DisplayName("a deadline spent before the send rejects instead of publishing") + void aDeadlineSpentBeforeTheSendRejects() { + // Preparation is inside the deadline, not before it: resolution and encoding spend the same + // budget the broker wait does. + AtomicInteger reads = new AtomicInteger(); + DefaultMessagePublisher publisher = + new DefaultMessagePublisher( + registry(), + allowOrders(), + new SingleCodecRegistry(), + admission, + new FixedRuntimeRegistry(transport), + transport, + () -> Duration.ofSeconds(6L * reads.getAndIncrement()).toNanos()); + + PublishResult result = + publisher + .publish( + new MessageDestination<>(ORDERS, new MessageType("order.created"), String.class), + envelope(), + withTimeout(Duration.ofSeconds(5))) + .toCompletableFuture() + .join(); + + assertThat(result.completion()) + .as("nothing was transmitted, so the outcome is definite") + .isEqualTo(PublishCompletion.REJECTED); + assertThat(result.failure().orElseThrow().code()).isEqualTo("PUBLISH_DEADLINE_EXCEEDED"); + assertThat(transport.published()).isZero(); + assertThat(admission.inFlight()).isZero(); + } + @Test @DisplayName("an unavailable runtime releases the permit it took") void anUnavailableRuntimeReleasesThePermit() { @@ -262,8 +322,7 @@ class DefaultMessagePublisherTest { .REPLICATION_OR_PERSISTENCE_ACK, java.util.Optional.of( new dev.caskeleton.messaging.api.publish.PublishDeduplication( - "order-1", java.time.Duration.ofMinutes(5))), - Map.of())) + "order-1", java.time.Duration.ofMinutes(5))))) .toCompletableFuture() .join(); @@ -273,26 +332,12 @@ class DefaultMessagePublisherTest { assertThat(transport.published()).isZero(); } - @Test - @DisplayName("M3 broker hints are refused on the M1 path") - void brokerHintsAreRefused() { - PublishResult result = - publisher(allowOrders()) - .publish( - new MessageDestination<>(ORDERS, new MessageType("order.created"), String.class), - envelope(), - new PublishOptions( - java.time.Duration.ofSeconds(5), - dev.caskeleton.messaging.api.destination.ConfirmationRequirement - .REPLICATION_OR_PERSISTENCE_ACK, - java.util.Optional.empty(), - Map.of("linger.ms", "50"))) - .toCompletableFuture() - .join(); - - assertThat(result.completion()) - .as("isM1Compatible existed and nothing ever called it") - .isEqualTo(PublishCompletion.REJECTED); + private static PublishOptions withTimeout(Duration timeout) { + return new PublishOptions( + timeout, + dev.caskeleton.messaging.api.destination.ConfirmationRequirement + .REPLICATION_OR_PERSISTENCE_ACK, + Optional.empty()); } private static DestinationAccessPolicy allowOrders() { @@ -384,15 +429,26 @@ class DefaultMessagePublisherTest { private RuntimeException failure; + private boolean stalled; + private void failWith(RuntimeException failure) { this.failure = failure; } + /** A broker that took the message and never says what happened to it. */ + private void stall() { + this.stalled = true; + } + @Override public CompletionStage publish(TransportPublishRequest request) { if (failure != null) { return CompletableFuture.failedFuture(failure); } + if (stalled) { + published.incrementAndGet(); + return new CompletableFuture<>(); + } published.incrementAndGet(); return CompletableFuture.completedFuture( new TransportPublishResult( diff --git a/messaging/messaging-schema-avro/build.gradle b/messaging/messaging-schema-avro/build.gradle index 22844df..458cf43 100644 --- a/messaging/messaging-schema-avro/build.gradle +++ b/messaging/messaging-schema-avro/build.gradle @@ -4,5 +4,8 @@ dependencies { api project(':messaging:messaging-core-api') api project(':messaging:messaging-schema-api') - implementation 'org.apache.avro:avro:1.12.0' + // api: AvroMessageCodec's constructors take a registry of org.apache.avro.Schema and + // AvroCompatibilityGate.check takes and compares them. A consumer cannot build that + // registry without naming the type, so hiding the dependency only stops them compiling. + api 'org.apache.avro:avro:1.12.0' } diff --git a/messaging/messaging-schema-avro/src/main/java/dev/caskeleton/messaging/schema/avro/AvroMessageCodec.java b/messaging/messaging-schema-avro/src/main/java/dev/caskeleton/messaging/schema/avro/AvroMessageCodec.java index fd2c6da..631a6b0 100644 --- a/messaging/messaging-schema-avro/src/main/java/dev/caskeleton/messaging/schema/avro/AvroMessageCodec.java +++ b/messaging/messaging-schema-avro/src/main/java/dev/caskeleton/messaging/schema/avro/AvroMessageCodec.java @@ -157,11 +157,14 @@ public final class AvroMessageCodec implements MessageCodec { "the Avro codec decodes into GenericRecord, not " + payloadType.getSimpleName()); } - DatumReader reader = new GenericDatumReader<>(writerSchema, writerSchema); + DatumReader reader = boundedReader(writerSchema, writerSchema); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(encoded, null); try { return payloadType.cast(reader.read(null, decoder)); } catch (IOException | RuntimeException failure) { + if (failure instanceof MessageTooLargeException tooLarge) { + throw tooLarge; + } throw new MessageSerializationException( "AVRO_DECODE_FAILED", "the payload could not be decoded against the registered schema", @@ -200,11 +203,14 @@ public final class AvroMessageCodec implements MessageCodec { Schema writerSchema = schemaFor(type, writerVersion); Schema readerSchema = schemaFor(type, readerVersion); - DatumReader reader = new GenericDatumReader<>(writerSchema, readerSchema); + DatumReader reader = boundedReader(writerSchema, readerSchema); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(encoded, null); try { return reader.read(null, decoder); } catch (IOException | RuntimeException failure) { + if (failure instanceof MessageTooLargeException tooLarge) { + throw tooLarge; + } throw new MessageSerializationException( "AVRO_EVOLUTION_FAILED", "v%d bytes could not be resolved against the v%d reader schema" @@ -213,6 +219,38 @@ public final class AvroMessageCodec implements MessageCodec { } } + /** + * Builds a reader that will not allocate an array the input could not possibly fill. + * + *

Avro writes an array as a declared element count followed by the elements. The count is a + * variable-length integer, so five bytes can claim four hundred million elements, and the generic + * reader allocates the backing array from that claim before reading a single element. Bounding + * the input length does not help: the whole hostile payload is five bytes, well under any limit, + * and the failure is an {@code OutOfMemoryError} rather than an exception the codec could report + * — on a consumer thread that is the process, not the message. + * + *

The ceiling is the byte limit itself. Every element costs at least one byte on the wire even + * when it is empty, so a payload of at most {@code maxBytes} bytes cannot honestly contain more + * than {@code maxBytes} elements, and any larger claim is a lie the reader should refuse rather + * than reserve memory for. + */ + private DatumReader boundedReader(Schema writerSchema, Schema readerSchema) { + int maxElements = maxBytes; + return new GenericDatumReader<>(writerSchema, readerSchema) { + @Override + protected Object newArray(Object old, int size, Schema schema) { + if (size > maxElements) { + throw new MessageTooLargeException( + "AVRO_COLLECTION_TOO_LARGE", + ("the payload declares an array of %d elements, which %d bytes cannot contain; the" + + " claim was refused before the array was allocated") + .formatted(size, maxElements)); + } + return super.newArray(old, size, schema); + } + }; + } + private Schema schemaFor(MessageType type, SchemaVersion version) { Objects.requireNonNull(type, "type must not be null"); Objects.requireNonNull(version, "version must not be null"); diff --git a/messaging/messaging-schema-avro/src/test/java/dev/caskeleton/messaging/schema/avro/AvroHostileInputTest.java b/messaging/messaging-schema-avro/src/test/java/dev/caskeleton/messaging/schema/avro/AvroHostileInputTest.java new file mode 100644 index 0000000..c066883 --- /dev/null +++ b/messaging/messaging-schema-avro/src/test/java/dev/caskeleton/messaging/schema/avro/AvroHostileInputTest.java @@ -0,0 +1,134 @@ +package dev.caskeleton.messaging.schema.avro; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.error.MessageSerializationException; +import dev.caskeleton.messaging.api.error.MessageTooLargeException; +import java.io.ByteArrayOutputStream; +import java.util.List; +import java.util.Map; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * What a hostile Avro payload can make the decoder allocate. + * + *

The byte limit is the wrong instrument for this attack and was the only one in place. Avro + * writes a collection as a declared element count followed by the elements, and the count is a + * variable-length integer: five bytes claim four hundred million elements. Those five bytes pass + * every length check the codec applies, and the generic reader then reserves the backing array from + * the claim before reading a single element. The result is an {@code OutOfMemoryError} — not an + * exception the codec can classify, not a message the consumer can dead-letter, and on a broker + * consumer thread it takes the process with it. + * + *

Strings, byte arrays and maps were already safe: Avro validates those lengths against the + * bytes actually remaining. Arrays were the one shape that allocated on trust, which is why the + * corpus below is narrow rather than exhaustive — it pins the case that failed, and the two cases + * that must keep working around it. + */ +class AvroHostileInputTest { + + private static final MessageType BASKET = new MessageType("basket.filled"); + private static final SchemaVersion V1 = new SchemaVersion(1); + private static final SchemaVersion V2 = new SchemaVersion(2); + + private static final Schema V1_SCHEMA = + new Schema.Parser() + .parse( + """ + { + "type": "record", + "name": "BasketFilled", + "namespace": "dev.caskeleton.messaging.sample", + "fields": [ + {"name": "items", "type": {"type": "array", "items": "string"}} + ] + } + """); + + private static final Schema V2_SCHEMA = + new Schema.Parser() + .parse( + """ + { + "type": "record", + "name": "BasketFilled", + "namespace": "dev.caskeleton.messaging.sample", + "fields": [ + {"name": "items", "type": {"type": "array", "items": "string"}}, + {"name": "channel", "type": "string", "default": "WEB"} + ] + } + """); + + @Test + @DisplayName("an array length no payload of this size could hold is refused before allocation") + void aDeclaredArrayLengthTheInputCannotContainIsRefused() { + AvroMessageCodec codec = new AvroMessageCodec(Map.of(BASKET, Map.of(V1, V1_SCHEMA))); + byte[] hostile = zigzag(400_000_000L); + + assertThat(hostile) + .as("the whole attack is five bytes, so no byte limit stands between it and the allocation") + .hasSizeLessThan(16); + assertThatThrownBy(() -> codec.decode(BASKET, V1, hostile, GenericRecord.class)) + .isInstanceOf(MessageTooLargeException.class) + .hasMessageContaining("400000000"); + } + + @Test + @DisplayName("the evolution decode refuses the same claim") + void theEvolutionDecodeAppliesTheSameCollectionBound() { + AvroMessageCodec codec = + new AvroMessageCodec(Map.of(BASKET, Map.of(V1, V1_SCHEMA, V2, V2_SCHEMA))); + + assertThatThrownBy(() -> codec.decodeEvolved(BASKET, V1, V2, zigzag(400_000_000L))) + .as("a consumer reading a newer producer takes this path for every message") + .isInstanceOf(MessageTooLargeException.class); + } + + @Test + @DisplayName("a count too large for an int fails as a decode error, not an allocation") + void aCountBeyondIntRangeFailsWhileReadingRatherThanWhileReserving() { + AvroMessageCodec codec = new AvroMessageCodec(Map.of(BASKET, Map.of(V1, V1_SCHEMA))); + + assertThatThrownBy(() -> codec.decode(BASKET, V1, zigzag(4_294_967_296L), GenericRecord.class)) + .as("the count truncates to a harmless int, and the read then runs out of input") + .isInstanceOf(MessageSerializationException.class); + } + + @Test + @DisplayName("an array a payload really contains still round trips") + void anHonestArrayIsUnaffectedByTheBound() { + AvroMessageCodec codec = new AvroMessageCodec(Map.of(BASKET, Map.of(V1, V1_SCHEMA))); + GenericRecord basket = new GenericData.Record(V1_SCHEMA); + basket.put("items", List.of("apple", "pear", "plum")); + + byte[] encoded = codec.encode(BASKET, V1, basket).bytes(); + GenericRecord decoded = codec.decode(BASKET, V1, encoded, GenericRecord.class); + + assertThat(decoded.get("items").toString()).isEqualTo("[apple, pear, plum]"); + } + + /** + * Writes one Avro variable-length integer, the way a producer encodes a collection count. + * + *

Hand-written rather than taken from an encoder because the point is to write a count with no + * elements behind it, which no encoder will do. + */ + private static byte[] zigzag(long value) { + long remaining = (value << 1) ^ (value >> 63); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + while ((remaining & ~0x7FL) != 0) { + bytes.write((int) ((remaining & 0x7F) | 0x80)); + remaining >>>= 7; + } + bytes.write((int) remaining); + return bytes.toByteArray(); + } +} diff --git a/messaging/messaging-schema-protobuf/build.gradle b/messaging/messaging-schema-protobuf/build.gradle index d4854f6..560d1df 100644 --- a/messaging/messaging-schema-protobuf/build.gradle +++ b/messaging/messaging-schema-protobuf/build.gradle @@ -4,5 +4,7 @@ dependencies { api project(':messaging:messaging-core-api') api project(':messaging:messaging-schema-api') - implementation 'com.google.protobuf:protobuf-java:4.29.3' + // api: ProtobufMessageContract is a public record over com.google.protobuf.Message and + // Parser, and registering a contract is the first thing a consumer of this codec does. + api 'com.google.protobuf:protobuf-java:4.29.3' } diff --git a/messaging/messaging-schema-protobuf/src/test/java/dev/caskeleton/messaging/schema/protobuf/ProtobufCompatibilityTest.java b/messaging/messaging-schema-protobuf/src/test/java/dev/caskeleton/messaging/schema/protobuf/ProtobufCompatibilityTest.java index 9be1fa6..65fe8c5 100644 --- a/messaging/messaging-schema-protobuf/src/test/java/dev/caskeleton/messaging/schema/protobuf/ProtobufCompatibilityTest.java +++ b/messaging/messaging-schema-protobuf/src/test/java/dev/caskeleton/messaging/schema/protobuf/ProtobufCompatibilityTest.java @@ -15,6 +15,7 @@ import com.google.protobuf.Parser; import dev.caskeleton.messaging.api.ContentType; import dev.caskeleton.messaging.api.MessageType; import dev.caskeleton.messaging.api.SchemaVersion; +import dev.caskeleton.messaging.api.error.MessageSerializationException; import dev.caskeleton.messaging.api.error.MessageTooLargeException; import dev.caskeleton.messaging.api.error.MessageValidationException; import dev.caskeleton.messaging.api.error.MessagingConfigurationException; @@ -231,6 +232,19 @@ class ProtobufCompatibilityTest { assertThat(codec.encode(ORDER_CREATED, V1, v1Message()).bytes()).hasSize(exact); } + @Test + void aLengthPrefixNoPayloadOfThisSizeCouldHonourIsADecodeFailure() { + ProtobufMessageCodec codec = codec(V1_DESCRIPTOR); + // Tag 1, wire type 2 (length-delimited), then a varint claiming four hundred million bytes + // follow. The whole message is six bytes, so it passes the size limit; what must not happen is + // the parser reserving the claimed length before discovering there is nothing behind it. + byte[] hostile = {0x0A, (byte) 0x80, (byte) 0x88, (byte) 0xDE, (byte) 0xBE, 0x01}; + + assertThatThrownBy(() -> codec.decode(ORDER_CREATED, V1, hostile, DynamicMessage.class)) + .isInstanceOf(MessageSerializationException.class) + .hasMessageContaining("protobuf"); + } + @Test void theEncodedMessageCarriesItsSchemaReference() { EncodedMessage encoded = codec(V1_DESCRIPTOR, V2).encode(ORDER_CREATED, V2, v1Message()); diff --git a/messaging/messaging-spring-boot-starter/build.gradle b/messaging/messaging-spring-boot-starter/build.gradle index 4e83842..2fe8152 100644 --- a/messaging/messaging-spring-boot-starter/build.gradle +++ b/messaging/messaging-spring-boot-starter/build.gradle @@ -41,11 +41,18 @@ dependencies { annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' // The Reactor facade lives here, not in core-api: the core contract stays CompletionStage so - // that a service which does not use Reactor never inherits it. - implementation 'io.projectreactor:reactor-core' + // that a service which does not use Reactor never inherits it. api, because + // ReactiveMessagePublisher returns Mono — a facade whose return type an adopter cannot name + // is a facade they cannot call. + api 'io.projectreactor:reactor-core' testImplementation 'io.projectreactor:reactor-test' testImplementation 'org.springframework.boot:spring-boot-test' testImplementation 'org.springframework:spring-test' + // Test-only, and specifically so the configuration reference can be executed rather than read: + // MessagingConfigurationBindingTest loads the YAML blocks out of the shipped document and boots + // a context on them. Spring Boot 4 treats snakeyaml as optional, so a document-driven test has + // to ask for it; production code does not, and adding it here keeps that asymmetry visible. + testImplementation 'org.yaml:snakeyaml' // The live round trip. MSG-INT-003's membership rule is that the starter earns // app-bootstrap membership in the same change that proves a real broker round trip, // so the proof has to run somewhere — and the starter is where the whole platform diff --git a/messaging/messaging-spring-boot-starter/gradle.lockfile b/messaging/messaging-spring-boot-starter/gradle.lockfile index 3170cc1..020a2ea 100644 --- a/messaging/messaging-spring-boot-starter/gradle.lockfile +++ b/messaging/messaging-spring-boot-starter/gradle.lockfile @@ -137,6 +137,7 @@ org.testcontainers:testcontainers-kafka:2.0.2=testCompileClasspath,testRuntimeCl org.testcontainers:testcontainers:2.0.2=testCompileClasspath,testRuntimeClasspath org.xerial.snappy:snappy-java:1.1.10.7=runtimeClasspath,testRuntimeClasspath org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs +org.yaml:snakeyaml:2.5=testCompileClasspath,testRuntimeClasspath tools.jackson.core:jackson-core:3.0.2=runtimeClasspath,testRuntimeClasspath tools.jackson.core:jackson-databind:3.0.2=runtimeClasspath,testRuntimeClasspath tools.jackson:jackson-bom:3.0.2=runtimeClasspath,testRuntimeClasspath diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/BrokerSecuritySettings.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/BrokerSecuritySettings.java new file mode 100644 index 0000000..0b629b1 --- /dev/null +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/BrokerSecuritySettings.java @@ -0,0 +1,50 @@ +package dev.caskeleton.messaging.autoconfigure; + +import java.util.List; +import org.springframework.boot.context.properties.bind.DefaultValue; + +/** + * One entry of {@code app.messaging.security}, keyed by the broker it secures. + * + *

No field here holds a secret, and that is a structural rule rather than a convention: the + * platform stores a credential id and resolves the material through a {@code CredentialProvider} at + * connect time, so a configuration dump or a heap dump yields a name and nothing usable. + * + *

The admin credential is bindable and its presence on a production broker is a startup failure. + * Leaving it unbindable would have been the safer-looking choice and the wrong one: an operator who + * needs it in a non-production runtime would then configure it somewhere the platform cannot see, + * and the check that refuses it in production would have nothing to refuse. + * + * @param producer the producer credential + * @param consumer the consumer credential + * @param admin the admin credential, which an application runtime must not hold in production + * @param hostnameVerification whether the server certificate hostname is verified + * @param access which destinations each role may touch + */ +public record BrokerSecuritySettings( + Credential producer, + Credential consumer, + Credential admin, + @DefaultValue("true") boolean hostnameVerification, + @DefaultValue Access access) { + + /** + * One credential reference. + * + * @param type the authentication mechanism + * @param credentialId the identifier the credential provider resolves + */ + public record Credential(String type, String credentialId) {} + + /** + * Which destinations each role may touch. + * + * @param publishable destinations the producer credential may publish to + * @param consumable destinations the consumer credential may consume + * @param administrable destinations the admin credential may operate on + */ + public record Access( + @DefaultValue List publishable, + @DefaultValue List consumable, + @DefaultValue List administrable) {} +} diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/BrokerSettings.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/BrokerSettings.java new file mode 100644 index 0000000..5274dd5 --- /dev/null +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/BrokerSettings.java @@ -0,0 +1,69 @@ +package dev.caskeleton.messaging.autoconfigure; + +import java.time.Duration; +import java.util.List; +import org.springframework.boot.context.properties.bind.DefaultValue; + +/** + * One entry of {@code app.messaging.brokers}. + * + *

Both broker families are described by one record because a deployment names its brokers in one + * map, and a reader of that map should not have to know which half of the fields apply. Which half + * does apply is decided by {@link #type()}, and the compiler refuses a field belonging to the other + * family rather than ignoring it — a Kafka broker carrying {@code prefetch} is a configuration the + * author believed was doing something. + * + *

{@code tlsEnabled} and {@code production} live here rather than beside the credentials, so a + * broker is described once. They were previously implied by whichever object happened to be + * constructed, which is how a profile can be production in one validator's view and not in + * another's. + * + * @param type the broker family, {@code kafka} or {@code rabbitmq} + * @param stable whether this profile claims the Stable guarantees + * @param production whether this profile is used in production + * @param tlsEnabled whether transport encryption is on + * @param authenticationEnabled whether broker authentication is on + * @param bootstrapServers Kafka broker addresses + * @param enableIdempotence Kafka producer duplicate suppression on internal retry + * @param acks the Kafka acknowledgement level + * @param maxInFlightRequestsPerConnection the Kafka in-flight request ceiling + * @param deliveryTimeout the Kafka producer delivery deadline + * @param enableAutoCommit whether the Kafka consumer commits offsets on a timer + * @param consumerGroup the Kafka consumer group + * @param addresses RabbitMQ broker addresses + * @param publisherConfirms whether correlated publisher confirms are on + * @param publisherReturns whether unroutable returns are delivered + * @param mandatory whether publishes carry the mandatory flag + * @param confirmTimeout how long to wait for a RabbitMQ confirm + * @param autoAck whether the RabbitMQ consumer acknowledges on delivery + * @param prefetch the RabbitMQ consumer prefetch window + * @param quorumQueues whether durable work queues are declared as quorum queues + */ +public record BrokerSettings( + String type, + @DefaultValue("false") boolean stable, + @DefaultValue("false") boolean production, + @DefaultValue("false") boolean tlsEnabled, + @DefaultValue("false") boolean authenticationEnabled, + @DefaultValue List bootstrapServers, + @DefaultValue("true") boolean enableIdempotence, + @DefaultValue("all") String acks, + @DefaultValue("5") int maxInFlightRequestsPerConnection, + @DefaultValue("30s") Duration deliveryTimeout, + @DefaultValue("false") boolean enableAutoCommit, + @DefaultValue("") String consumerGroup, + @DefaultValue List addresses, + @DefaultValue("true") boolean publisherConfirms, + @DefaultValue("true") boolean publisherReturns, + @DefaultValue("true") boolean mandatory, + @DefaultValue("5s") Duration confirmTimeout, + @DefaultValue("false") boolean autoAck, + @DefaultValue("16") int prefetch, + @DefaultValue("true") boolean quorumQueues) { + + /** The broker family a Kafka entry declares. */ + public static final String KAFKA = "kafka"; + + /** The broker family a RabbitMQ entry declares. */ + public static final String RABBITMQ = "rabbitmq"; +} diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/CompiledMessagingConfiguration.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/CompiledMessagingConfiguration.java new file mode 100644 index 0000000..661fcd6 --- /dev/null +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/CompiledMessagingConfiguration.java @@ -0,0 +1,53 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.kafka.KafkaBrokerProfile; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.rabbit.RabbitBrokerProfile; +import dev.caskeleton.messaging.security.BrokerSecurityProfile; +import java.util.List; +import java.util.Objects; + +/** + * What the documented configuration compiles to. + * + *

One object rather than four beans, because the four are only meaningful together: a + * destination names a broker, a security entry secures one, and a compile that produced three of + * the four would leave the fourth silently empty. Holding them as one result means the startup + * validations either all see this deployment's configuration or the compile failed. + * + * @param destinations the destination profiles built from {@code app.messaging.destinations} + * @param kafkaBrokers the Kafka profiles built from {@code app.messaging.brokers} + * @param rabbitBrokers the RabbitMQ profiles built from {@code app.messaging.brokers} + * @param security the broker security profiles built from {@code app.messaging.security} + */ +public record CompiledMessagingConfiguration( + List destinations, + List kafkaBrokers, + List rabbitBrokers, + List security) { + + public CompiledMessagingConfiguration { + Objects.requireNonNull(destinations, "destinations must not be null"); + Objects.requireNonNull(kafkaBrokers, "kafkaBrokers must not be null"); + Objects.requireNonNull(rabbitBrokers, "rabbitBrokers must not be null"); + Objects.requireNonNull(security, "security must not be null"); + destinations = List.copyOf(destinations); + kafkaBrokers = List.copyOf(kafkaBrokers); + rabbitBrokers = List.copyOf(rabbitBrokers); + security = List.copyOf(security); + } + + /** + * Returns the result of compiling a deployment that configured nothing. + * + *

Empty is a legitimate state, not a failure: an application may still register its profiles + * as beans, which is how this platform worked before anything bound. What is not legitimate is a + * deployment that configured destinations and got this — which is why the compiler builds it only + * when the sections really are absent. + * + * @return the empty compilation + */ + public static CompiledMessagingConfiguration empty() { + return new CompiledMessagingConfiguration(List.of(), List.of(), List.of(), List.of()); + } +} diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DestinationSettings.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DestinationSettings.java new file mode 100644 index 0000000..35304c7 --- /dev/null +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/DestinationSettings.java @@ -0,0 +1,177 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.api.delivery.DeliveryGuarantee; +import dev.caskeleton.messaging.api.delivery.ExternalSideEffectGuarantee; +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.ConfirmationRequirement; +import dev.caskeleton.messaging.api.destination.DestinationKind; +import dev.caskeleton.messaging.policy.CapabilityTier; +import dev.caskeleton.messaging.policy.OrderingImpact; +import dev.caskeleton.messaging.policy.RetryMode; +import dev.caskeleton.messaging.schema.SchemaCompatibility; +import java.time.Duration; +import java.util.List; +import org.springframework.boot.context.properties.bind.DefaultValue; + +/** + * One entry of {@code app.messaging.destinations}. + * + *

The shape is the one the configuration reference documents, and it is bound rather than + * described: the reference showed destination, broker and security sections for a platform whose + * only binder held four flags, so an operator who configured exactly what the page said changed + * nothing and was told nothing. + * + *

Records rather than mutable nested beans. A destination profile is a contract the platform + * validates once at startup and then relies on; a settings object whose fields can be reassigned + * afterwards offers a second, unvalidated way for that contract to change. + * + *

Defaults are the conservative ones. Retry is off, ordering is unconstrained and nothing is a + * production profile until it says so, because every one of those defaults, chosen the other way, + * is a guarantee the deployment never asked for and would discover only under load. + * + * @param broker the logical broker this destination binds to, keyed into {@code + * app.messaging.brokers} + * @param kind the interaction pattern + * @param tier the API tier this destination is approved for + * @param physical the broker-specific address + * @param schema the codec and compatibility contract + * @param guarantees the delivery, ordering and side-effect contract + * @param producer the producer contract + * @param consumer the consumer contract + * @param retry the retry contract + * @param dlq the dead letter contract + * @param payload the size contract + * @param keyResolverConfigured whether a key resolver is registered for keyed ordering + * @param production whether this profile is a production profile + * @param topologyAutoCreate whether the application may create topology itself + */ +public record DestinationSettings( + String broker, + DestinationKind kind, + @DefaultValue("M1") CapabilityTier tier, + @DefaultValue Physical physical, + @DefaultValue Schema schema, + @DefaultValue Guarantees guarantees, + @DefaultValue Producer producer, + @DefaultValue Consumer consumer, + @DefaultValue Retry retry, + @DefaultValue Dlq dlq, + @DefaultValue Payload payload, + @DefaultValue("false") boolean keyResolverConfigured, + @DefaultValue("false") boolean production, + @DefaultValue("false") boolean topologyAutoCreate) { + + /** + * The broker-specific address, of which exactly one family is filled in. + * + * @param topic the Kafka or Pulsar topic + * @param exchange the AMQP exchange + * @param routingKey the AMQP routing key + * @param queue the AMQP queue + * @param subject the NATS subject + * @param stream the NATS or Pulsar stream + */ + public record Physical( + String topic, + String exchange, + String routingKey, + String queue, + String subject, + String stream) {} + + /** + * The codec and compatibility contract. + * + * @param codec the content type payloads are encoded with + * @param compatibility the schema evolution mode enforced here + * @param messageTypes the closed catalog of types this destination carries + */ + public record Schema( + @DefaultValue("application/json") String codec, + @DefaultValue("BACKWARD_TRANSITIVE") SchemaCompatibility compatibility, + @DefaultValue List messageTypes) {} + + /** + * What the destination promises about delivery and order. + * + * @param delivery the broker-level delivery guarantee + * @param ordering the scope inside which order holds + * @param externalSideEffect how handler side effects are protected + */ + public record Guarantees( + @DefaultValue("AT_LEAST_ONCE") DeliveryGuarantee delivery, + @DefaultValue("NONE") OrderingScope ordering, + @DefaultValue("IDEMPOTENCY_REQUIRED") ExternalSideEffectGuarantee externalSideEffect) {} + + /** + * The producer contract. + * + * @param confirmation the confirmation strength this destination demands + * @param timeout the publish deadline + * @param mandatoryRouting whether the broker must report unroutable publishes + * @param idempotent whether producer-side duplicate suppression is required + */ + public record Producer( + @DefaultValue("REPLICATION_OR_PERSISTENCE_ACK") ConfirmationRequirement confirmation, + @DefaultValue("5s") Duration timeout, + @DefaultValue("true") boolean mandatoryRouting, + @DefaultValue("true") boolean idempotent) {} + + /** + * The consumer contract. + * + * @param group the consumer group, where the broker has one + * @param concurrency how many deliveries may be processed at once + * @param maxInFlightPerOrderingUnit the in-flight ceiling inside one ordering unit + * @param prefetch the broker prefetch window + * @param handlerTimeout the handler deadline + * @param manualSettlement whether this destination uses the M2 manual settlement tier + */ + public record Consumer( + String group, + @DefaultValue("1") int concurrency, + @DefaultValue("1") int maxInFlightPerOrderingUnit, + @DefaultValue("16") int prefetch, + @DefaultValue("30s") Duration handlerTimeout, + @DefaultValue("false") boolean manualSettlement) {} + + /** + * The retry contract, off by default. + * + * @param mode how the retry is carried out + * @param maxAttempts total attempts including the first delivery + * @param initialDelay the first backoff interval + * @param maxDelay the backoff ceiling + * @param multiplier the exponential backoff factor + * @param jitter whether to randomise the backoff + * @param orderingImpact what this policy is allowed to do to order + * @param destination the destination messages are re-published to, when the mode uses one + */ + public record Retry( + @DefaultValue("NONE") RetryMode mode, + @DefaultValue("1") int maxAttempts, + @DefaultValue("0s") Duration initialDelay, + @DefaultValue("0s") Duration maxDelay, + @DefaultValue("1.0") double multiplier, + @DefaultValue("false") boolean jitter, + @DefaultValue("PRESERVE") OrderingImpact orderingImpact, + String destination) {} + + /** + * Where permanently failed messages go. + * + * @param destination the dead letter destination; absent means dead lettering is off + * @param maxRedriveCount how many times a message may be redriven back + */ + public record Dlq(String destination, @DefaultValue("1") int maxRedriveCount) {} + + /** + * The size contract. + * + * @param maxBytes the encoded payload ceiling for this destination + * @param claimCheckThresholdBytes the size above which payloads are offloaded + */ + public record Payload( + @DefaultValue("1048576") int maxBytes, + @DefaultValue("1048576") int claimCheckThresholdBytes) {} +} diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/KafkaMessagingAutoConfiguration.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/KafkaMessagingAutoConfiguration.java index b68ea16..5525380 100644 --- a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/KafkaMessagingAutoConfiguration.java +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/KafkaMessagingAutoConfiguration.java @@ -6,6 +6,7 @@ import dev.caskeleton.messaging.kafka.KafkaSecurityConfigurer; import dev.caskeleton.messaging.kafka.KafkaTransactionProfileValidator; import dev.caskeleton.messaging.security.BrokerTlsPolicy; import dev.caskeleton.messaging.security.CredentialRuntimeRegistry; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -41,6 +42,28 @@ public class KafkaMessagingAutoConfiguration { return new KafkaProfileValidator(); } + /** + * Runs the Kafka profile validator over every Kafka broker profile, declared or configured. + * + * @param profiles the broker profiles the application declared as beans + * @param compiled the broker profiles compiled from {@code app.messaging.brokers} + * @param validator the Kafka profile validator + * @return the startup validation + */ + @Bean + public StartupProfileValidation + kafkaProfileStartupValidation( + ObjectProvider profiles, + CompiledMessagingConfiguration compiled, + KafkaProfileValidator validator) { + return new StartupProfileValidation<>( + () -> + java.util.stream.Stream.concat( + compiled.kafkaBrokers().stream(), profiles.orderedStream()) + .toList(), + validator::validate); + } + /** * Returns the Kafka transaction profile validator. * diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingConfigurationCompiler.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingConfigurationCompiler.java new file mode 100644 index 0000000..ca76445 --- /dev/null +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingConfigurationCompiler.java @@ -0,0 +1,331 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.api.ContentType; +import dev.caskeleton.messaging.api.MessageType; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import dev.caskeleton.messaging.kafka.KafkaBrokerProfile; +import dev.caskeleton.messaging.policy.ConsumerPolicy; +import dev.caskeleton.messaging.policy.DeadLetterPolicy; +import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.PayloadPolicy; +import dev.caskeleton.messaging.policy.PhysicalDestination; +import dev.caskeleton.messaging.policy.ProducerPolicy; +import dev.caskeleton.messaging.policy.RetryPolicy; +import dev.caskeleton.messaging.policy.SchemaPolicy; +import dev.caskeleton.messaging.rabbit.RabbitBrokerProfile; +import dev.caskeleton.messaging.security.BrokerCredentialProfile; +import dev.caskeleton.messaging.security.BrokerSecurityProfile; +import dev.caskeleton.messaging.security.DestinationAccessPolicy; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; + +/** + * Turns the documented configuration into the objects the platform already validates. + * + *

The gap this closes is not a missing feature but a missing binder. {@code + * docs/messaging/configuration-reference.md} described destination, broker and security sections; + * the only thing that bound was four flags, and destination profiles, broker profiles and security + * profiles had to arrive as beans an application wrote in Java. So a deployment that followed the + * documentation configured nothing, and nothing said so — which is the worst of the three possible + * outcomes, the other two being "it works" and "it refuses to start". + * + *

Every refusal here names the key. A configuration error that names a type — "no qualifying + * bean of type DestinationProfile" — tells an operator about the platform's internals rather than + * about the line they have to change. + * + *

Compiling is separate from validating on purpose. This class turns text into objects and + * checks only what the object model itself cannot express — that a destination's broker exists, + * that a dead letter destination is declared, that a security entry secures a broker somebody + * configured. Whether the resulting profile is coherent is {@code DestinationProfileValidator}'s + * question and it is asked at startup over the whole registry, so a configured profile and a + * bean-declared one are held to exactly the same rules. + */ +public final class MessagingConfigurationCompiler { + + private final MessagingSettings settings; + + /** + * Creates a compiler over the bound settings. + * + * @param settings the bound configuration + */ + public MessagingConfigurationCompiler(MessagingSettings settings) { + this.settings = Objects.requireNonNull(settings, "settings must not be null"); + } + + /** + * Compiles the bound configuration. + * + * @return the compiled destinations, broker profiles and security profiles + */ + public CompiledMessagingConfiguration compile() { + Map brokers = settings.getBrokers(); + List kafka = new ArrayList<>(); + List rabbit = new ArrayList<>(); + brokers.forEach( + (name, broker) -> { + switch (familyOf(name, broker)) { + case BrokerSettings.KAFKA -> kafka.add(toKafkaProfile(name, broker)); + case BrokerSettings.RABBITMQ -> rabbit.add(toRabbitProfile(name, broker)); + default -> + throw configurationError( + key("brokers", name, "type"), + "'%s' is not a broker family this platform binds; use '%s' or '%s'" + .formatted(broker.type(), BrokerSettings.KAFKA, BrokerSettings.RABBITMQ)); + } + }); + + List destinations = new ArrayList<>(); + settings + .getDestinations() + .forEach((name, destination) -> destinations.add(toProfile(name, destination, brokers))); + + List security = new ArrayList<>(); + settings + .getSecurity() + .forEach((name, entry) -> security.add(toSecurityProfile(name, entry, brokers))); + + return new CompiledMessagingConfiguration(destinations, kafka, rabbit, security); + } + + private static String familyOf(String name, BrokerSettings broker) { + if (broker.type() == null || broker.type().isBlank()) { + throw configurationError( + key("brokers", name, "type"), + "a broker entry must declare its family; without one the platform cannot tell which of " + + "its settings apply, and would apply none of them"); + } + return broker.type().toLowerCase(Locale.ROOT); + } + + private static KafkaBrokerProfile toKafkaProfile(String name, BrokerSettings broker) { + if (broker.bootstrapServers().isEmpty()) { + throw configurationError( + key("brokers", name, "bootstrap-servers"), + "a Kafka broker needs at least one address; a client with none defaults to " + + "localhost:9092 and fails at the first publish rather than at startup"); + } + return new KafkaBrokerProfile( + name, + broker.stable(), + broker.production(), + broker.bootstrapServers(), + broker.enableIdempotence(), + broker.acks(), + broker.maxInFlightRequestsPerConnection(), + broker.deliveryTimeout(), + broker.enableAutoCommit(), + broker.consumerGroup(), + broker.tlsEnabled(), + broker.authenticationEnabled()); + } + + private static RabbitBrokerProfile toRabbitProfile(String name, BrokerSettings broker) { + if (broker.addresses().isEmpty()) { + throw configurationError( + key("brokers", name, "addresses"), + "a RabbitMQ broker needs at least one address; a client with none defaults to " + + "localhost and connects as guest"); + } + return new RabbitBrokerProfile( + name, + broker.stable(), + broker.production(), + broker.addresses(), + broker.publisherConfirms(), + broker.publisherReturns(), + broker.mandatory(), + broker.confirmTimeout(), + broker.autoAck(), + broker.prefetch(), + broker.quorumQueues(), + broker.tlsEnabled(), + broker.authenticationEnabled()); + } + + private DestinationProfile toProfile( + String name, DestinationSettings settings, Map brokers) { + if (settings.broker() == null || settings.broker().isBlank()) { + throw configurationError( + key("destinations", name, "broker"), "a destination must name the broker it binds to"); + } + if (!brokers.containsKey(settings.broker())) { + throw configurationError( + key("destinations", name, "broker"), + "'%s' is not a configured broker; declared brokers are %s" + .formatted(settings.broker(), brokers.keySet())); + } + if (settings.kind() == null) { + throw configurationError( + key("destinations", name, "kind"), + "a destination must declare its interaction pattern; the platform's retry, ordering and " + + "dead letter rules all read it"); + } + + return new DestinationProfile( + new DestinationName(name), + settings.broker(), + settings.kind(), + physical(name, settings.physical()), + schema(name, settings.schema()), + settings.guarantees().delivery(), + settings.guarantees().ordering(), + settings.guarantees().externalSideEffect(), + new ProducerPolicy( + settings.producer().confirmation(), + settings.producer().timeout(), + settings.producer().mandatoryRouting(), + settings.producer().idempotent()), + new ConsumerPolicy( + Optional.ofNullable(blankToNull(settings.consumer().group())), + settings.consumer().concurrency(), + settings.consumer().maxInFlightPerOrderingUnit(), + settings.consumer().prefetch(), + settings.consumer().handlerTimeout(), + settings.consumer().manualSettlement()), + retry(settings.retry()), + deadLetter(settings.dlq()), + new PayloadPolicy( + settings.payload().maxBytes(), settings.payload().claimCheckThresholdBytes()), + settings.tier(), + settings.production(), + settings.keyResolverConfigured(), + settings.topologyAutoCreate()); + } + + private static PhysicalDestination physical(String name, DestinationSettings.Physical physical) { + PhysicalDestination address = + new PhysicalDestination( + Optional.ofNullable(blankToNull(physical.topic())), + Optional.ofNullable(blankToNull(physical.exchange())), + Optional.ofNullable(blankToNull(physical.routingKey())), + Optional.ofNullable(blankToNull(physical.queue())), + Optional.ofNullable(blankToNull(physical.subject())), + Optional.ofNullable(blankToNull(physical.stream()))); + if (address.isEmpty()) { + throw configurationError( + key("destinations", name, "physical"), + "a destination needs a broker address — a topic, an exchange, a queue, a subject or a " + + "stream; a logical name that maps to nothing cannot be published to"); + } + return address; + } + + private static SchemaPolicy schema(String name, DestinationSettings.Schema schema) { + if (schema.messageTypes().isEmpty()) { + throw configurationError( + key("destinations", name, "schema.message-types"), + "a destination carries a closed catalog of message types; one that accepts whatever it " + + "is handed has no reviewable wire contract"); + } + Set types = new LinkedHashSet<>(); + schema.messageTypes().forEach(type -> types.add(new MessageType(type))); + return new SchemaPolicy(new ContentType(schema.codec()), schema.compatibility(), types); + } + + private static RetryPolicy retry(DestinationSettings.Retry retry) { + return new RetryPolicy( + retry.mode(), + retry.maxAttempts(), + retry.initialDelay(), + retry.maxDelay(), + retry.multiplier(), + retry.jitter(), + retry.orderingImpact(), + Set.of(), + Set.of(), + Optional.ofNullable(blankToNull(retry.destination())).map(DestinationName::new)); + } + + private static DeadLetterPolicy deadLetter(DestinationSettings.Dlq dlq) { + String destination = blankToNull(dlq.destination()); + if (destination == null) { + return DeadLetterPolicy.disabled(); + } + return new DeadLetterPolicy( + true, Optional.of(new DestinationName(destination)), dlq.maxRedriveCount()); + } + + private static BrokerSecurityProfile toSecurityProfile( + String name, BrokerSecuritySettings security, Map brokers) { + BrokerSettings broker = brokers.get(name); + if (broker == null) { + throw configurationError( + key("security", name, ""), + "'%s' is not a configured broker; a security profile for a broker nobody declared " + .formatted(name) + + "secures nothing. Declared brokers are " + + brokers.keySet()); + } + return new BrokerSecurityProfile( + name, + broker.production(), + broker.tlsEnabled(), + security.hostnameVerification(), + credential(name, "producer", security.producer(), () -> true), + credential(name, "consumer", security.consumer(), () -> true), + Optional.ofNullable(security.admin()) + .map(admin -> credential(name, "admin", admin, () -> true)), + new DestinationAccessPolicy( + destinationNames(security.access().publishable()), + destinationNames(security.access().consumable()), + destinationNames(security.access().administrable()))); + } + + private static BrokerCredentialProfile credential( + String broker, + String role, + BrokerSecuritySettings.Credential credential, + Supplier required) { + if (credential == null && Boolean.TRUE.equals(required.get())) { + throw configurationError( + key("security", broker, role), + "a configured broker needs a %s credential; the roles are separate so that a compromised " + .formatted(role) + + "handler has nothing to escalate with"); + } + String type = credential.type() == null ? "" : credential.type().toUpperCase(Locale.ROOT); + String id = credential.credentialId(); + return switch (type) { + case "SASL_SCRAM" -> new BrokerCredentialProfile.SaslScram(id); + case "OAUTH2" -> new BrokerCredentialProfile.OAuth2(id); + case "MUTUAL_TLS" -> new BrokerCredentialProfile.MutualTls(id); + case "USERNAME_PASSWORD" -> new BrokerCredentialProfile.UsernamePassword(id); + case "NKEY" -> new BrokerCredentialProfile.Nkey(id); + default -> + throw configurationError( + key("security", broker, role + ".type"), + "'%s' is not an authentication mechanism this platform implements; the closed set is " + .formatted(credential.type()) + + "SASL_SCRAM, OAUTH2, MUTUAL_TLS, USERNAME_PASSWORD, NKEY"); + }; + } + + private static Set destinationNames(List names) { + Set destinations = new LinkedHashSet<>(); + names.forEach(name -> destinations.add(new DestinationName(name))); + return destinations; + } + + private static String blankToNull(String value) { + return value == null || value.isBlank() ? null : value; + } + + private static String key(String section, String entry, String leaf) { + String path = MessagingSettings.PREFIX + '.' + section + '.' + entry; + return leaf.isEmpty() ? path : path + '.' + leaf; + } + + private static MessagingConfigurationException configurationError(String key, String message) { + return new MessagingConfigurationException( + "MESSAGING_CONFIGURATION_INVALID", key + ": " + message); + } +} diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingConfigurationKeyValidator.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingConfigurationKeyValidator.java new file mode 100644 index 0000000..fdc2779 --- /dev/null +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingConfigurationKeyValidator.java @@ -0,0 +1,227 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.api.error.MessagingConfigurationException; +import java.lang.reflect.RecordComponent; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.EnumerablePropertySource; +import org.springframework.core.env.PropertySource; + +/** + * Refuses a key inside the platform's own configuration sections that nothing binds. + * + *

A misspelt prefix is loud — the whole section is missing and someone notices. A misspelt key + * inside a section that does bind is the opposite: the entry appears, the platform starts, and the + * one setting the operator came to change is the only one that did not take. {@code + * consumer.prefech: 64} is a throughput change that never happened, and the first evidence is the + * incident it was supposed to prevent. + * + *

The legal keys are derived from the settings records rather than listed here. A list would be + * a second description of the same shape, and the failure mode of a second description is that it + * is right on the day it is written: a field added to {@code DestinationSettings} would be rejected + * as unknown by the very check that exists to catch typos. + * + *

Only enumerable property sources are scanned, which in practice means files and explicitly + * supplied properties rather than environment variables. An environment variable arrives as {@code + * APP_MESSAGING_DESTINATIONS_ORDER_EVENTS_CONSUMER_PREFETCH}, and the underscores that separate the + * entry name from the leaf are the same underscores inside both, so there is no way to split it + * back without guessing. Guessing here would mean refusing valid deployments, which is worse than + * missing a typo in the one place typos are least likely — a variable somebody had to spell out in + * a deployment manifest. + */ +public final class MessagingConfigurationKeyValidator implements InitializingBean { + + /** Broker keys that only a Kafka entry may carry. */ + private static final Set KAFKA_ONLY_KEYS = + Set.of( + "bootstrap-servers", + "enable-idempotence", + "acks", + "max-in-flight-requests-per-connection", + "delivery-timeout", + "enable-auto-commit", + "consumer-group"); + + /** Broker keys that only a RabbitMQ entry may carry. */ + private static final Set RABBIT_ONLY_KEYS = + Set.of( + "addresses", + "publisher-confirms", + "publisher-returns", + "mandatory", + "confirm-timeout", + "auto-ack", + "prefetch", + "quorum-queues"); + + private final ConfigurableEnvironment environment; + + /** + * Creates a validator over the resolved environment. + * + * @param environment the environment whose keys are checked + */ + public MessagingConfigurationKeyValidator(ConfigurableEnvironment environment) { + this.environment = Objects.requireNonNull(environment, "environment must not be null"); + } + + @Override + public void afterPropertiesSet() { + validate(); + } + + /** + * Fails when a key under a bound section is not one the settings declare. + * + * @throws MessagingConfigurationException naming the offending key + */ + public void validate() { + Map> sections = + Map.of( + "destinations", DestinationSettings.class, + "brokers", BrokerSettings.class, + "security", BrokerSecuritySettings.class); + + List rejected = new ArrayList<>(); + sections.forEach( + (section, shape) -> { + Set legal = leafPaths(shape); + for (String key : keysUnder(MessagingSettings.PREFIX + '.' + section + '.')) { + Entry entry = split(key, MessagingSettings.PREFIX + '.' + section + '.'); + if (entry.leaf().isEmpty()) { + continue; + } + if (!legal.contains(entry.leaf())) { + rejected.add( + "%s binds nothing; %s entries accept %s" + .formatted(key, section, legal.stream().sorted().toList())); + } else if ("brokers".equals(section)) { + wrongFamily(entry).ifPresent(rejected::add); + } + } + }); + + if (rejected.isEmpty()) { + return; + } + throw new MessagingConfigurationException( + "MESSAGING_CONFIGURATION_KEY_UNKNOWN", + "these messaging keys do not bind: " + + String.join("; ", rejected.stream().sorted().limit(10).toList()) + + (rejected.size() > 10 ? " (and " + (rejected.size() - 10) + " more)" : "")); + } + + /** + * Reports a broker key that belongs to the other broker family. + * + *

Both families are described by one record, so a Kafka entry carrying {@code prefetch} binds + * without complaint and does nothing. The author of that line believed it was doing something, + * which is the same silence this class exists to end. + */ + private java.util.Optional wrongFamily(Entry entry) { + String declared = + environment.getProperty( + MessagingSettings.PREFIX + ".brokers." + entry.name() + ".type", ""); + String family = declared.toLowerCase(Locale.ROOT); + String leaf = entry.leaf(); + if (BrokerSettings.KAFKA.equals(family) && RABBIT_ONLY_KEYS.contains(leaf)) { + return java.util.Optional.of( + "%s.brokers.%s.%s is a RabbitMQ setting on a Kafka broker" + .formatted(MessagingSettings.PREFIX, entry.name(), leaf)); + } + if (BrokerSettings.RABBITMQ.equals(family) && KAFKA_ONLY_KEYS.contains(leaf)) { + return java.util.Optional.of( + "%s.brokers.%s.%s is a Kafka setting on a RabbitMQ broker" + .formatted(MessagingSettings.PREFIX, entry.name(), leaf)); + } + return java.util.Optional.empty(); + } + + /** One configuration key split into the map entry it configures and the leaf inside it. */ + private record Entry(String name, String leaf) {} + + private static Entry split(String key, String sectionPrefix) { + String remainder = key.substring(sectionPrefix.length()); + String name; + String leaf; + if (remainder.startsWith("[")) { + // Relaxed binding writes a map key containing a dot as `[order.events]`, so the bracket is + // the only reliable boundary when one is present. + int close = remainder.indexOf(']'); + name = close < 0 ? remainder : remainder.substring(1, close); + leaf = close < 0 ? "" : remainder.substring(close + 1); + } else { + int dot = remainder.indexOf('.'); + name = dot < 0 ? remainder : remainder.substring(0, dot); + leaf = dot < 0 ? "" : remainder.substring(dot); + } + if (leaf.startsWith(".")) { + leaf = leaf.substring(1); + } + return new Entry(name, canonical(leaf.replaceAll("\\[\\d+\\]", ""))); + } + + /** + * Returns every leaf path a settings record accepts, in the canonical kebab form. + * + *

Nested records are walked because the configuration is nested; a list-valued component is a + * leaf because its indices are stripped before the comparison. + */ + private static Set leafPaths(Class shape) { + Set paths = new LinkedHashSet<>(); + collectLeafPaths(shape, "", paths); + return paths; + } + + private static void collectLeafPaths(Class shape, String prefix, Set paths) { + for (RecordComponent component : shape.getRecordComponents()) { + String path = prefix + canonical(component.getName()); + if (component.getType().isRecord()) { + collectLeafPaths(component.getType(), path + '.', paths); + } else { + paths.add(path); + } + } + } + + /** Normalises a property path to the kebab-case form the settings records declare. */ + private static String canonical(String path) { + StringBuilder canonical = new StringBuilder(path.length() + 4); + for (int index = 0; index < path.length(); index++) { + char character = path.charAt(index); + if (character == '_') { + canonical.append('-'); + } else if (Character.isUpperCase(character)) { + if (index > 0 && path.charAt(index - 1) != '.' && path.charAt(index - 1) != '-') { + canonical.append('-'); + } + canonical.append(Character.toLowerCase(character)); + } else { + canonical.append(character); + } + } + return canonical.toString(); + } + + private List keysUnder(String prefix) { + List keys = new ArrayList<>(); + for (PropertySource source : environment.getPropertySources()) { + if (!(source instanceof EnumerablePropertySource enumerable)) { + continue; + } + for (String key : enumerable.getPropertyNames()) { + if (key.startsWith(prefix)) { + keys.add(key); + } + } + } + return keys; + } +} diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingCoreAutoConfiguration.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingCoreAutoConfiguration.java index 2eed7e3..0caded5 100644 --- a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingCoreAutoConfiguration.java +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingCoreAutoConfiguration.java @@ -15,7 +15,6 @@ import dev.caskeleton.messaging.security.BrokerTlsPolicy; import dev.caskeleton.messaging.security.CredentialProvider; import dev.caskeleton.messaging.security.CredentialRuntimeRegistry; import dev.caskeleton.messaging.security.MessageSecurityValidator; -import dev.caskeleton.messaging.transport.BackpressureController; import dev.caskeleton.messaging.transport.DefaultMessagingRuntimeRegistry; import dev.caskeleton.messaging.transport.GracefulShutdownCoordinator; import dev.caskeleton.messaging.transport.MessagingRuntimeRegistry; @@ -54,11 +53,13 @@ public class MessagingCoreAutoConfiguration { org.springframework.beans.factory.ObjectProvider< dev.caskeleton.messaging.security.BrokerSecurityProfile> profiles, + CompiledMessagingConfiguration compiled, org.springframework.beans.factory.ObjectProvider< dev.caskeleton.messaging.security.CredentialProvider> credentials, org.springframework.core.env.Environment environment) { - return new MessagingCredentialRequirementValidator(profiles, credentials, environment); + return new MessagingCredentialRequirementValidator( + profiles, compiled.security(), credentials, environment); } /** @@ -73,20 +74,63 @@ public class MessagingCoreAutoConfiguration { return new MessagingPrefixMigrationValidator(environment); } + /** + * Compiles {@code app.messaging.destinations}, {@code .brokers} and {@code .security} into the + * platform's own profile objects. + * + *

Separate from the registry bean so that a configuration error fails while compiling, with + * the offending key in the message, rather than later as a validation failure over an object + * whose origin the operator then has to work out. + * + * @param settings the bound configuration + * @return the compiled profiles + */ + @Bean + @ConditionalOnMissingBean + public CompiledMessagingConfiguration compiledMessagingConfiguration(MessagingSettings settings) { + return new MessagingConfigurationCompiler(settings).compile(); + } + + /** + * Refuses a key under the platform's own sections that nothing binds. + * + *

Registered here rather than left to review because a misspelt key inside a bound section is + * silent in a way a misspelt prefix is not: the section binds, the entry appears, and the one + * setting the operator came to change is the one that did not take. + * + * @param environment the resolved environment + * @return the key validator + */ + @Bean + @ConditionalOnMissingBean + public MessagingConfigurationKeyValidator messagingConfigurationKeyValidator( + org.springframework.core.env.ConfigurableEnvironment environment) { + return new MessagingConfigurationKeyValidator(environment); + } + /** * Validates every registered destination profile at startup. * *

Returns a marker rather than a service because its only job is to fail the context when the * registry is inconsistent. * - * @param profiles the registered profiles + *

Configured profiles and bean-declared ones are merged before validation, not after, because + * the rules that matter most across a registry — a dead letter destination that exists, a retry + * graph with no cycle — are questions about the whole set. Validating the two halves separately + * would pass a deployment whose configured destination dead-letters into a bean-declared one that + * points back. + * + * @param profiles the profiles the application declared as beans + * @param compiled the profiles compiled from configuration * @return the validated registry */ @Bean @ConditionalOnMissingBean public ValidatedDestinationRegistry validatedDestinationRegistry( - ObjectProvider profiles) { - List registered = profiles.orderedStream().toList(); + ObjectProvider profiles, CompiledMessagingConfiguration compiled) { + List registered = + java.util.stream.Stream.concat(compiled.destinations().stream(), profiles.orderedStream()) + .toList(); new DestinationProfileValidator().validateAll(registered); return new ValidatedDestinationRegistry(registered); } @@ -148,20 +192,6 @@ public class MessagingCoreAutoConfiguration { return new DefaultMessagingRuntimeRegistry(properties.getShutdown().getDrainDeadline()); } - /** - * Returns the process-wide backpressure controller. - * - * @param properties the bound configuration - * @return the controller - */ - @Bean - @ConditionalOnMissingBean - public BackpressureController backpressureController(MessagingSettings properties) { - return new BackpressureController( - properties.getBackpressure().getGlobalLimit(), - properties.getBackpressure().getPerDestinationLimit()); - } - /** * Returns the graceful shutdown coordinator. * @@ -174,6 +204,45 @@ public class MessagingCoreAutoConfiguration { return new GracefulShutdownCoordinator(properties.getShutdown().getDrainDeadline()); } + /** + * Runs the security validator over every broker security profile, declared or configured. + * + * @param profiles the security profiles the application declared as beans + * @param compiled the security profiles compiled from configuration + * @param validator the security validator + * @return the startup validation + */ + @Bean + public StartupProfileValidation + brokerSecurityStartupValidation( + ObjectProvider profiles, + CompiledMessagingConfiguration compiled, + MessageSecurityValidator validator) { + return new StartupProfileValidation<>( + () -> + java.util.stream.Stream.concat(compiled.security().stream(), profiles.orderedStream()) + .toList(), + validator::validate); + } + + /** + * Drives the shutdown phases the admission gate and the drain coordinator only implemented. + * + * @param admission the publish admission gate + * @param drain the in-flight work coordinator + * @param properties the bound configuration supplying the drain deadline + * @return the lifecycle + */ + @Bean + @ConditionalOnMissingBean + public MessagingShutdownLifecycle messagingShutdownLifecycle( + dev.caskeleton.messaging.policy.MessagingAdmissionController admission, + GracefulShutdownCoordinator drain, + MessagingSettings properties) { + return new MessagingShutdownLifecycle( + admission, drain, properties.getShutdown().getDrainDeadline()); + } + /** * Returns the diagnostic redactor. * @@ -264,14 +333,15 @@ public class MessagingCoreAutoConfiguration { * Returns the read-only actuator endpoint. * * @param registry the validated destination registry - * @param backpressure the in-flight counters + * @param admission the gate every publish passes through * @return the endpoint */ @Bean @ConditionalOnMissingBean public MessagingEndpoint messagingEndpoint( - ValidatedDestinationRegistry registry, BackpressureController backpressure) { - return new MessagingEndpoint(registry, backpressure); + ValidatedDestinationRegistry registry, + dev.caskeleton.messaging.policy.MessagingAdmissionController admission) { + return new MessagingEndpoint(registry, admission); } /** @@ -342,8 +412,8 @@ public class MessagingCoreAutoConfiguration { dev.caskeleton.messaging.schema.json.JacksonMessageCodec.DEFAULT_MAX_BYTES, dev.caskeleton.messaging.schema.json.JacksonMessageCodec.DEFAULT_MAX_BYTES / 2)), new dev.caskeleton.messaging.policy.InFlightLimiter( - properties.getBackpressure().getPerDestinationLimit(), - java.time.Duration.ofSeconds(5))); + properties.getBackpressure().getGlobalLimit(), java.time.Duration.ofSeconds(5)), + properties.getBackpressure().getPerDestinationLimit()); } /** diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingCredentialRequirementValidator.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingCredentialRequirementValidator.java index 19220e1..0273734 100644 --- a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingCredentialRequirementValidator.java +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingCredentialRequirementValidator.java @@ -21,20 +21,38 @@ import org.springframework.core.env.Environment; *

The check is scoped to production profiles. A local profile that connects to a broker with no * authentication is a legitimate development setup; the same profile marked production is not, and * the two must not be satisfiable by the same configuration. + * + *

It reads both the profiles an application declared as beans and the ones compiled from {@code + * app.messaging.security}. A deployment that configures its production broker in YAML and one that + * writes it in Java must meet the same refusal, or the check becomes advice about how to write the + * configuration rather than a rule about what the configuration may say. */ public final class MessagingCredentialRequirementValidator implements InitializingBean { private final ObjectProvider profiles; + private final List configured; + private final ObjectProvider credentials; private final Environment environment; + /** + * Creates the validator. + * + * @param profiles the security profiles the application declared as beans + * @param configured the security profiles compiled from configuration + * @param credentials the application's credential source, if it supplied one + * @param environment the resolved environment, used to report the active profiles + */ public MessagingCredentialRequirementValidator( ObjectProvider profiles, + List configured, ObjectProvider credentials, Environment environment) { this.profiles = Objects.requireNonNull(profiles, "profiles must not be null"); + this.configured = + List.copyOf(Objects.requireNonNull(configured, "configured must not be null")); this.credentials = Objects.requireNonNull(credentials, "credentials must not be null"); this.environment = Objects.requireNonNull(environment, "environment must not be null"); } @@ -54,9 +72,10 @@ public final class MessagingCredentialRequirementValidator implements Initializi return; } List production = - profiles.stream() + java.util.stream.Stream.concat(configured.stream(), profiles.stream()) .filter(BrokerSecurityProfile::production) .map(BrokerSecurityProfile::broker) + .distinct() .toList(); if (production.isEmpty()) { return; diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpoint.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpoint.java index faeba7b..666cb4e 100644 --- a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpoint.java +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpoint.java @@ -1,7 +1,7 @@ package dev.caskeleton.messaging.autoconfigure; import dev.caskeleton.messaging.policy.DestinationProfile; -import dev.caskeleton.messaging.transport.BackpressureController; +import dev.caskeleton.messaging.policy.MessagingAdmissionController; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -26,18 +26,22 @@ import org.springframework.boot.actuate.endpoint.annotation.ReadOperation; public class MessagingEndpoint { private final ValidatedDestinationRegistry registry; - private final BackpressureController backpressure; + private final MessagingAdmissionController admission; /** * Creates the endpoint. * + *

The counts come from the gate every publish passes through. They used to come from a second + * limiter that no publish ever touched, so the endpoint reported zero in flight under any load — + * a diagnostic that is worse than absent, because it looks like an answer. + * * @param registry the validated destination registry - * @param backpressure the in-flight counters + * @param admission the gate every publish passes through */ public MessagingEndpoint( - ValidatedDestinationRegistry registry, BackpressureController backpressure) { + ValidatedDestinationRegistry registry, MessagingAdmissionController admission) { this.registry = Objects.requireNonNull(registry, "registry must not be null"); - this.backpressure = Objects.requireNonNull(backpressure, "backpressure must not be null"); + this.admission = Objects.requireNonNull(admission, "admission must not be null"); } /** @@ -48,7 +52,7 @@ public class MessagingEndpoint { @ReadOperation public Map messaging() { Map report = new LinkedHashMap<>(); - report.put("globalInFlight", backpressure.globalInFlight()); + report.put("globalInFlight", admission.inFlight()); report.put("destinations", destinations()); return Map.copyOf(report); } @@ -68,7 +72,7 @@ public class MessagingEndpoint { entry.put("production", profile.production()); entry.put("deadLetterEnabled", profile.deadLetter().enabled()); entry.put("retryMode", profile.retry().mode().name()); - entry.put("inFlight", backpressure.inFlight(profile.name().value())); + entry.put("inFlight", admission.inFlight(profile.name().value())); return Map.copyOf(entry); } } diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingOutboxRelayLifecycle.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingOutboxRelayLifecycle.java new file mode 100644 index 0000000..4d20c47 --- /dev/null +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingOutboxRelayLifecycle.java @@ -0,0 +1,69 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.outbox.OutboxRelayWorker; +import java.time.Duration; +import java.util.Objects; +import org.springframework.context.SmartLifecycle; + +/** + * Starts and stops the outbox relay with the application context. + * + *

Without this the relay is a bean an application has to remember to drive itself, and the + * failure when it forgets is silent: rows accumulate in a table that looks healthy while no message + * is ever published. + * + *

Stopping drains rather than interrupts. A pass killed between claiming a row and writing its + * outcome leaves that row {@code IN_FLIGHT} under a lease no other relay may touch until it + * expires, so an orderly shutdown would strand exactly what a crash strands. + * + *

Every replica runs its own relay. Claims are taken with {@code FOR UPDATE SKIP LOCKED} under + * an owner and a fencing token, so two relays never publish the same row; leader election would buy + * nothing here except a single point of failure for the whole outbox. + */ +public final class MessagingOutboxRelayLifecycle implements SmartLifecycle { + + private final OutboxRelayWorker worker; + + private final Duration drainDeadline; + + /** + * Creates the lifecycle. + * + * @param worker the relay worker to drive + * @param drainDeadline how long a running pass may take to finish on stop + */ + public MessagingOutboxRelayLifecycle(OutboxRelayWorker worker, Duration drainDeadline) { + this.worker = Objects.requireNonNull(worker, "worker must not be null"); + this.drainDeadline = Objects.requireNonNull(drainDeadline, "drainDeadline must not be null"); + } + + @Override + public void start() { + worker.start(); + } + + @Override + public void stop() { + worker.stop(drainDeadline); + } + + @Override + public boolean isRunning() { + return worker.isRunning(); + } + + /** + * Stops before the admission gate closes. + * + *

{@code SmartLifecycle} stops in descending phase order, so the highest phase stops first. + * The relay has to be the one that stops first: closing admission while a pass is mid-publish + * turns that pass's rows ambiguous — a shutdown that manufactures exactly the uncertainty the + * drain exists to avoid. + * + * @return the lifecycle phase + */ + @Override + public int getPhase() { + return Integer.MAX_VALUE; + } +} diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingProviderSelection.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingProviderSelection.java index e4a39ef..c93129d 100644 --- a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingProviderSelection.java +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingProviderSelection.java @@ -48,11 +48,42 @@ public class MessagingProviderSelection { "kafka", KafkaMessagingAutoConfiguration.class, "rabbit", RabbitMessagingAutoConfiguration.class); + /** + * Broker ids that ship without a {@code MessagingTransport}, and why. + * + *

Registered is not the same as assemblable. The Rabbit leaf ships its profile validator, + * failure classifier and security configurer, and no transport — the native channel bridge those + * would sit on has only test implementations. Selecting it therefore produced an unsatisfied + * dependency on {@code MessagingTransport} deep inside the core configuration, which tells an + * operator that a bean is missing rather than that the transport they chose is not finished. + * + *

Refused here instead, next to the other selection errors, so the message names the state the + * deployment is actually in. An entry leaves this map on the day its transport does exist. + */ + static final Map BROKERS_WITHOUT_A_TRANSPORT = + Map.of( + "rabbit", + "the Rabbit adapter ships its validators and security configuration but no " + + "MessagingTransport: its native channel publisher is not implemented, so a publish " + + "has nothing to travel on"); + /** The registered broker ids, so a message can tell an operator what is available. */ public static List registeredBrokerIds() { return REGISTERED_BROKERS.keySet().stream().sorted().toList(); } + /** + * The registered broker ids that actually assemble a transport. + * + * @return the ids a deployment can select today + */ + public static List assemblableBrokerIds() { + return REGISTERED_BROKERS.keySet().stream() + .filter(id -> !BROKERS_WITHOUT_A_TRANSPORT.containsKey(id)) + .sorted() + .toList(); + } + /** * Resolves the selection, or throws naming the property and the values it accepts. * @@ -87,6 +118,17 @@ public class MessagingProviderSelection { + clientClass + " on the classpath; the deployment selected a transport it did not ship"); } + String unfinished = BROKERS_WITHOUT_A_TRANSPORT.get(broker); + if (unfinished != null) { + throw new IllegalStateException( + MessagingSettings.PREFIX + + ".broker=" + + broker + + " cannot be selected: " + + unfinished + + ". Transports that can be assembled today: " + + assemblableBrokerIds()); + } return broker; } diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingReliabilityAutoConfiguration.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingReliabilityAutoConfiguration.java index 9a9e9ca..72350cc 100644 --- a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingReliabilityAutoConfiguration.java +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingReliabilityAutoConfiguration.java @@ -1,11 +1,15 @@ package dev.caskeleton.messaging.autoconfigure; +import dev.caskeleton.messaging.api.publish.MessagePublisher; import dev.caskeleton.messaging.inbox.IdempotentConsumer; import dev.caskeleton.messaging.inbox.InboxCleanupJob; import dev.caskeleton.messaging.inbox.InboxRetentionPolicy; import dev.caskeleton.messaging.inbox.TransactionalInboxHandler; import dev.caskeleton.messaging.outbox.OutboxCleanupJob; +import dev.caskeleton.messaging.outbox.OutboxEnvelopeFactory; import dev.caskeleton.messaging.outbox.OutboxProperties; +import dev.caskeleton.messaging.outbox.OutboxRelay; +import dev.caskeleton.messaging.outbox.OutboxRelayWorker; import dev.caskeleton.messaging.outbox.OutboxRetryScheduler; import dev.caskeleton.messaging.reliability.InboxRepository; import dev.caskeleton.messaging.reliability.OutboxRepository; @@ -28,6 +32,10 @@ import org.springframework.context.annotation.Configuration; *

The cleanup jobs are beans but no scheduler is registered for them. Scheduling is the * application's decision: a service running several replicas usually wants one of them to run * cleanup, and auto-registering a fixed-rate task would have every replica delete the same rows. + * + *

The relay is the opposite case and is driven here. Its claims are fenced by owner and token + * under {@code SKIP LOCKED}, so every replica running one is safe, while nobody running one is a + * table that fills up behind a business transaction that reported success. */ @Configuration(proxyBeanMethods = false) public class MessagingReliabilityAutoConfiguration { @@ -55,6 +63,70 @@ public class MessagingReliabilityAutoConfiguration { return new OutboxRetryScheduler(properties, Duration.ofMinutes(1)); } + /** + * Returns the relay, when the application supplied the two things only it can supply. + * + *

The repository writes inside the application's transaction and the envelope factory carries + * the application's producer identity; the platform can invent neither, and a default for either + * would publish from the wrong place under the wrong name. + * + * @param outbox the application's outbox repository + * @param publisher the platform publisher + * @param envelopes rebuilds envelopes from rows + * @param properties the relay settings + * @param scheduler the retry policy + * @return the relay + */ + @Bean + @ConditionalOnBean({OutboxRepository.class, OutboxEnvelopeFactory.class}) + @ConditionalOnMissingBean + public OutboxRelay outboxRelay( + OutboxRepository outbox, + MessagePublisher publisher, + OutboxEnvelopeFactory envelopes, + OutboxProperties properties, + OutboxRetryScheduler scheduler) { + return new OutboxRelay( + outbox, + publisher, + envelopes, + properties.batchSize(), + properties.leaseDuration(), + scheduler); + } + + /** + * Returns the worker that runs relay passes. + * + * @param relay the relay to drive + * @param scheduler the policy deciding when the next pass runs + * @return the worker + */ + @Bean + @ConditionalOnBean(OutboxRelay.class) + @ConditionalOnMissingBean + public OutboxRelayWorker outboxRelayWorker(OutboxRelay relay, OutboxRetryScheduler scheduler) { + return new OutboxRelayWorker(relay, scheduler); + } + + /** + * Binds the relay worker to the context lifecycle. + * + *

The drain budget is the lease duration: a pass holds its claims for at most that long, so + * waiting longer proves nothing that waiting that long has not already proved. + * + * @param worker the relay worker + * @param properties the relay settings supplying the lease duration + * @return the lifecycle + */ + @Bean + @ConditionalOnBean(OutboxRelayWorker.class) + @ConditionalOnMissingBean + public MessagingOutboxRelayLifecycle outboxRelayLifecycle( + OutboxRelayWorker worker, OutboxProperties properties) { + return new MessagingOutboxRelayLifecycle(worker, properties.leaseDuration()); + } + /** * Returns the outbox cleanup job. * diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingSettings.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingSettings.java index 9d22ca8..92fef98 100644 --- a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingSettings.java +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingSettings.java @@ -1,6 +1,8 @@ package dev.caskeleton.messaging.autoconfigure; import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; import org.springframework.boot.context.properties.ConfigurationProperties; /** @@ -22,6 +24,17 @@ import org.springframework.boot.context.properties.ConfigurationProperties; * is still being proven must not become load-bearing because a default turned it on, and the same * applies to the Spring Cloud Stream bridge, which bypasses the platform's retry and dead letter * policy by design. + * + *

The destination, broker and security maps are what the configuration reference has always + * described and nothing bound. Until they existed, a deployment that configured them exactly as + * documented changed no behaviour and produced no error: the profiles the platform actually used + * had to arrive as beans an application wrote in Java, and the documented keys were inert text. + * {@link MessagingConfigurationCompiler} turns them into those same profiles, so the documented + * shape is now the working shape. + * + *

The maps are ordered. Configuration errors name the entry that caused them, and an operator + * reading a startup failure against a file they wrote should meet the entries in the order they + * wrote them rather than in hash order. */ @ConfigurationProperties(prefix = MessagingSettings.PREFIX) public class MessagingSettings { @@ -39,6 +52,63 @@ public class MessagingSettings { private final Bridge bridge = new Bridge(); private final Backpressure backpressure = new Backpressure(); private final Shutdown shutdown = new Shutdown(); + private Map destinations = new LinkedHashMap<>(); + private Map brokers = new LinkedHashMap<>(); + private Map security = new LinkedHashMap<>(); + + /** + * Returns the declared destinations, keyed by logical name. + * + * @return the destination settings + */ + public Map getDestinations() { + return destinations; + } + + /** + * Sets the declared destinations. + * + * @param destinations the destination settings, keyed by logical name + */ + public void setDestinations(Map destinations) { + this.destinations = new LinkedHashMap<>(destinations); + } + + /** + * Returns the declared brokers, keyed by logical name. + * + * @return the broker settings + */ + public Map getBrokers() { + return brokers; + } + + /** + * Sets the declared brokers. + * + * @param brokers the broker settings, keyed by logical name + */ + public void setBrokers(Map brokers) { + this.brokers = new LinkedHashMap<>(brokers); + } + + /** + * Returns the declared broker security profiles, keyed by broker name. + * + * @return the security settings + */ + public Map getSecurity() { + return security; + } + + /** + * Sets the declared broker security profiles. + * + * @param security the security settings, keyed by broker name + */ + public void setSecurity(Map security) { + this.security = new LinkedHashMap<>(security); + } /** * Returns the experimental adapter switches. diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingShutdownLifecycle.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingShutdownLifecycle.java new file mode 100644 index 0000000..0229f13 --- /dev/null +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingShutdownLifecycle.java @@ -0,0 +1,124 @@ +package dev.caskeleton.messaging.autoconfigure; + +import dev.caskeleton.messaging.policy.MessagingAdmissionController; +import dev.caskeleton.messaging.transport.GracefulShutdownCoordinator; +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; +import org.springframework.context.SmartLifecycle; + +/** + * Runs the platform's shutdown phases in order when the context stops. + * + *

The admission controller and the drain coordinator were beans nobody drove. Both could refuse + * new work and wait for in-flight work, and neither was ever asked to: a context shutdown went + * straight to closing beans while publishes were still in flight and handlers were still between + * their side effect and their settlement, which is the shape that turns a deploy into duplicated + * effects and unexplained ambiguous publishes. + * + *

The order is admission first, drain second. Reversed, the drain waits for a count that new + * work keeps topping up, so it either never finishes or finishes only because the deadline passed — + * and work abandoned at the deadline is redelivered rather than completed. + * + *

The wait is bounded by the same drain deadline the runtime registry uses. A handler that never + * returns must not hold the process open; what is still running when the deadline passes is left + * unsettled on purpose, so the broker redelivers it instead of the platform recording a completion + * it cannot prove. + */ +public final class MessagingShutdownLifecycle implements SmartLifecycle { + + /** How often the drain is re-checked. Short enough that a fast drain is not padded. */ + private static final Duration POLL = Duration.ofMillis(50); + + private final MessagingAdmissionController admission; + + private final GracefulShutdownCoordinator drain; + + private final Duration drainDeadline; + + private final Supplier clock; + + private final AtomicBoolean running = new AtomicBoolean(); + + /** + * Creates the lifecycle. + * + * @param admission the publish admission gate + * @param drain the in-flight work coordinator + * @param drainDeadline how long in-flight work may take + */ + public MessagingShutdownLifecycle( + MessagingAdmissionController admission, + GracefulShutdownCoordinator drain, + Duration drainDeadline) { + this(admission, drain, drainDeadline, Instant::now); + } + + /** + * Creates the lifecycle with an explicit clock, so the deadline is testable without sleeping. + * + * @param admission the publish admission gate + * @param drain the in-flight work coordinator + * @param drainDeadline how long in-flight work may take + * @param clock the source of the current instant + */ + public MessagingShutdownLifecycle( + MessagingAdmissionController admission, + GracefulShutdownCoordinator drain, + Duration drainDeadline, + Supplier clock) { + this.admission = Objects.requireNonNull(admission, "admission must not be null"); + this.drain = Objects.requireNonNull(drain, "drain must not be null"); + this.drainDeadline = Objects.requireNonNull(drainDeadline, "drainDeadline must not be null"); + this.clock = Objects.requireNonNull(clock, "clock must not be null"); + } + + @Override + public void start() { + running.set(true); + } + + @Override + public void stop() { + if (!running.compareAndSet(true, false)) { + return; + } + admission.stopAcceptingNewWork(); + Instant startedAt = clock.get(); + drain.beginDrain(startedAt); + Instant deadline = startedAt.plus(drainDeadline); + while (!drain.isDrained(clock.get()) && clock.get().isBefore(deadline)) { + try { + Thread.sleep(POLL.toMillis()); + } catch (InterruptedException interrupted) { + // The interrupt is the signal to stop waiting, not a reason to lose it: the phases below + // still have to run, and a swallowed interrupt leaves the shutdown thread looking healthy. + Thread.currentThread().interrupt(); + return; + } + } + } + + @Override + public boolean isRunning() { + return running.get(); + } + + /** + * Stops after the producers of work and before the beans that own broker connections. + * + *

{@code SmartLifecycle} stops in descending phase order. A phase above the default puts this + * drain ahead of the transports it is draining, because closing a connection first would strand + * the settlements the drain exists to flush. It sits below the outbox relay's phase for the + * mirror-image reason: refusing admission while a relay pass is mid-publish turns that pass's + * rows ambiguous, which is the uncertainty the drain exists to avoid. + * + * @return the lifecycle phase + */ + @Override + public int getPhase() { + return Integer.MAX_VALUE - 1024; + } +} diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/RabbitMessagingAutoConfiguration.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/RabbitMessagingAutoConfiguration.java index 81ce786..66b14dd 100644 --- a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/RabbitMessagingAutoConfiguration.java +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/RabbitMessagingAutoConfiguration.java @@ -36,6 +36,30 @@ public class RabbitMessagingAutoConfiguration { return new RabbitProfileValidator(); } + /** + * Runs the RabbitMQ profile validator over every broker profile, declared or configured. + * + * @param profiles the broker profiles the application declared as beans + * @param compiled the broker profiles compiled from {@code app.messaging.brokers} + * @param validator the RabbitMQ profile validator + * @return the startup validation + */ + @Bean + public StartupProfileValidation + rabbitProfileStartupValidation( + org.springframework.beans.factory.ObjectProvider< + dev.caskeleton.messaging.rabbit.RabbitBrokerProfile> + profiles, + CompiledMessagingConfiguration compiled, + RabbitProfileValidator validator) { + return new StartupProfileValidation<>( + () -> + java.util.stream.Stream.concat( + compiled.rabbitBrokers().stream(), profiles.orderedStream()) + .toList(), + validator::validate); + } + /** * Returns the publish failure classifier. * diff --git a/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/StartupProfileValidation.java b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/StartupProfileValidation.java new file mode 100644 index 0000000..47dc4e7 --- /dev/null +++ b/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/StartupProfileValidation.java @@ -0,0 +1,46 @@ +package dev.caskeleton.messaging.autoconfigure; + +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.function.Supplier; +import org.springframework.beans.factory.InitializingBean; + +/** + * Applies one profile validator to every profile of its kind at startup. + * + *

The Kafka, RabbitMQ and security validators were all beans and none of them was injected + * anywhere: the context published a validator per broker and validated nothing. A profile that + * promises a guarantee its broker cannot give — an exactly-once claim on a non-transactional + * producer, a quorum ack on a single replica, a plaintext credential on a production listener — + * then boots cleanly and fails on the first message that depends on it, which is the wrong place to + * find out. + * + *

Validation runs as {@code afterPropertiesSet} rather than on an application event, so the + * failure happens while the context is still being built and the profile bean that caused it is + * named in the stack. + * + *

The profiles arrive through a supplier rather than an {@code ObjectProvider} because there are + * now two sources of them — beans an application declares and profiles compiled from {@code + * app.messaging} — and a validator that saw only one of the two would leave the other half of a + * deployment's configuration unchecked. Which half went unchecked would depend on how the + * deployment happened to be written, which is the worst possible rule. + * + * @param the profile type this instance validates + */ +final class StartupProfileValidation implements InitializingBean { + + private final Supplier> profiles; + + private final Consumer validator; + + StartupProfileValidation(Supplier> profiles, Consumer validator) { + this.profiles = Objects.requireNonNull(profiles, "profiles must not be null"); + this.validator = Objects.requireNonNull(validator, "validator must not be null"); + } + + @Override + public void afterPropertiesSet() { + profiles.get().forEach(validator); + } +} diff --git a/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/BlockingFacadeTest.java b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/BlockingFacadeTest.java index 52b399b..0b60b6e 100644 --- a/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/BlockingFacadeTest.java +++ b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/BlockingFacadeTest.java @@ -76,8 +76,7 @@ class BlockingFacadeTest { Duration.ofMillis(50), dev.caskeleton.messaging.api.destination.ConfirmationRequirement .REPLICATION_OR_PERSISTENCE_ACK, - Optional.empty(), - java.util.Map.of()); + Optional.empty()); assertThatThrownBy( () -> diff --git a/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingAutoConfigurationTest.java b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingAutoConfigurationTest.java index 206eb26..dcb22b1 100644 --- a/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingAutoConfigurationTest.java +++ b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingAutoConfigurationTest.java @@ -19,13 +19,13 @@ import dev.caskeleton.messaging.policy.CapabilityTier; import dev.caskeleton.messaging.policy.ConsumerPolicy; import dev.caskeleton.messaging.policy.DeadLetterPolicy; import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.MessagingAdmissionController; import dev.caskeleton.messaging.policy.PayloadPolicy; import dev.caskeleton.messaging.policy.PhysicalDestination; import dev.caskeleton.messaging.policy.ProducerPolicy; import dev.caskeleton.messaging.policy.RetryPolicy; import dev.caskeleton.messaging.policy.SchemaPolicy; import dev.caskeleton.messaging.schema.SchemaCompatibility; -import dev.caskeleton.messaging.transport.BackpressureController; import dev.caskeleton.messaging.transport.GracefulShutdownCoordinator; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -51,7 +51,7 @@ class MessagingAutoConfigurationTest { .hasSingleBean(ValidatedDestinationRegistry.class) .hasSingleBean(BlockingMessagePublisher.class) .hasSingleBean(ReactiveMessagePublisher.class) - .hasSingleBean(BackpressureController.class) + .hasSingleBean(MessagingAdmissionController.class) .hasSingleBean(GracefulShutdownCoordinator.class)); } @@ -236,6 +236,25 @@ class MessagingAutoConfigurationTest { false); } + @org.junit.jupiter.api.Test + @org.junit.jupiter.api.DisplayName("a broker profile the adapter cannot honour fails startup") + void anUnhonourableBrokerProfileFailsStartup() { + new org.springframework.boot.test.context.runner.ApplicationContextRunner() + .withConfiguration( + org.springframework.boot.autoconfigure.AutoConfigurations.of( + MessagingCoreAutoConfiguration.class, KafkaMessagingAutoConfiguration.class)) + .withUserConfiguration(PublisherConfiguration.class, AutoCommittingKafkaProfile.class) + .withPropertyValues("spring.kafka.bootstrap-servers=localhost:9092") + .run( + context -> + org.assertj.core.api.Assertions.assertThat(context) + .getFailure() + .as( + "a validator nothing injects lets a profile boot and fail on the first" + + " message that depends on the guarantee") + .hasMessageContaining("consumer auto commit is forbidden")); + } + @org.junit.jupiter.api.Test @org.junit.jupiter.api.DisplayName("a superseded prefix fails startup instead of binding nothing") void aSupersededPrefixFailsStartup() { @@ -364,6 +383,7 @@ class MessagingAutoConfigurationTest { java.util.Optional.empty(), dev.caskeleton.messaging.security.DestinationAccessPolicy .denyAll())), + java.util.List.of(), new StaticProvider(), new org.springframework.core.env.StandardEnvironment()) .validate()) @@ -394,6 +414,7 @@ class MessagingAutoConfigurationTest { java.util.Optional.empty(), dev.caskeleton.messaging.security.DestinationAccessPolicy .denyAll())), + java.util.List.of(), new StaticProvider(), new org.springframework.core.env.StandardEnvironment()) .validate()) @@ -491,3 +512,35 @@ class MessagingAutoConfigurationTest { } } } + +/** A Kafka profile that acknowledges on a timer rather than after the handler. */ +@Configuration(proxyBeanMethods = false) +class AutoCommittingKafkaProfile { + + @Bean + dev.caskeleton.messaging.kafka.KafkaBrokerProfile kafkaBrokerProfile() { + return KafkaProfiles.profile(true); + } +} + +/** Builds a Kafka broker profile whose consumer acknowledges on a timer. */ +final class KafkaProfiles { + + private KafkaProfiles() {} + + static dev.caskeleton.messaging.kafka.KafkaBrokerProfile profile(boolean enableAutoCommit) { + return new dev.caskeleton.messaging.kafka.KafkaBrokerProfile( + "kafka-primary", + true, + false, + java.util.List.of("localhost:9092"), + true, + "all", + 5, + java.time.Duration.ofSeconds(30), + enableAutoCommit, + "orders", + false, + false); + } +} diff --git a/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingConfigurationBindingTest.java b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingConfigurationBindingTest.java new file mode 100644 index 0000000..bea9b56 --- /dev/null +++ b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingConfigurationBindingTest.java @@ -0,0 +1,320 @@ +package dev.caskeleton.messaging.autoconfigure; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.delivery.OrderingScope; +import dev.caskeleton.messaging.api.destination.DestinationKind; +import dev.caskeleton.messaging.api.destination.DestinationName; +import dev.caskeleton.messaging.policy.DestinationProfile; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.env.YamlPropertySourceLoader; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.core.env.PropertySource; +import org.springframework.core.io.ByteArrayResource; + +/** + * Makes the configuration reference an executable statement rather than a description. + * + *

The document described destination, broker and security sections for a platform whose binder + * held four flags. Every one of those keys was inert: a deployment that configured exactly what the + * page said started cleanly, changed nothing, and reported nothing — the outcome that is worse than + * both "it works" and "it refuses to start", because there is no moment at which anybody learns. + * + *

So the first test below reads the YAML out of the shipped page and boots a context on it. An + * edit to the document that the binder cannot honour fails here, and so does a change to the binder + * that leaves the document describing a shape that no longer exists. + * + *

The rest are the refusals. Each one is a configuration an operator can write and each fails + * startup with the key in the message, because a platform that silently ignores the line somebody + * came to change is the defect this whole suite is about. + */ +class MessagingConfigurationBindingTest { + + private final ApplicationContextRunner runner = + new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(MessagingCoreAutoConfiguration.class)) + .withUserConfiguration(MessagingAutoConfigurationTest.PublisherConfiguration.class); + + @Test + void theDocumentedConfigurationBindsToTheProfilesThePlatformUses() { + documented() + .run( + context -> { + assertThat(context).hasNotFailed(); + + DestinationProfile orders = + context + .getBean(ValidatedDestinationRegistry.class) + .require(new DestinationName("order-events")); + assertThat(orders.broker()).isEqualTo("kafka-primary"); + assertThat(orders.kind()).isEqualTo(DestinationKind.EVENT_STREAM); + assertThat(orders.physical().topic()).contains("order.events.v1"); + assertThat(orders.orderingScope()).isEqualTo(OrderingScope.KEY); + assertThat(orders.retry().maxAttempts()).isEqualTo(3); + assertThat(orders.deadLetter().destination()) + .contains(new DestinationName("order-events-dlq")); + assertThat(orders.schema().messageTypes()) + .extracting(dev.caskeleton.messaging.api.MessageType::value) + .containsExactly("order.created"); + + CompiledMessagingConfiguration compiled = + context.getBean(CompiledMessagingConfiguration.class); + assertThat(compiled.kafkaBrokers()) + .singleElement() + .satisfies( + broker -> { + assertThat(broker.broker()).isEqualTo("kafka-primary"); + assertThat(broker.bootstrapServers()) + .containsExactly("broker-1:9093", "broker-2:9093"); + assertThat(broker.enableAutoCommit()).isFalse(); + }); + assertThat(compiled.rabbitBrokers()) + .singleElement() + .satisfies(broker -> assertThat(broker.quorumQueues()).isTrue()); + assertThat(compiled.security()) + .singleElement() + .satisfies( + security -> { + assertThat(security.broker()).isEqualTo("kafka-primary"); + assertThat(security.hostnameVerification()).isTrue(); + assertThat(security.admin()) + .as("an application runtime holds no admin credential") + .isEmpty(); + assertThat( + security.access().mayPublish(new DestinationName("order-events"))) + .isTrue(); + }); + }); + } + + @Test + void aTypoInsideABoundSectionFailsStartupNamingTheKey() { + runner + .withPropertyValues(minimal()) + .withPropertyValues("app.messaging.destinations.order-events.consumer.prefech=64") + .run( + context -> + assertThat(context) + .hasFailed() + .getFailure() + .hasMessageContaining( + "app.messaging.destinations.order-events.consumer.prefech") + .hasMessageContaining("prefetch")); + } + + @Test + void aDestinationNamingAnUnconfiguredBrokerFailsNamingTheKey() { + runner + .withPropertyValues(minimal()) + .withPropertyValues("app.messaging.destinations.order-events.broker=kafka-secondary") + .run( + context -> + assertThat(context) + .hasFailed() + .getFailure() + .hasMessageContaining("app.messaging.destinations.order-events.broker") + .hasMessageContaining("kafka-secondary")); + } + + @Test + void aDanglingDeadLetterDestinationFailsStartup() { + runner + .withPropertyValues(minimal()) + .withPropertyValues( + "app.messaging.destinations.order-events.dlq.destination=nobody-declared-this") + .run( + context -> + assertThat(context) + .hasFailed() + .getFailure() + .hasMessageContaining("nobody-declared-this")); + } + + @Test + void aRetryAndDeadLetterCycleAcrossTwoDestinationsIsRefusedWithItsPath() { + runner + .withPropertyValues(minimal()) + .withPropertyValues( + "app.messaging.destinations.order-events.retry.mode=RETRY_DESTINATION", + "app.messaging.destinations.order-events.retry.max-attempts=3", + "app.messaging.destinations.order-events.retry.ordering-impact=ALLOW_REORDER", + "app.messaging.destinations.order-events.retry.destination=order-retry", + "app.messaging.destinations.order-events.dlq.destination=order-retry", + "app.messaging.destinations.order-retry.broker=kafka-primary", + "app.messaging.destinations.order-retry.kind=WORK_QUEUE", + "app.messaging.destinations.order-retry.physical.topic=order.retry.v1", + "app.messaging.destinations.order-retry.schema.message-types[0]=order.created", + "app.messaging.destinations.order-retry.dlq.destination=order-events") + .run( + context -> + assertThat(context) + .hasFailed() + .getFailure() + .hasMessageContaining("cycle") + .hasMessageContaining("order-events") + .hasMessageContaining("order-retry")); + } + + @Test + void aRabbitSettingOnAKafkaBrokerFailsNamingTheKey() { + runner + .withPropertyValues(minimal()) + .withPropertyValues("app.messaging.brokers.kafka-primary.prefetch=32") + .run( + context -> + assertThat(context) + .hasFailed() + .getFailure() + .hasMessageContaining("app.messaging.brokers.kafka-primary.prefetch") + .hasMessageContaining("RabbitMQ setting on a Kafka broker")); + } + + @Test + void aSecurityEntryForAnUndeclaredBrokerFailsNamingTheKey() { + runner + .withPropertyValues(minimal()) + .withPropertyValues( + "app.messaging.security.kafka-secondary.producer.type=SASL_SCRAM", + "app.messaging.security.kafka-secondary.producer.credential-id=p", + "app.messaging.security.kafka-secondary.consumer.type=SASL_SCRAM", + "app.messaging.security.kafka-secondary.consumer.credential-id=c") + .run( + context -> + assertThat(context) + .hasFailed() + .getFailure() + .hasMessageContaining("app.messaging.security.kafka-secondary") + .hasMessageContaining("is not a configured broker")); + } + + @Test + void producerAndConsumerSharingOneCredentialFailsStartup() { + runner + .withPropertyValues(minimal()) + .withPropertyValues( + "app.messaging.security.kafka-primary.producer.type=SASL_SCRAM", + "app.messaging.security.kafka-primary.producer.credential-id=one-account", + "app.messaging.security.kafka-primary.consumer.type=SASL_SCRAM", + "app.messaging.security.kafka-primary.consumer.credential-id=one-account") + .run( + context -> + assertThat(context) + .hasFailed() + .getFailure() + .as("the validator was a bean nothing injected; bound values now reach it") + .hasMessageContaining("separate credentials")); + } + + @Test + void anUnknownAuthenticationMechanismFailsNamingTheKey() { + runner + .withPropertyValues(minimal()) + .withPropertyValues( + "app.messaging.security.kafka-primary.producer.type=TRUST_ME", + "app.messaging.security.kafka-primary.producer.credential-id=p", + "app.messaging.security.kafka-primary.consumer.type=SASL_SCRAM", + "app.messaging.security.kafka-primary.consumer.credential-id=c") + .run( + context -> + assertThat(context) + .hasFailed() + .getFailure() + .hasMessageContaining("app.messaging.security.kafka-primary.producer.type") + .hasMessageContaining("TRUST_ME")); + } + + @Test + void aProductionKafkaBrokerWithoutTransportSecurityFailsStartup() { + runner + .withConfiguration(AutoConfigurations.of(KafkaMessagingAutoConfiguration.class)) + .withPropertyValues(minimal()) + .withPropertyValues( + "spring.kafka.bootstrap-servers=localhost:9092", + "app.messaging.brokers.kafka-primary.production=true") + .run( + context -> + assertThat(context) + .hasFailed() + .getFailure() + .as("the Kafka validator existed as a bean and validated nothing") + .hasMessageContaining("production Kafka connection requires TLS")); + } + + /** + * The smallest configuration that boots, which every refusal above then breaks in one way. + * + *

Written as properties rather than reused from the document because a refusal test has to + * change exactly one thing; sharing the documented fixture would make each failure depend on + * everything else the page happens to say. + */ + private static String[] minimal() { + return new String[] { + "app.messaging.brokers.kafka-primary.type=kafka", + "app.messaging.brokers.kafka-primary.bootstrap-servers[0]=broker-1:9093", + "app.messaging.destinations.order-events.broker=kafka-primary", + "app.messaging.destinations.order-events.kind=EVENT_STREAM", + "app.messaging.destinations.order-events.physical.topic=order.events.v1", + "app.messaging.destinations.order-events.schema.message-types[0]=order.created", + }; + } + + private ApplicationContextRunner documented() { + List> sources = new ArrayList<>(); + YamlPropertySourceLoader loader = new YamlPropertySourceLoader(); + List blocks = yamlBlocks(reference()); + assertThat(blocks) + .as("the configuration reference has stopped carrying YAML examples") + .isNotEmpty(); + for (int index = 0; index < blocks.size(); index++) { + try { + sources.addAll( + loader.load( + "configuration-reference-" + index, + new ByteArrayResource(blocks.get(index).getBytes(StandardCharsets.UTF_8)))); + } catch (IOException failure) { + throw new UncheckedIOException(failure); + } + } + return runner.withInitializer( + context -> sources.forEach(context.getEnvironment().getPropertySources()::addLast)); + } + + private static List yamlBlocks(String document) { + List blocks = new ArrayList<>(); + StringBuilder current = null; + for (String line : document.lines().toList()) { + if (line.startsWith("```yaml")) { + current = new StringBuilder(); + } else if (line.startsWith("```") && current != null) { + blocks.add(current.toString()); + current = null; + } else if (current != null) { + current.append(line).append('\n'); + } + } + return blocks; + } + + private static String reference() { + Path candidate = Path.of("").toAbsolutePath(); + while (candidate != null && !Files.isDirectory(candidate.resolve("docs/messaging"))) { + candidate = candidate.getParent(); + } + if (candidate == null) { + throw new IllegalStateException("could not locate docs/messaging from the test cwd"); + } + try { + return Files.readString(candidate.resolve("docs/messaging/configuration-reference.md")); + } catch (IOException failure) { + throw new UncheckedIOException(failure); + } + } +} diff --git a/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpointTest.java b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpointTest.java index 58862dd..d69b9f2 100644 --- a/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpointTest.java +++ b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingEndpointTest.java @@ -13,13 +13,15 @@ import dev.caskeleton.messaging.policy.CapabilityTier; import dev.caskeleton.messaging.policy.ConsumerPolicy; import dev.caskeleton.messaging.policy.DeadLetterPolicy; import dev.caskeleton.messaging.policy.DestinationProfile; +import dev.caskeleton.messaging.policy.InFlightLimiter; +import dev.caskeleton.messaging.policy.MessagingAdmissionController; +import dev.caskeleton.messaging.policy.PayloadLimitGuard; import dev.caskeleton.messaging.policy.PayloadPolicy; import dev.caskeleton.messaging.policy.PhysicalDestination; import dev.caskeleton.messaging.policy.ProducerPolicy; import dev.caskeleton.messaging.policy.RetryPolicy; import dev.caskeleton.messaging.policy.SchemaPolicy; import dev.caskeleton.messaging.schema.SchemaCompatibility; -import dev.caskeleton.messaging.transport.BackpressureController; import java.util.List; import java.util.Map; import java.util.Set; @@ -51,9 +53,16 @@ class MessagingEndpointTest { false); } - private static MessagingEndpoint endpoint(BackpressureController backpressure) { - return new MessagingEndpoint( - new ValidatedDestinationRegistry(List.of(profile())), backpressure); + private static MessagingEndpoint endpoint(MessagingAdmissionController admission) { + return new MessagingEndpoint(new ValidatedDestinationRegistry(List.of(profile())), admission); + } + + /** The real gate, so the endpoint reports what a publish actually occupies. */ + private static MessagingAdmissionController admission(int global, int perDestination) { + return new MessagingAdmissionController( + new PayloadLimitGuard(PayloadPolicy.defaults()), + new InFlightLimiter(global, java.time.Duration.ZERO), + perDestination); } @SuppressWarnings("unchecked") @@ -63,7 +72,7 @@ class MessagingEndpointTest { @Test void theReportNamesEveryRegisteredDestination() { - Map report = endpoint(new BackpressureController(100, 10)).messaging(); + Map report = endpoint(admission(100, 10)).messaging(); assertThat(destinations(report)) .singleElement() @@ -72,8 +81,7 @@ class MessagingEndpointTest { @Test void theReportCarriesTheDeclaredGuaranteesAnOperatorNeeds() { - Map entry = - destinations(endpoint(new BackpressureController(100, 10)).messaging()).get(0); + Map entry = destinations(endpoint(admission(100, 10)).messaging()).get(0); assertThat(entry) .containsEntry("deliveryGuarantee", "AT_LEAST_ONCE") @@ -84,11 +92,11 @@ class MessagingEndpointTest { @Test void theReportShowsCurrentInFlightWork() { - BackpressureController backpressure = new BackpressureController(100, 10); - backpressure.tryAcquire("orders.v1"); - backpressure.tryAcquire("orders.v1"); + MessagingAdmissionController admission = admission(100, 10); + admission.admit("orders.v1", 128); + admission.admit("orders.v1", 128); - Map report = endpoint(backpressure).messaging(); + Map report = endpoint(admission).messaging(); assertThat(report).containsEntry("globalInFlight", 2); assertThat(destinations(report).get(0)).containsEntry("inFlight", 2); @@ -96,8 +104,7 @@ class MessagingEndpointTest { @Test void theReportCarriesNoPerMessageIdentity() { - Map entry = - destinations(endpoint(new BackpressureController(100, 10)).messaging()).get(0); + Map entry = destinations(endpoint(admission(100, 10)).messaging()).get(0); assertThat(entry.keySet()) .as("an actuator response that echoes message content is a re-identification surface") diff --git a/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingLiveRoundTripQualificationTest.java b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingLiveRoundTripQualificationTest.java index 56e9fd7..ceaa89c 100644 --- a/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingLiveRoundTripQualificationTest.java +++ b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingLiveRoundTripQualificationTest.java @@ -59,7 +59,11 @@ class MessagingLiveRoundTripQualificationTest { private static final String TOPIC = "order-events"; private static final DestinationName DLQ = new DestinationName("order-events-dlq"); - @Container static final KafkaContainer KAFKA = new KafkaContainer("apache/kafka:3.8.0"); + // The version the support matrix certifies and every other fixture pins. Qualifying runtime + // membership against 3.8.0 proved the membership on a broker two minors below the one the matrix + // calls certified, so the one test that decides whether the starter may reach a deployment's + // classpath was the only one not run against what a deployment gets. + @Container static final KafkaContainer KAFKA = new KafkaContainer("apache/kafka:4.1.0"); static boolean dockerAvailable() { return DockerClientFactory.instance().isDockerAvailable(); diff --git a/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingOutboxRelayLifecycleTest.java b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingOutboxRelayLifecycleTest.java new file mode 100644 index 0000000..4f283df --- /dev/null +++ b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingOutboxRelayLifecycleTest.java @@ -0,0 +1,203 @@ +package dev.caskeleton.messaging.autoconfigure; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.MessageEnvelope; +import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.ProducerId; +import dev.caskeleton.messaging.api.destination.MessageDestination; +import dev.caskeleton.messaging.api.publish.MessagePublisher; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.outbox.OutboxEnvelopeFactory; +import dev.caskeleton.messaging.reliability.OutboxLease; +import dev.caskeleton.messaging.reliability.OutboxRecord; +import dev.caskeleton.messaging.reliability.OutboxRepository; +import dev.caskeleton.messaging.reliability.OutboxTransitionResult; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Whether a started context drives the outbox relay. + * + *

The auto-configuration built the relay's settings and its retry scheduler and never built a + * relay, let alone ran one. An application that added the starter got a table that fills up behind + * business transactions reporting success. + */ +class MessagingOutboxRelayLifecycleTest { + + private final ApplicationContextRunner runner = + new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(MessagingReliabilityAutoConfiguration.class)) + .withUserConfiguration(RelayDependencies.class); + + @Test + @DisplayName("a started context runs relay passes without the application driving them") + void aStartedContextRunsRelayPasses() { + runner.run( + context -> { + assertThat(context).hasSingleBean(MessagingOutboxRelayLifecycle.class); + CountingOutboxRepository repository = context.getBean(CountingOutboxRepository.class); + + assertThat(repository.claimed.await(5, TimeUnit.SECONDS)) + .as("a relay nobody calls is an outbox that stages messages and publishes none") + .isTrue(); + assertThat(context.getBean(MessagingOutboxRelayLifecycle.class).isRunning()).isTrue(); + }); + } + + @Test + @DisplayName("closing the context stops the relay rather than leaving it publishing") + void closingTheContextStopsTheRelay() { + MessagingOutboxRelayLifecycle[] captured = new MessagingOutboxRelayLifecycle[1]; + + runner.run( + context -> { + captured[0] = context.getBean(MessagingOutboxRelayLifecycle.class); + assertThat(captured[0].isRunning()).isTrue(); + }); + + assertThat(captured[0].isRunning()) + .as("a relay thread outliving its context publishes from a half-disposed application") + .isFalse(); + } + + /** The two beans only the application can supply, plus the publisher the platform needs. */ + @Configuration(proxyBeanMethods = false) + static class RelayDependencies { + + @Bean + CountingOutboxRepository outboxRepository() { + return new CountingOutboxRepository(); + } + + @Bean + OutboxEnvelopeFactory outboxEnvelopeFactory() { + return new OutboxEnvelopeFactory(new ProducerId("order-api")); + } + + @Bean + MessagePublisher messagePublisher() { + return new MessagePublisher() { + @Override + public CompletionStage publish( + MessageDestination destination, MessageEnvelope message, PublishOptions options) { + throw new UnsupportedOperationException("this relay never claims a row to publish"); + } + }; + } + } + + /** + * An outbox that has nothing to publish and counts being asked. + * + *

Only the claim is implemented: the point under test is whether anything asks, not what the + * relay does with what it gets. Every other method throws so that a test which starts depending + * on one fails loudly instead of silently exercising a stub. + */ + static final class CountingOutboxRepository implements OutboxRepository { + + private final CountDownLatch claimed = new CountDownLatch(1); + + @Override + public List claimBatch( + String owner, int batchSize, Duration leaseDuration, Instant now, int maxAttempts) { + claimed.countDown(); + return List.of(); + } + + @Override + public void append(OutboxRecord record) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public List leaseBatch(int batchSize, Duration leaseDuration, Instant now) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public List claimBatch( + String owner, int batchSize, Duration leaseDuration, Instant now) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public void markPublished(MessageId messageId, Instant now) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public OutboxTransitionResult markPublished(OutboxLease lease, Instant now) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public void markAmbiguous(MessageId messageId, String failureCode, Instant now) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public OutboxTransitionResult markAmbiguous( + OutboxLease lease, String failureCode, Instant now) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public OutboxTransitionResult markAmbiguous( + OutboxLease lease, String failureCode, Instant now, Instant nextAttemptAt) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public OutboxTransitionResult markExhausted(OutboxLease lease, String reason, Instant now) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public void markFailed(MessageId messageId, String failureCode, Instant now) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public OutboxTransitionResult markFailed(OutboxLease lease, String failureCode, Instant now) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public void releaseLease(MessageId messageId) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public OutboxTransitionResult releaseLease(OutboxLease lease) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public Optional find(MessageId messageId) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public int purgePublishedBefore(Instant publishedBefore) { + throw new UnsupportedOperationException("not used by this test"); + } + + @Override + public int purgePublishedBefore(Instant publishedBefore, int limit) { + throw new UnsupportedOperationException("not used by this test"); + } + } +} diff --git a/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingShutdownLifecycleTest.java b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingShutdownLifecycleTest.java new file mode 100644 index 0000000..4c8d470 --- /dev/null +++ b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingShutdownLifecycleTest.java @@ -0,0 +1,78 @@ +package dev.caskeleton.messaging.autoconfigure; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.caskeleton.messaging.api.error.MessageBackpressureException; +import dev.caskeleton.messaging.policy.InFlightLimiter; +import dev.caskeleton.messaging.policy.MessagingAdmissionController; +import dev.caskeleton.messaging.policy.PayloadLimitGuard; +import dev.caskeleton.messaging.policy.PayloadPolicy; +import dev.caskeleton.messaging.transport.GracefulShutdownCoordinator; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * What a context stop actually does to in-flight messaging work. + * + *

The admission gate and the drain coordinator both existed and neither was driven by anything, + * so a shutdown went straight to destroying beans: publishes in flight, handlers between their side + * effect and their settlement, and no phase in between. + */ +class MessagingShutdownLifecycleTest { + + private static final Instant NOW = Instant.parse("2026-08-15T12:00:00Z"); + + private final MessagingAdmissionController admission = + new MessagingAdmissionController( + new PayloadLimitGuard(PayloadPolicy.defaults()), + new InFlightLimiter(4, Duration.ofMillis(10))); + + private final GracefulShutdownCoordinator drain = + new GracefulShutdownCoordinator(Duration.ofSeconds(30)); + + @Test + @DisplayName("stopping refuses new publishes before waiting for the ones in flight") + void stoppingRefusesNewPublishesFirst() { + MessagingShutdownLifecycle lifecycle = lifecycle(Duration.ofSeconds(30), NOW); + lifecycle.start(); + + lifecycle.stop(); + + assertThatThrownBy(() -> admission.admit("order-events", 16)) + .as("a drain that new work keeps topping up finishes only when the deadline passes") + .isInstanceOf(MessageBackpressureException.class); + assertThat(drain.isAcceptingWork()).isFalse(); + assertThat(lifecycle.isRunning()).isFalse(); + } + + @Test + @DisplayName("a handler that never returns does not hold the process open") + void aLeakedHandlerDoesNotHoldTheProcessOpen() { + // Work that begins and never ends: the drain can only finish by reaching its deadline. + GracefulShutdownCoordinator leaking = new GracefulShutdownCoordinator(Duration.ofMillis(100)); + leaking.tryBeginWork(); + AtomicReference clock = new AtomicReference<>(NOW); + MessagingShutdownLifecycle lifecycle = + new MessagingShutdownLifecycle( + admission, + leaking, + Duration.ofMillis(150), + () -> clock.getAndUpdate(current -> current.plusMillis(100))); + lifecycle.start(); + + lifecycle.stop(); + + assertThat(leaking.inFlight()) + .as("work still running at the deadline is abandoned unsettled, not counted as done") + .isEqualTo(1); + assertThat(leaking.abandonedWorkAtDeadline(clock.get())).isTrue(); + } + + private MessagingShutdownLifecycle lifecycle(Duration drainDeadline, Instant fixed) { + return new MessagingShutdownLifecycle(admission, drain, drainDeadline, () -> fixed); + } +} diff --git a/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingStarterOffContractTest.java b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingStarterOffContractTest.java index 7c07939..2d15475 100644 --- a/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingStarterOffContractTest.java +++ b/messaging/messaging-spring-boot-starter/src/test/java/dev/caskeleton/messaging/autoconfigure/MessagingStarterOffContractTest.java @@ -132,18 +132,41 @@ class MessagingStarterOffContractTest { } @Test - @DisplayName("selecting Rabbit assembles Rabbit and not Kafka") - void selectingRabbitAssemblesOnlyRabbit() { - withAPublisher() + @DisplayName("selecting Rabbit is refused, and the message says the transport is not finished") + void selectingRabbitIsRefused() { + // This test used to run under `withAPublisher()` and assert the context started. The fake + // MessagePublisher tripped @ConditionalOnMissingBean and removed the very bean whose missing + // dependency is the defect — so a configuration that cannot start in any deployment passed as + // "assembles Rabbit and not Kafka". Without the fake, the real behaviour appears. + // + // Rabbit ships validators and a security configurer and no MessagingTransport, because its + // native channel publisher has only test implementations. That is refused at selection now, + // where the message can say so, rather than surfacing as "No qualifying bean of type + // MessagingTransport" from inside the core configuration. + runner .withPropertyValues("app.messaging.enabled=true", "app.messaging.broker=rabbit") .run( context -> { - assertThat(context).hasNotFailed(); - assertThat(context).hasSingleBean(RabbitMessagingAutoConfiguration.class); - assertThat(context).doesNotHaveBean(KafkaMessagingAutoConfiguration.class); + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()) + .rootCause() + .hasMessageContaining("app.messaging.broker=rabbit") + .hasMessageContaining("no MessagingTransport") + .hasMessageContaining("kafka"); }); } + @Test + @DisplayName("a fake publisher does not make an unassemblable transport look assemblable") + void aFakePublisherDoesNotHideAnUnassemblableTransport() { + // The guard on the test above. Supplying a MessagePublisher removes the dependency that fails, + // so if the refusal ever moves back to bean wiring this case goes green while the deployment + // stays broken. Selection-time refusal is what makes it fail either way. + withAPublisher() + .withPropertyValues("app.messaging.enabled=true", "app.messaging.broker=rabbit") + .run(context -> assertThat(context).hasFailed()); + } + @Test @DisplayName("MSG-INT-003: a selected transport now assembles a publisher") void aSelectedTransportAssemblesAPublisher() { diff --git a/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/BrokerCertificationEvidence.java b/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/BrokerCertificationEvidence.java index 524f6ec..ce02606 100644 --- a/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/BrokerCertificationEvidence.java +++ b/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/BrokerCertificationEvidence.java @@ -1,20 +1,27 @@ package dev.caskeleton.messaging.testkit; import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.Objects; /** * One scenario actually exercised against one broker image. * - *

{@code BrokerFailureMatrix.shipped()} hard-coded every scenario as {@code LIVE_BROKER} for - * both brokers, and the suite that "verified" the matrix asserted the contents of that same map. A - * test comparing a constant to itself passes on a machine with no Docker at all, and the support - * matrix it feeds then tells an adopter that connection-cut and confirm-timeout are certified when - * neither has ever run. + *

Coverage used to be a constant: every scenario marked live-broker for both brokers, with the + * suite that "verified" the matrix asserting the contents of that same map. A test comparing a + * constant to itself passes on a machine with no Docker at all, and the support matrix it feeds + * then tells an adopter that connection-cut and confirm-timeout are certified when neither has ever + * run. * *

This record is what a lane emits after running. The image reference and the commit are here * because "certified" is a claim about a specific broker build and a specific source tree; without * them the evidence cannot be checked against anything later. + * + *

{@link #toJson()} and {@link #fromJson(String)} are the same statement in both directions: the + * certification lane writes a line per scenario it proved and the compatibility matrix reads those + * lines back. One line is one object rather than an array, so appending a scenario is an append and + * a diff of two runs is a diff of two sets of lines. */ public record BrokerCertificationEvidence( String adapter, @@ -37,24 +44,103 @@ public record BrokerCertificationEvidence( throw new IllegalArgumentException( "evidence names the adapter, the broker image and the test that produced it"); } + // The serialised form is a flat object written by hand, so a quote or a backslash in any field + // would produce a manifest line that the reader silently truncates instead of rejecting. Every + // value here is an identifier, an image reference or an instant; none of them needs escaping, + // and a value that does is a sign the caller is recording something else. + requireUnescaped(adapter, "adapter"); + requireUnescaped(brokerImage, "brokerImage"); + requireUnescaped(testId, "testId"); + requireUnescaped(outcome, "outcome"); + requireUnescaped(gitCommit, "gitCommit"); } - /** The machine-readable form a release lane writes out. */ + /** The order fields appear in, shared by the writer and the reader. */ + private static final String[] FIELDS = { + "adapter", "brokerImage", "scenario", "testId", "outcome", "gitCommit", "observedAt" + }; + + /** + * The machine-readable form a release lane writes out. + * + * @return one JSON object on a single line + */ public String toJson() { - return "{\"adapter\":\"" - + adapter - + "\",\"brokerImage\":\"" - + brokerImage - + "\",\"scenario\":\"" - + scenario.name() - + "\",\"testId\":\"" - + testId - + "\",\"outcome\":\"" - + outcome - + "\",\"gitCommit\":\"" - + gitCommit - + "\",\"observedAt\":\"" - + observedAt - + "\"}"; + String[] values = { + adapter, brokerImage, scenario.name(), testId, outcome, gitCommit, observedAt.toString() + }; + StringBuilder json = new StringBuilder("{"); + for (int index = 0; index < FIELDS.length; index++) { + if (index > 0) { + json.append(','); + } + json.append('"').append(FIELDS[index]).append("\":\"").append(values[index]).append('"'); + } + return json.append('}').toString(); + } + + /** + * Reads back one line a lane wrote. + * + *

Hand-rolled rather than pulled from a JSON library because this module is the test support + * for a platform whose {@code *-api} leaves carry no dependencies at all, and because the shape + * is fixed by {@link #toJson()} above. A missing or unknown field is refused: evidence that + * cannot be fully read is not partial coverage, it is a manifest nobody can act on. + * + * @param json one line produced by {@link #toJson()} + * @return the evidence that line records + */ + public static BrokerCertificationEvidence fromJson(String json) { + Objects.requireNonNull(json, "json must not be null"); + Map fields = parse(json.trim()); + for (String field : FIELDS) { + if (!fields.containsKey(field)) { + throw new IllegalArgumentException( + "certification evidence is missing '" + field + "': " + json); + } + } + if (fields.size() != FIELDS.length) { + throw new IllegalArgumentException( + "certification evidence carries fields this release does not define: " + fields.keySet()); + } + return new BrokerCertificationEvidence( + fields.get("adapter"), + fields.get("brokerImage"), + NetworkFaultScenario.byName(fields.get("scenario")), + fields.get("testId"), + fields.get("outcome"), + fields.get("gitCommit"), + Instant.parse(fields.get("observedAt"))); + } + + private static Map parse(String json) { + if (json.length() < 2 || json.charAt(0) != '{' || json.charAt(json.length() - 1) != '}') { + throw new IllegalArgumentException("certification evidence is not a JSON object: " + json); + } + Map fields = new LinkedHashMap<>(); + String body = json.substring(1, json.length() - 1); + int cursor = 0; + while (cursor < body.length()) { + int keyStart = body.indexOf('"', cursor); + if (keyStart < 0) { + break; + } + int keyEnd = body.indexOf('"', keyStart + 1); + int valueStart = body.indexOf('"', keyEnd + 1); + int valueEnd = valueStart < 0 ? -1 : body.indexOf('"', valueStart + 1); + if (keyEnd < 0 || valueStart < 0 || valueEnd < 0) { + throw new IllegalArgumentException("certification evidence is truncated: " + json); + } + fields.put(body.substring(keyStart + 1, keyEnd), body.substring(valueStart + 1, valueEnd)); + cursor = valueEnd + 1; + } + return fields; + } + + private static void requireUnescaped(String value, String field) { + if (value.indexOf('"') >= 0 || value.indexOf('\\') >= 0) { + throw new IllegalArgumentException( + field + " must not contain a quote or a backslash: " + field + "='" + value + "'"); + } } } diff --git a/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/BrokerFailureMatrix.java b/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/BrokerFailureMatrix.java index 012ea2e..acb7d50 100644 --- a/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/BrokerFailureMatrix.java +++ b/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/BrokerFailureMatrix.java @@ -75,6 +75,17 @@ public final class BrokerFailureMatrix { .toList(); } + /** + * Reports whether an adapter has been run against a live broker at all. + * + * @param adapter the adapter module name + * @return true when at least one scenario has live-broker evidence + */ + public boolean hasLiveBrokerCoverage(String adapter) { + return NetworkFaultScenario.all().stream() + .anyMatch(scenario -> coverageOf(adapter, scenario) == Coverage.LIVE_BROKER); + } + /** * Reports whether an adapter covers every scenario. * @@ -94,22 +105,13 @@ public final class BrokerFailureMatrix { return List.copyOf(coverage.keySet()); } - /** - * Returns the matrix the platform ships with. - * - *

Kafka and RabbitMQ are Stable and cover everything. The experimental adapters cover the - * deterministic scenarios their contract suite exercises and are explicitly recorded as having no - * live-broker coverage — which is what makes their Experimental tier a statement of fact rather - * than a disclaimer. - * - * @return the shipped coverage matrix - */ /** * Builds a matrix from evidence a lane actually produced. * - *

Anything with no evidence is {@link Coverage#NOT_COVERED}. That is the whole difference from - * {@link #shipped()}: a scenario is covered because a test ran it against a broker, not because - * somebody wrote the constant. + *

Anything with no evidence is {@link Coverage#NOT_COVERED}. A hard-coded matrix marked every + * scenario as live-broker coverage for both brokers, and the suite that checked it asserted the + * same constants, so the whole claim passed on a machine with no Docker. A scenario is covered + * because a test ran it against a broker, not because somebody wrote the constant. * * @param evidence what the certification lane observed */ @@ -118,43 +120,28 @@ public final class BrokerFailureMatrix { Objects.requireNonNull(evidence, "evidence must not be null"); BrokerFailureMatrix matrix = new BrokerFailureMatrix(); for (BrokerCertificationEvidence observed : evidence) { + requireOutcomeMatchesExpectation(observed); matrix.record(observed.adapter(), observed.scenario(), Coverage.LIVE_BROKER); } return matrix; } /** - * The declared shipping matrix. + * Refuses evidence whose observed outcome contradicts the scenario it claims to cover. * - * @deprecated These constants describe what the platform intends to certify, not what any run - * proved. Every scenario was marked {@code LIVE_BROKER} for both brokers and the suite that - * checked the matrix asserted the same map, so it passed on a machine with no Docker. Use - * {@link #from(java.util.Collection)} with a lane's evidence; this remains only so an - * existing caller keeps compiling while it migrates. + *

Nothing compared the two, so a run that reported a definite rejection where the scenario + * says the outcome is unknowable counted as coverage of that scenario. That is worse than no + * evidence: it certifies the adapter for exactly the behaviour it got wrong, and the support + * matrix then tells an adopter the case is handled. */ - @Deprecated - public static BrokerFailureMatrix shipped() { - BrokerFailureMatrix matrix = new BrokerFailureMatrix(); - for (NetworkFaultScenario scenario : NetworkFaultScenario.all()) { - matrix.record("messaging-kafka", scenario, Coverage.LIVE_BROKER); - matrix.record("messaging-rabbit", scenario, Coverage.LIVE_BROKER); + private static void requireOutcomeMatchesExpectation(BrokerCertificationEvidence observed) { + String expected = observed.scenario().expectation().name(); + if (!expected.equals(observed.outcome())) { + throw new IllegalArgumentException( + "%s recorded %s for %s, which expects %s; a run that disagrees with the scenario is a" + .formatted( + observed.adapter(), observed.outcome(), observed.scenario().name(), expected) + + " failing certification, not coverage of it (%s)".formatted(observed.testId())); } - matrix.record( - "messaging-pulsar-experimental", - NetworkFaultScenario.CONNECTION_CUT_AFTER_WRITE, - Coverage.DETERMINISTIC); - matrix.record( - "messaging-pulsar-experimental", - NetworkFaultScenario.CONFIRM_TIMEOUT, - Coverage.DETERMINISTIC); - matrix.record( - "messaging-nats-experimental", - NetworkFaultScenario.CONNECTION_CUT_AFTER_WRITE, - Coverage.DETERMINISTIC); - matrix.record( - "messaging-nats-experimental", - NetworkFaultScenario.CONFIRM_TIMEOUT, - Coverage.DETERMINISTIC); - return matrix; } } diff --git a/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/CertifiedEvidence.java b/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/CertifiedEvidence.java new file mode 100644 index 0000000..6d7f614 --- /dev/null +++ b/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/CertifiedEvidence.java @@ -0,0 +1,84 @@ +package dev.caskeleton.messaging.testkit; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Set; + +/** + * The fault scenarios this repository's certification lane has actually exercised. + * + *

Read from a manifest a run produced, not from a list somebody typed. It was the list: an + * author added a line and the compatibility matrix, the cross-broker suite and the support matrix + * all believed it, on a machine that may never have started a container. "Certified against a live + * broker" then meant "a developer wrote that down", which is the one thing a certification claim + * must not mean. + * + *

The manifest is committed because a release claim has to survive the build that produced it — + * a matrix that reads {@code build/} answers differently on a clean checkout. What stops it from + * being hand-written again is the gate: {@code verifyMessagingCertificationEvidence} runs the lane + * against a real broker and refuses to pass when this file claims a scenario the run did not + * produce. Editing the file without running the lane fails the build; running the lane rewrites the + * file. + * + *

An absent or empty manifest is not an error here. It means no scenario has evidence, so every + * scenario is {@link BrokerFailureMatrix.Coverage#NOT_COVERED} and every Stable claim that depends + * on evidence fails — which is the correct reading of "nothing has been run". + */ +public final class CertifiedEvidence { + + /** Where the certification lane writes what it proved, and where the matrix reads it. */ + public static final String MANIFEST_RESOURCE = "/messaging/broker-certification-evidence.jsonl"; + + private static final List RECORDED = load(); + + private CertifiedEvidence() {} + + /** + * Returns what the certification lane recorded. + * + * @return the evidence this repository can show + */ + public static List recorded() { + return RECORDED; + } + + /** + * Returns the scenarios an adapter has no evidence for. + * + *

Named explicitly so the gap is a fact in the build rather than a silent absence. + * + * @param adapter the adapter module name + * @return the uncovered scenarios + */ + public static List knownGaps(String adapter) { + Set covered = + recorded().stream() + .filter(evidence -> evidence.adapter().equals(adapter)) + .map(BrokerCertificationEvidence::scenario) + .collect(java.util.stream.Collectors.toUnmodifiableSet()); + return NetworkFaultScenario.all().stream() + .filter(scenario -> !covered.contains(scenario)) + .toList(); + } + + private static List load() { + try (InputStream manifest = CertifiedEvidence.class.getResourceAsStream(MANIFEST_RESOURCE)) { + if (manifest == null) { + return List.of(); + } + return new String(manifest.readAllBytes(), StandardCharsets.UTF_8) + .lines() + .map(String::trim) + .filter(line -> !line.isEmpty()) + .map(BrokerCertificationEvidence::fromJson) + .toList(); + } catch (IOException failure) { + // Not swallowed into "no evidence": a manifest that exists and cannot be read is a broken + // build, and reporting it as an absence would quietly demote every adapter instead. + throw new IllegalStateException( + "could not read the broker certification manifest " + MANIFEST_RESOURCE, failure); + } + } +} diff --git a/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/CompatibilityMatrix.java b/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/CompatibilityMatrix.java index 0b0c07b..0eb721b 100644 --- a/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/CompatibilityMatrix.java +++ b/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/CompatibilityMatrix.java @@ -9,8 +9,11 @@ import java.util.Objects; * *

Held as code with a test over it rather than as a document, because a support matrix that * lives only in prose drifts the first time an adapter is added. The test asserts the rules the - * design fixes: a Stable adapter must run the shared contract, and an Experimental adapter must be - * disabled by default. + * design fixes: a Stable adapter must run the shared contract and be certified against a live + * broker, and an Experimental adapter must be disabled by default. + * + *

The certification claim is derived from lane evidence rather than declared beside the tier, so + * an adapter cannot be promoted by editing a boolean. */ public final class CompatibilityMatrix { @@ -36,15 +39,27 @@ public final class CompatibilityMatrix { * @param tier the support tier * @param runsSharedContract whether the module runs {@link MessagingAdapterContract} * @param enabledByDefault whether the adapter is active without an explicit opt-in - * @param hasLiveBrokerCertification whether a container-backed suite exercises a real broker */ public record Entry( String adapter, List brokerVersions, Tier tier, boolean runsSharedContract, - boolean enabledByDefault, - boolean hasLiveBrokerCertification) { + boolean enabledByDefault) { + + /** + * Reports whether a lane has run this adapter against a real broker. + * + *

Read from the evidence rather than declared. As a field it was a boolean an author set + * next to the tier, and RabbitMQ carried {@code true} while no fault scenario had ever been + * executed against it — the same shape as the coverage matrix that used to assert its own + * constants. + * + * @return true when {@link CertifiedEvidence} has live-broker evidence for this adapter + */ + public boolean hasLiveBrokerCertification() { + return BrokerFailureMatrix.from(CertifiedEvidence.recorded()).hasLiveBrokerCoverage(adapter); + } public Entry { Objects.requireNonNull(tier, "tier must not be null"); @@ -67,16 +82,20 @@ public final class CompatibilityMatrix { // the locked client is 4.1.1, so the matrix and the support document agreed with each // other and with nothing that had executed. A version returns here when a lane runs // against its image. - new Entry("messaging-kafka", List.of("4.1"), Tier.STABLE, true, true, true), + new Entry("messaging-kafka", List.of("4.1"), Tier.STABLE, true, true), "messaging-rabbit", - new Entry("messaging-rabbit", List.of("4.3"), Tier.STABLE, true, true, true), + // Experimental until a lane runs a fault scenario against a RabbitMQ container. The + // adapter passes the shared contract and its broker IT exercises the routable and + // unroutable happy paths, but Stable in this repository means fault evidence, and + // there is none: connection-refused, cut-after-write, confirm-timeout, + // settlement-lost and high-latency have never been executed here. + new Entry("messaging-rabbit", List.of("4.3"), Tier.EXPERIMENTAL, true, false), "messaging-kafka-share-experimental", new Entry( "messaging-kafka-share-experimental", List.of("4.2", "4.3"), Tier.EXPERIMENTAL, false, - false, false), "messaging-pulsar-experimental", new Entry( @@ -84,16 +103,10 @@ public final class CompatibilityMatrix { List.of("4.0", "4.2"), Tier.EXPERIMENTAL, false, - false, false), "messaging-nats-experimental", new Entry( - "messaging-nats-experimental", - List.of("2.14"), - Tier.EXPERIMENTAL, - false, - false, - false)); + "messaging-nats-experimental", List.of("2.14"), Tier.EXPERIMENTAL, false, false)); /** * Returns every certified adapter. diff --git a/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/NetworkFaultScenario.java b/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/NetworkFaultScenario.java index 665afb8..403c954 100644 --- a/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/NetworkFaultScenario.java +++ b/messaging/messaging-testkit/src/main/java/dev/caskeleton/messaging/testkit/NetworkFaultScenario.java @@ -111,6 +111,30 @@ public record NetworkFaultScenario( "latency above the publish deadline is indistinguishable from a lost confirm at the " + "moment the decision has to be made"); + /** + * Resolves a scenario by the name a recorded run wrote down. + * + *

Evidence travels as text, so the read side has to answer what a name that this release does + * not define means. It means the manifest describes a scenario the code no longer has, and + * treating it as coverage would credit an adapter for a fault nobody can reproduce — so it is + * refused rather than skipped. + * + * @param name the scenario name as it appears in recorded evidence + * @return the scenario + */ + public static NetworkFaultScenario byName(String name) { + return all().stream() + .filter(scenario -> scenario.name().equals(name)) + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + "no fault scenario is named '" + + name + + "'; evidence for a scenario this release does not define is coverage of" + + " nothing")); + } + /** * Returns every scenario the shared fault suite runs. * diff --git a/messaging/messaging-testkit/src/main/resources/messaging/broker-certification-evidence.jsonl b/messaging/messaging-testkit/src/main/resources/messaging/broker-certification-evidence.jsonl new file mode 100644 index 0000000..5a94717 --- /dev/null +++ b/messaging/messaging-testkit/src/main/resources/messaging/broker-certification-evidence.jsonl @@ -0,0 +1,4 @@ +{"adapter":"messaging-kafka","brokerImage":"apache/kafka:4.1.0","scenario":"confirm-timeout","testId":"KafkaBrokerCertificationIT#aConfirmThatNeverArrivesIsReportedAsAmbiguousAndTheRecordIsThere","outcome":"AMBIGUOUS","gitCommit":"e98b56eb03ecab588c21fd1e7dbcaa493c1d8645","observedAt":"2026-08-19T14:21:30.826332993Z"} +{"adapter":"messaging-kafka","brokerImage":"apache/kafka:4.1.0","scenario":"connection-cut-after-write","testId":"KafkaBrokerCertificationIT#aConnectionCutAfterTheWriteIsReportedAsAmbiguous","outcome":"AMBIGUOUS","gitCommit":"e98b56eb03ecab588c21fd1e7dbcaa493c1d8645","observedAt":"2026-08-19T14:21:34.906464945Z"} +{"adapter":"messaging-kafka","brokerImage":"apache/kafka:4.1.0","scenario":"high-latency","testId":"KafkaBrokerCertificationIT#latencyBeyondThePublishDeadlineIsReportedAsAmbiguous","outcome":"AMBIGUOUS","gitCommit":"e98b56eb03ecab588c21fd1e7dbcaa493c1d8645","observedAt":"2026-08-19T14:21:21.870705263Z"} +{"adapter":"messaging-kafka","brokerImage":"apache/kafka:4.1.0","scenario":"settlement-lost","testId":"KafkaBrokerCertificationIT#aDeliveryLostBeforeSettlementIsRedelivered","outcome":"REDELIVERED","gitCommit":"e98b56eb03ecab588c21fd1e7dbcaa493c1d8645","observedAt":"2026-08-19T14:21:42.058283607Z"} diff --git a/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CertifiedEvidence.java b/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CertifiedEvidence.java deleted file mode 100644 index b152a32..0000000 --- a/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CertifiedEvidence.java +++ /dev/null @@ -1,72 +0,0 @@ -package dev.caskeleton.messaging.testkit; - -import java.time.Instant; -import java.util.List; -import java.util.Set; - -/** - * The fault scenarios this repository's container lanes actually exercise today. - * - *

Committed rather than generated because the generated form belongs to a release run: a - * certification lane writes {@link BrokerCertificationEvidence} per scenario, and the release gate - * compares that output to what the support matrix claims. This file is the same statement for the - * lanes that exist in the repository right now, so the cross-broker suite has something real to - * compare against instead of comparing a hard-coded matrix to itself. - * - *

Adding a line here without a lane that produces it re-creates the defect. The rule is: a - * scenario appears when a test in this repository runs it against a container, and the test id - * names that test. - */ -final class CertifiedEvidence { - - private static final String COMMIT = "repository-local"; - - private CertifiedEvidence() {} - - /** What the container lanes in this repository exercise. */ - static List recorded() { - Instant observedAt = Instant.parse("2026-08-15T00:00:00Z"); - return List.of( - new BrokerCertificationEvidence( - "messaging-kafka", - "apache/kafka:4.1.0", - NetworkFaultScenario.CONNECTION_CUT_AFTER_WRITE, - "KafkaAmbiguityChaosIT#aConnectionCutAfterWriteIsAmbiguous", - "AMBIGUOUS", - COMMIT, - observedAt), - new BrokerCertificationEvidence( - "messaging-kafka", - "apache/kafka:4.1.0", - NetworkFaultScenario.SETTLEMENT_LOST, - "KafkaConsumerSettlementIT#aLostSettlementIsRedelivered", - "AMBIGUOUS", - COMMIT, - observedAt), - new BrokerCertificationEvidence( - "messaging-rabbit", - "rabbitmq:4.3-management", - NetworkFaultScenario.CONNECTION_CUT_AFTER_WRITE, - "RabbitBrokerIT#anUnroutablePublishIsRejected", - "REJECTED", - COMMIT, - observedAt)); - } - - /** - * The scenarios an adapter has no evidence for. - * - *

Named explicitly so the gap is a fact in the build rather than a silent absence. These are - * what MSG-014 asks the certification lane to close. - */ - static List knownGaps(String adapter) { - Set covered = - recorded().stream() - .filter(evidence -> evidence.adapter().equals(adapter)) - .map(BrokerCertificationEvidence::scenario) - .collect(java.util.stream.Collectors.toUnmodifiableSet()); - return NetworkFaultScenario.all().stream() - .filter(scenario -> !covered.contains(scenario)) - .toList(); - } -} diff --git a/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CertifiedEvidenceTest.java b/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CertifiedEvidenceTest.java new file mode 100644 index 0000000..0ed64f1 --- /dev/null +++ b/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CertifiedEvidenceTest.java @@ -0,0 +1,127 @@ +package dev.caskeleton.messaging.testkit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * Holds the line between evidence a run produced and evidence a developer typed. + * + *

The distinction is the whole of the finding this suite exists for. A committed list of + * scenarios reads exactly like a manifest and passes exactly the same assertions, so the only thing + * that can tell them apart is where the values come from — which is what these tests check. + */ +class CertifiedEvidenceTest { + + @Test + void everyRecordedScenarioIsALineTheCertificationLaneWrote() throws IOException { + List manifestLines = manifest(); + + assertThat(manifestLines) + .as("the certification lane writes this file; nothing else may") + .isNotEmpty(); + assertThat(CertifiedEvidence.recorded()) + .as("coverage is read from the manifest, not from a list somebody maintained beside it") + .containsExactlyElementsOf( + manifestLines.stream().map(BrokerCertificationEvidence::fromJson).toList()); + } + + @Test + void everyClaimNamesTheImageItRanAgainst() { + assertThat(CertifiedEvidence.recorded()) + .allSatisfy( + evidence -> + assertThat(evidence.brokerImage()) + .as("a floating tag makes a certification unrepeatable") + .contains(":") + .doesNotEndWith(":latest")); + } + + @Test + void everyClaimNamesTheTestThatProducedIt() { + assertThat(CertifiedEvidence.recorded()) + .allSatisfy(evidence -> assertThat(evidence.testId()).contains("#")); + } + + @Test + void aScenarioWithNoLineInTheManifestIsAGapRatherThanAnAbsence() { + assertThat(CertifiedEvidence.knownGaps("messaging-kafka")) + .as("a Kafka producer buffers before it learns a connection exists, so this stays unproven") + .contains(NetworkFaultScenario.CONNECTION_REFUSED); + assertThat(CertifiedEvidence.knownGaps("messaging-rabbit")) + .as("no lane runs a fault scenario against RabbitMQ, so every scenario is a gap") + .containsExactlyElementsOf(NetworkFaultScenario.all()); + } + + @Test + void anEvidenceLineRoundTripsThroughItsSerialisedForm() { + BrokerCertificationEvidence evidence = + new BrokerCertificationEvidence( + "messaging-kafka", + "apache/kafka:4.1.0", + NetworkFaultScenario.CONFIRM_TIMEOUT, + "SomeIT#someScenario", + "AMBIGUOUS", + "0123456789abcdef", + java.time.Instant.parse("2026-08-15T00:00:00Z")); + + assertThat(BrokerCertificationEvidence.fromJson(evidence.toJson())).isEqualTo(evidence); + } + + @Test + void aManifestLineMissingAFieldIsRefusedRatherThanPartiallyRead() { + assertThatThrownBy( + () -> + BrokerCertificationEvidence.fromJson( + "{\"adapter\":\"messaging-kafka\",\"scenario\":\"confirm-timeout\"}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("brokerImage"); + } + + @Test + void aManifestLineNamingAScenarioThisReleaseDoesNotDefineIsRefused() { + assertThatThrownBy( + () -> + BrokerCertificationEvidence.fromJson( + "{\"adapter\":\"messaging-kafka\",\"brokerImage\":\"apache/kafka:4.1.0\"," + + "\"scenario\":\"disk-full\",\"testId\":\"X#y\",\"outcome\":\"AMBIGUOUS\"," + + "\"gitCommit\":\"abc\",\"observedAt\":\"2026-08-15T00:00:00Z\"}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("disk-full"); + } + + @Test + void aValueThatWouldBreakTheManifestIsRefusedWhenTheEvidenceIsBuilt() { + assertThatThrownBy( + () -> + new BrokerCertificationEvidence( + "messaging-kafka", + "apache/kafka:4.1.0", + NetworkFaultScenario.CONFIRM_TIMEOUT, + "Some\"IT#someScenario", + "AMBIGUOUS", + "abc", + java.time.Instant.parse("2026-08-15T00:00:00Z"))) + .as("a quote would truncate the line the release gate reads back") + .isInstanceOf(IllegalArgumentException.class); + } + + private static List manifest() throws IOException { + try (InputStream stream = + CertifiedEvidence.class.getResourceAsStream(CertifiedEvidence.MANIFEST_RESOURCE)) { + assertThat(stream) + .as("%s is missing; no lane has produced evidence", CertifiedEvidence.MANIFEST_RESOURCE) + .isNotNull(); + return new String(stream.readAllBytes(), StandardCharsets.UTF_8) + .lines() + .map(String::trim) + .filter(line -> !line.isEmpty()) + .toList(); + } + } +} diff --git a/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CompatibilityMatrixTest.java b/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CompatibilityMatrixTest.java index 0099d09..c2bd66d 100644 --- a/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CompatibilityMatrixTest.java +++ b/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CompatibilityMatrixTest.java @@ -37,11 +37,26 @@ class CompatibilityMatrixTest { } @Test - void theStableSetIsExactlyKafkaAndRabbit() { + void theStableSetIsExactlyWhatALaneHasCertified() { assertThat(CompatibilityMatrix.entries().values()) .filteredOn(entry -> entry.tier() == CompatibilityMatrix.Tier.STABLE) .extracting(CompatibilityMatrix.Entry::adapter) - .containsExactlyInAnyOrder("messaging-kafka", "messaging-rabbit"); + .as("RabbitMQ passes the shared contract, but no fault scenario has been run against it") + .containsExactly("messaging-kafka"); + } + + @Test + void aCertificationClaimCannotBeMadeWithoutEvidence() { + assertThat(CompatibilityMatrix.entries().values()) + .allSatisfy( + entry -> + assertThat(entry.hasLiveBrokerCertification()) + .as( + "%s claims certification, so a lane must have produced evidence", + entry.adapter()) + .isEqualTo( + BrokerFailureMatrix.from(CertifiedEvidence.recorded()) + .hasLiveBrokerCoverage(entry.adapter()))); } @Test @@ -75,7 +90,7 @@ class CompatibilityMatrixTest { assertThatThrownBy( () -> new CompatibilityMatrix.Entry( - "messaging-x", List.of(), CompatibilityMatrix.Tier.STABLE, true, true, true)) + "messaging-x", List.of(), CompatibilityMatrix.Tier.STABLE, true, true)) .isInstanceOf(IllegalArgumentException.class); } @@ -99,4 +114,15 @@ class CompatibilityMatrixTest { .filteredOn(entry -> entry.tier() == CompatibilityMatrix.Tier.EXPERIMENTAL) .allSatisfy(entry -> assertThat(entry.hasLiveBrokerCertification()).isFalse()); } + + @Test + void anAdapterWithoutEvidenceCannotBeCalledStable() { + CompatibilityMatrix.Entry undeclared = + new CompatibilityMatrix.Entry( + "messaging-artemis", List.of("2.42"), CompatibilityMatrix.Tier.STABLE, true, false); + + assertThat(undeclared.hasLiveBrokerCertification()) + .as("promoting an adapter used to be a boolean an author could set beside the tier") + .isFalse(); + } } diff --git a/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CrossBrokerContractSuite.java b/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CrossBrokerContractSuite.java index c3a5079..adb286a 100644 --- a/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CrossBrokerContractSuite.java +++ b/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/CrossBrokerContractSuite.java @@ -22,14 +22,14 @@ class CrossBrokerContractSuite { /** * The matrix built from the evidence this repository actually has. * - *

It used to be {@code BrokerFailureMatrix.shipped()} — a hard-coded map marking every - * scenario {@code LIVE_BROKER} for both brokers — and the assertions below then compared that map - * to itself. This suite passed on a machine with no Docker, and the support matrix it feeds told - * adopters that connection-cut and confirm-timeout were certified when neither had ever run. + *

A hard-coded map marking every scenario live-broker for both brokers would leave the + * assertions below comparing that map to itself: the suite passes on a machine with no Docker, + * and the support matrix it feeds tells adopters that connection-cut and confirm-timeout are + * certified when neither has ever run. * - *

{@link CertifiedEvidence#recorded()} is the committed record of what the container lanes - * exercise today. The assertions below now fail when the shipped claim exceeds it, which is the - * failure they were written to catch. + *

{@link CertifiedEvidence#recorded()} is the manifest the certification lane wrote, so the + * assertions below fail when the shipped claim exceeds what a run produced — which is the failure + * they were written to catch. */ private final BrokerFailureMatrix matrix = BrokerFailureMatrix.from(CertifiedEvidence.recorded()); @@ -53,6 +53,25 @@ class CrossBrokerContractSuite { .isEqualTo(CertifiedEvidence.knownGaps("messaging-rabbit")); } + @Test + void evidenceThatDisagreesWithItsScenarioIsNotCoverage() { + BrokerCertificationEvidence contradicting = + new BrokerCertificationEvidence( + "messaging-rabbit", + "rabbitmq:4.3-management", + NetworkFaultScenario.CONNECTION_CUT_AFTER_WRITE, + "RabbitBrokerIT#anUnroutablePublishIsRejectedEvenThoughTheExchangeConfirmedIt", + "REJECTED", + "repository-local", + java.time.Instant.parse("2026-08-15T00:00:00Z")); + + org.assertj.core.api.Assertions.assertThatThrownBy( + () -> BrokerFailureMatrix.from(List.of(contradicting))) + .as("certifying an adapter for the behaviour it got wrong is worse than no evidence") + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("AMBIGUOUS"); + } + @Test void aScenarioWithNoEvidenceIsNotCovered() { assertThat(matrix.coverageOf("messaging-kafka", NetworkFaultScenario.CONNECTION_REFUSED)) @@ -64,7 +83,7 @@ class CrossBrokerContractSuite { void anEvidencedScenarioIsCovered() { assertThat( matrix.coverageOf("messaging-kafka", NetworkFaultScenario.CONNECTION_CUT_AFTER_WRITE)) - .as("KafkaAmbiguityChaosIT exercises this against a container") + .as("the certification lane cuts the connection after the write against a container") .isEqualTo(Coverage.LIVE_BROKER); } diff --git a/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/contract/MessagingPublicSurfaceContractTest.java b/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/contract/MessagingPublicSurfaceContractTest.java new file mode 100644 index 0000000..c29034c --- /dev/null +++ b/messaging/messaging-testkit/src/test/java/dev/caskeleton/messaging/testkit/contract/MessagingPublicSurfaceContractTest.java @@ -0,0 +1,350 @@ +package dev.caskeleton.bootstrap.contract.messaging; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Whether an adopter can compile against the messaging types this platform hands them. + * + *

A Gradle {@code implementation} dependency is not encapsulation. It removes the library from + * the consumer's compile classpath, but it does not remove the library's types from this module's + * public signatures. When both are true the artifact ships an API nobody can call: the method is + * there, the javadoc describes it, and the caller cannot name its parameter or its return type + * without guessing which library and which version to add to their own build. Every case below was + * exactly that — a Kafka {@code Producer}, a Spring AMQP {@code Message}, a CloudEvent, a + * Micrometer {@code MeterRegistry}, an Avro {@code Schema}, a protobuf {@code Parser}, a Reactor + * {@code Mono}. + * + *

So the rule is one-directional and mechanical: a vendor library named in a public or protected + * signature is declared {@code api}; one used only behind private members stays {@code + * implementation}. {@code src/messaging/CLAUDE.md} states it as policy — this holds it to the + * source so the two cannot drift. + * + *

The libraries checked are the ones an adopter would otherwise have to reverse-engineer into + * their own build: broker clients, codec libraries, the metrics registry, the reactive facade. + * Spring Framework and Spring Boot are deliberately out of scope. They appear only inside the + * composition boundary, where the container constructs the types rather than an adopter naming + * them, and that boundary is where the family policy already confines them. + */ +class MessagingPublicSurfaceContractTest { + + /** + * Package prefix to the Gradle coordinate that supplies it. + * + *

Prefixes rather than exact packages, because a library's types are spread across its own + * sub-packages and a consumer needs the whole artifact either way. + */ + private static final Map VENDOR_LIBRARIES = + Map.of( + "org.apache.kafka", "org.apache.kafka:kafka-clients", + "org.springframework.amqp", "org.springframework.amqp:spring-rabbit", + "org.apache.avro", "org.apache.avro:avro", + "com.google.protobuf", "com.google.protobuf:protobuf-java", + "tools.jackson", "tools.jackson.core:jackson-databind", + "io.cloudevents", "io.cloudevents:cloudevents-api", + "io.micrometer", "io.micrometer:micrometer-core", + "reactor", "io.projectreactor:reactor-core", + "io.nats", "io.nats:jnats", + "org.apache.pulsar", "org.apache.pulsar:pulsar-client"); + + private static final Pattern IMPORT = Pattern.compile("(?m)^import\\s+(?:static\\s+)?([^;]+);"); + + private static final Pattern TYPE_KEYWORD = + Pattern.compile("\\b(class|interface|record|enum)\\b"); + + @Test + @DisplayName( + "a vendor type in a public signature has its library declared api, not implementation") + void vendorTypesInPublicSignaturesAreDeclaredApi() { + List hidden = new ArrayList<>(); + + for (Path leaf : messagingLeaves()) { + String buildScript = read(leaf.resolve("build.gradle")); + for (String coordinate : exposedLibraries(leaf)) { + if (!declaresAsApi(buildScript, coordinate)) { + hidden.add(leaf.getFileName() + " exposes " + coordinate + " without declaring it api"); + } + } + } + + assertThat(hidden) + .as( + "each of these ships a public signature naming a type the consumer's compile classpath " + + "does not contain; the method exists and cannot be called") + .isEmpty(); + } + + @Test + @DisplayName("the scan reaches the leaves it claims to check") + void theScanIsNotVacuous() { + List leaves = messagingLeaves(); + + assertThat(leaves) + .as("a scan that found no leaf would report every violation as absent") + .hasSizeGreaterThan(20); + assertThat( + exposedLibraries( + leaves.stream().filter(isNamed("messaging-kafka")).findFirst().orElseThrow())) + .as("the Kafka adapter's publish seam takes a Producer, so this is a positive control") + .contains("org.apache.kafka:kafka-clients"); + assertThat( + exposedLibraries( + leaves.stream().filter(isNamed("messaging-schema-json")).findFirst().orElseThrow())) + .as("the JSON codec keeps its ObjectMapper private, so nothing is required of its build") + .isEmpty(); + } + + @Test + @DisplayName("an adopter of the starter does not compile against either broker client") + void neitherBrokerSdkReachesAnAdoptersCompileClasspath() { + Map> exported = new LinkedHashMap<>(); + for (Path leaf : messagingLeaves()) { + exported.put( + leaf.getFileName().toString(), apiDependenciesOf(read(leaf.resolve("build.gradle")))); + } + + Set reachable = apiClosure("messaging-spring-boot-starter", exported); + + assertThat(reachable) + .as( + "an empty closure would satisfy the assertion below without proving anything; the " + + "metrics registry and the reactive facade are exported on purpose") + .contains("io.micrometer:micrometer-core", "io.projectreactor:reactor-core"); + assertThat(reachable) + .as( + "both adapters were exported, so every adopter compiled against two broker SDKs at " + + "once and each adapter's conditional-on-class guard was true whatever the " + + "deployment had chosen") + .doesNotContain("org.apache.kafka:kafka-clients", "org.springframework.amqp:spring-rabbit"); + } + + /** The vendor coordinates this leaf names in a public or protected signature. */ + private static Set exposedLibraries(Path leaf) { + Set exposed = new TreeSet<>(); + for (Path source : productionSources(leaf)) { + String text = read(source); + String declarations = declarationsOnly(text); + for (Map.Entry library : VENDOR_LIBRARIES.entrySet()) { + if (isNamedInDeclarations(text, declarations, library.getKey())) { + exposed.add(library.getValue()); + } + } + } + return exposed; + } + + /** + * Whether a library's types survive into this file's declarations. + * + *

Two ways a type gets named: imported and used by its simple name, or written out in full. + * Both are checked, because the auto-configuration style in this repository uses fully qualified + * names for one-off types rather than adding an import. + */ + private static boolean isNamedInDeclarations(String source, String declarations, String prefix) { + if (declarations.contains(prefix + ".")) { + return true; + } + Matcher imports = IMPORT.matcher(source); + while (imports.find()) { + String imported = imports.group(1).trim(); + if (!imported.startsWith(prefix + ".")) { + continue; + } + String simpleName = imported.substring(imported.lastIndexOf('.') + 1); + if (Pattern.compile("\\b" + Pattern.quote(simpleName) + "\\b").matcher(declarations).find()) { + return true; + } + } + return false; + } + + /** + * Reduces a source file to its declarations: type headers, member signatures, record components. + * + *

Method bodies are dropped, because a vendor type used inside one is an implementation detail + * and is exactly the case that may stay {@code implementation}. Field initialisers are dropped + * for the same reason. Anything a {@code private} modifier reaches is dropped last: a private + * member is not a signature a consumer compiles against. + * + *

A brace walk rather than a parser. It only has to answer "is this text a declaration or a + * body", which brace depth settles, and a parser here would be more machinery than the question + * deserves. + */ + private static String declarationsOnly(String source) { + String text = stripCommentsAndLiterals(source); + StringBuilder kept = new StringBuilder(); + StringBuilder pending = new StringBuilder(); + int depth = 0; + int bodyDepth = -1; + int initializerDepth = -1; + + for (int index = 0; index < text.length(); index++) { + char character = text.charAt(index); + if (character == '{') { + depth++; + if (initializerDepth >= 0) { + continue; + } + if (bodyDepth < 0) { + kept.append(pending).append(';'); + if (!TYPE_KEYWORD.matcher(pending).find() && pending.indexOf("(") >= 0) { + bodyDepth = depth; + } + } + pending.setLength(0); + continue; + } + if (character == '}') { + if (bodyDepth == depth) { + bodyDepth = -1; + } + depth--; + pending.setLength(0); + continue; + } + if (character == ';') { + if (initializerDepth == depth) { + initializerDepth = -1; + } else if (bodyDepth < 0) { + kept.append(pending).append(';'); + } + pending.setLength(0); + continue; + } + if (bodyDepth >= 0 || initializerDepth >= 0) { + continue; + } + if (character == '=' && isAssignment(text, index)) { + kept.append(pending).append(';'); + pending.setLength(0); + initializerDepth = depth; + continue; + } + pending.append(character); + } + + return Stream.of(kept.toString().split(";", -1)) + .filter(fragment -> !fragment.contains("private ")) + .filter(fragment -> !fragment.contains("import ")) + .reduce("", (left, right) -> left + " ; " + right); + } + + /** Distinguishes a field initialiser from {@code ==}, {@code <=}, {@code >=} and {@code !=}. */ + private static boolean isAssignment(String text, int index) { + char before = index == 0 ? ' ' : text.charAt(index - 1); + char after = index + 1 >= text.length() ? ' ' : text.charAt(index + 1); + return before != '=' && before != '<' && before != '>' && before != '!' && after != '='; + } + + private static String stripCommentsAndLiterals(String source) { + return source + .replaceAll("(?s)/\\*.*?\\*/", " ") + .replaceAll("(?m)//.*$", " ") + .replaceAll("\"\"\"(?s).*?\"\"\"", "\"\"") + .replaceAll("\"(\\\\.|[^\"\\\\])*\"", "\"\"") + .replaceAll("'(\\\\.|[^'\\\\])*'", "' '"); + } + + /** The external coordinates a build script exports to its consumers. */ + private static List apiDependenciesOf(String buildScript) { + List exported = new ArrayList<>(); + Matcher external = Pattern.compile("(?m)^\\s*api\\s+'([^']+)'").matcher(buildScript); + while (external.find()) { + String coordinate = external.group(1); + int lastColon = coordinate.lastIndexOf(':'); + boolean versioned = lastColon > 0 && coordinate.indexOf(':') != lastColon; + exported.add(versioned ? coordinate.substring(0, lastColon) : coordinate); + } + Matcher projects = + Pattern.compile("(?m)^\\s*api\\s+project\\('([^']+)'\\)").matcher(buildScript); + while (projects.find()) { + String path = projects.group(1); + exported.add(path.substring(path.lastIndexOf(':') + 1)); + } + return exported; + } + + /** Everything a consumer of one leaf ends up compiling against. */ + private static Set apiClosure(String leaf, Map> exported) { + Set reachable = new TreeSet<>(); + List pending = new ArrayList<>(exported.getOrDefault(leaf, List.of())); + while (!pending.isEmpty()) { + String next = pending.remove(0); + if (!reachable.add(next)) { + continue; + } + pending.addAll(exported.getOrDefault(next, List.of())); + } + return reachable; + } + + private static boolean declaresAsApi(String buildScript, String coordinate) { + return Pattern.compile("(?m)^\\s*api\\s+'" + Pattern.quote(coordinate) + "(:[^']*)?'") + .matcher(buildScript) + .find(); + } + + private static java.util.function.Predicate isNamed(String leaf) { + return path -> path.getFileName().toString().equals(leaf); + } + + private static List messagingLeaves() { + Path family = repositoryRoot().resolve("src/messaging"); + try (Stream children = Files.list(family)) { + return children + .filter(Files::isDirectory) + .filter(path -> Files.isRegularFile(path.resolve("build.gradle"))) + .sorted() + .toList(); + } catch (IOException unreadable) { + throw new UncheckedIOException(unreadable); + } + } + + private static List productionSources(Path leaf) { + Path main = leaf.resolve("src/main/java"); + if (!Files.isDirectory(main)) { + return List.of(); + } + try (Stream walk = Files.walk(main)) { + return walk.filter(path -> path.toString().endsWith(".java")).sorted().toList(); + } catch (IOException unreadable) { + throw new UncheckedIOException(unreadable); + } + } + + private static String read(Path path) { + try { + return Files.readString(path); + } catch (IOException unreadable) { + throw new UncheckedIOException(unreadable); + } + } + + private static Path repositoryRoot() { + Path candidate = Path.of("").toAbsolutePath(); + while (candidate != null + && !Files.isRegularFile(candidate.resolve("src/config/architecture/modules.json"))) { + candidate = candidate.getParent(); + } + if (candidate == null) { + throw new IllegalStateException("the repository root is not above the working directory"); + } + return candidate; + } +} diff --git a/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/BackpressureController.java b/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/BackpressureController.java deleted file mode 100644 index a126bd0..0000000 --- a/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/BackpressureController.java +++ /dev/null @@ -1,125 +0,0 @@ -package dev.caskeleton.messaging.transport; - -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Bounds in-flight work per destination and across the process. - * - *

Two limits, because one is not enough. A per-destination limit stops a single slow downstream - * from consuming every worker, and a global limit stops the sum of well-behaved destinations from - * exhausting memory. Without the global bound, adding a destination silently raises the process - * ceiling. - * - *

Refusing work is the point. An unbounded in-flight queue does not remove backpressure, it - * moves it into the heap, where it surfaces as an out-of-memory error instead of a visible, - * recoverable slowdown. - */ -public final class BackpressureController { - - private final int globalLimit; - private final int perDestinationLimit; - private final AtomicInteger global = new AtomicInteger(); - private final Map perDestination = new ConcurrentHashMap<>(); - - /** - * Creates a controller. - * - * @param globalLimit the process-wide in-flight ceiling - * @param perDestinationLimit the per-destination in-flight ceiling - */ - public BackpressureController(int globalLimit, int perDestinationLimit) { - if (globalLimit < 1) { - throw new IllegalArgumentException("globalLimit must be at least 1"); - } - if (perDestinationLimit < 1) { - throw new IllegalArgumentException("perDestinationLimit must be at least 1"); - } - if (perDestinationLimit > globalLimit) { - throw new IllegalArgumentException( - "perDestinationLimit cannot exceed the global limit, or the global limit is not a limit"); - } - this.globalLimit = globalLimit; - this.perDestinationLimit = perDestinationLimit; - } - - /** - * Tries to admit one unit of work. - * - * @param destination the logical destination - * @return true when the work may start - */ - public boolean tryAcquire(String destination) { - Objects.requireNonNull(destination, "destination must not be null"); - AtomicInteger counter = perDestination.computeIfAbsent(destination, key -> new AtomicInteger()); - - if (!increment(counter, perDestinationLimit)) { - return false; - } - if (!increment(global, globalLimit)) { - counter.decrementAndGet(); - return false; - } - return true; - } - - /** - * Releases one unit of work. - * - * @param destination the logical destination - */ - public void release(String destination) { - Objects.requireNonNull(destination, "destination must not be null"); - AtomicInteger counter = perDestination.get(destination); - if (counter == null) { - // A release for a destination that holds nothing. It used to decrement the global counter - // anyway, which frees a permit nobody took. - return; - } - int before = counter.getAndUpdate(current -> current > 0 ? current - 1 : current); - if (before <= 0) { - // Already at zero: a double release. Decrementing the global counter here is how the global - // limit stops limiting — it reads below the real in-flight count and admits more work than - // the process can carry. - return; - } - global.updateAndGet(current -> current > 0 ? current - 1 : current); - // Drop the entry once it reaches zero, atomically, so the map does not accumulate one counter - // per destination ever published to for the life of the process. - perDestination.computeIfPresent(destination, (key, value) -> value.get() == 0 ? null : value); - } - - /** - * Returns the process-wide in-flight count. - * - * @return the global in-flight count - */ - public int globalInFlight() { - return global.get(); - } - - /** - * Returns a destination's in-flight count. - * - * @param destination the logical destination - * @return the in-flight count - */ - public int inFlight(String destination) { - AtomicInteger counter = perDestination.get(destination); - return counter == null ? 0 : counter.get(); - } - - private static boolean increment(AtomicInteger counter, int limit) { - while (true) { - int current = counter.get(); - if (current >= limit) { - return false; - } - if (counter.compareAndSet(current, current + 1)) { - return true; - } - } - } -} diff --git a/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/DefaultMessagingRuntimeRegistry.java b/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/DefaultMessagingRuntimeRegistry.java index bc86c77..5bdc2de 100644 --- a/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/DefaultMessagingRuntimeRegistry.java +++ b/messaging/messaging-transport-spi/src/main/java/dev/caskeleton/messaging/transport/DefaultMessagingRuntimeRegistry.java @@ -22,7 +22,8 @@ import java.util.concurrent.atomic.AtomicInteger; *

A drain deadline bounds that wait. Without one, a single leaked lease would pin a revoked * credential open indefinitely, which turns a rotation into a security non-event. */ -public final class DefaultMessagingRuntimeRegistry implements MessagingRuntimeRegistry { +public final class DefaultMessagingRuntimeRegistry + implements MessagingRuntimeRegistry, AutoCloseable { private static final Duration DEFAULT_DRAIN_DEADLINE = Duration.ofSeconds(30); @@ -91,28 +92,17 @@ public final class DefaultMessagingRuntimeRegistry implements MessagingRuntimeRe } /** - * Closes retired generations whose drain deadline has passed. + * Closes retired generations whose own drain deadline has passed. + * + *

Per generation, not per call. A single {@code retiredAt} supplied by the caller and applied + * to the whole draining list gives a generation retired thirty seconds ago and one retired a + * moment ago the same deadline, and a generation that has already gone idle and closed stays in + * the list, counted as draining forever. * *

Called by the platform's scheduler. Exposed as a plain method taking the current instant so * that deadline behaviour is testable without sleeping. * * @param now the current instant - * @param retiredAt when the generations were retired - * @return how many generations were force closed - */ - public int closeExpiredDraining(Instant now, Instant retiredAt) { - return closeExpiredDraining(now); - } - - /** - * Closes retired generations whose own drain deadline has passed. - * - *

Per generation, not per call. The previous signature took one {@code retiredAt} and applied - * it to the whole draining list, so a generation retired thirty seconds ago and one retired a - * moment ago shared a deadline — and a generation that had already gone idle and closed stayed in - * the list, counted as draining forever. - * - * @param now the current instant * @return how many generations were force closed */ public int closeExpiredDraining(Instant now) { @@ -135,6 +125,36 @@ public final class DefaultMessagingRuntimeRegistry implements MessagingRuntimeRe return closed; } + /** + * Closes every generation this registry holds, current and draining. + * + *

Nothing closed the current generation. The registry only ever closed what a rotation had + * retired, so a process that shut down without rotating left its broker connections to the JVM's + * exit — which drops unflushed producer batches and leaves consumer sessions to time out on the + * broker instead of leaving the group. + * + *

Each generation closes exactly once, whichever path reaches it first: a drain that finished + * during shutdown must not be closed a second time here, because a second close on a broker + * client is where "already closed" exceptions come from during an otherwise clean stop. + */ + @Override + public void close() { + List pending; + synchronized (draining) { + pending = List.copyOf(draining); + draining.clear(); + } + for (Generation generation : pending) { + generation.forceClose(); + } + for (String brokerName : List.copyOf(current.keySet())) { + Generation generation = current.remove(brokerName); + if (generation != null) { + generation.forceClose(); + } + } + } + /** * Returns how many retired generations are still draining. * diff --git a/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/BackpressureAndShutdownTest.java b/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/BackpressureAndShutdownTest.java deleted file mode 100644 index 74d6936..0000000 --- a/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/BackpressureAndShutdownTest.java +++ /dev/null @@ -1,148 +0,0 @@ -package dev.caskeleton.messaging.transport; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import java.time.Duration; -import java.time.Instant; -import org.junit.jupiter.api.Test; - -class BackpressureAndShutdownTest { - - private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); - - @Test - void aSlowDestinationCannotConsumeEveryWorker() { - BackpressureController controller = new BackpressureController(10, 2); - - assertThat(controller.tryAcquire("order-events")).isTrue(); - assertThat(controller.tryAcquire("order-events")).isTrue(); - assertThat(controller.tryAcquire("order-events")).isFalse(); - assertThat(controller.tryAcquire("email-work")).isTrue(); - } - - @Test - void theGlobalLimitBoundsTheSumOfWellBehavedDestinations() { - BackpressureController controller = new BackpressureController(2, 2); - - assertThat(controller.tryAcquire("a")).isTrue(); - assertThat(controller.tryAcquire("b")).isTrue(); - assertThat(controller.tryAcquire("c")).isFalse(); - assertThat(controller.globalInFlight()).isEqualTo(2); - } - - @Test - void aRefusedAcquireDoesNotLeakAPerDestinationSlot() { - BackpressureController controller = new BackpressureController(1, 1); - controller.tryAcquire("a"); - - controller.tryAcquire("b"); - - assertThat(controller.inFlight("b")).isZero(); - } - - @Test - void releasingFreesBothCounters() { - BackpressureController controller = new BackpressureController(4, 2); - controller.tryAcquire("a"); - - controller.release("a"); - - assertThat(controller.inFlight("a")).isZero(); - assertThat(controller.globalInFlight()).isZero(); - } - - @Test - void aPerDestinationLimitAboveTheGlobalOneIsRejected() { - assertThatThrownBy(() -> new BackpressureController(2, 4)) - .isInstanceOf(IllegalArgumentException.class); - } - - @Test - void drainingStopsNewWorkButLetsRunningWorkFinish() { - GracefulShutdownCoordinator coordinator = - new GracefulShutdownCoordinator(Duration.ofSeconds(30)); - assertThat(coordinator.tryBeginWork()).isTrue(); - - coordinator.beginDrain(NOW); - - assertThat(coordinator.isAcceptingWork()).isFalse(); - assertThat(coordinator.tryBeginWork()).isFalse(); - assertThat(coordinator.inFlight()).isEqualTo(1); - assertThat(coordinator.isDrained(NOW)).isFalse(); - - coordinator.endWork(); - - assertThat(coordinator.isDrained(NOW)).isTrue(); - } - - @Test - void noRetryAttemptIsCreatedOnceDrainingBegins() { - GracefulShutdownCoordinator coordinator = - new GracefulShutdownCoordinator(Duration.ofSeconds(30)); - - assertThat(coordinator.mayCreateRetryAttempt()).isTrue(); - coordinator.beginDrain(NOW); - assertThat(coordinator.mayCreateRetryAttempt()).isFalse(); - } - - @Test - void workStillRunningAtTheDeadlineIsAbandonedUnsettled() { - GracefulShutdownCoordinator coordinator = - new GracefulShutdownCoordinator(Duration.ofSeconds(30)); - coordinator.tryBeginWork(); - coordinator.beginDrain(NOW); - - assertThat(coordinator.isDrained(NOW.plusSeconds(29))).isFalse(); - assertThat(coordinator.isDrained(NOW.plusSeconds(30))).isTrue(); - assertThat(coordinator.abandonedWorkAtDeadline(NOW.plusSeconds(30))).isTrue(); - } - - @Test - void anIdleCoordinatorIsNotDrainedBeforeShutdownStarts() { - GracefulShutdownCoordinator coordinator = - new GracefulShutdownCoordinator(Duration.ofSeconds(30)); - - assertThat(coordinator.isDrained(NOW)).isFalse(); - } - - @Test - @org.junit.jupiter.api.DisplayName("releasing an unknown destination frees no global permit") - void releasingAnUnknownDestinationFreesNoGlobalPermit() { - BackpressureController controller = new BackpressureController(2, 2); - assertThat(controller.tryAcquire("orders")).isTrue(); - - controller.release("something-else"); - - assertThat(controller.globalInFlight()) - .as("a permit nobody took cannot be given back; doing so makes the global limit fiction") - .isEqualTo(1); - } - - @Test - @org.junit.jupiter.api.DisplayName("a double release frees no second global permit") - void aDoubleReleaseFreesNoSecondPermit() { - BackpressureController controller = new BackpressureController(4, 4); - assertThat(controller.tryAcquire("orders")).isTrue(); - - controller.release("orders"); - controller.release("orders"); - - assertThat(controller.globalInFlight()).isZero(); - } - - @Test - @org.junit.jupiter.api.DisplayName("a double endWork cannot drive the in-flight count negative") - void aDoubleEndWorkCannotGoNegative() { - GracefulShutdownCoordinator shutdown = - new GracefulShutdownCoordinator(java.time.Duration.ofSeconds(5)); - assertThat(shutdown.tryBeginWork()).isTrue(); - - shutdown.endWork(); - shutdown.endWork(); - - assertThat(shutdown.inFlight()) - .as("a negative count reports the drain as finished while work is still running") - .isZero(); - } -} diff --git a/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/GracefulShutdownTest.java b/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/GracefulShutdownTest.java new file mode 100644 index 0000000..4c3df9f --- /dev/null +++ b/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/GracefulShutdownTest.java @@ -0,0 +1,75 @@ +package dev.caskeleton.messaging.transport; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Duration; +import java.time.Instant; +import org.junit.jupiter.api.Test; + +class GracefulShutdownTest { + + private static final Instant NOW = Instant.parse("2026-08-10T09:15:00Z"); + + @Test + void drainingStopsNewWorkButLetsRunningWorkFinish() { + GracefulShutdownCoordinator coordinator = + new GracefulShutdownCoordinator(Duration.ofSeconds(30)); + assertThat(coordinator.tryBeginWork()).isTrue(); + + coordinator.beginDrain(NOW); + + assertThat(coordinator.isAcceptingWork()).isFalse(); + assertThat(coordinator.tryBeginWork()).isFalse(); + assertThat(coordinator.inFlight()).isEqualTo(1); + assertThat(coordinator.isDrained(NOW)).isFalse(); + + coordinator.endWork(); + + assertThat(coordinator.isDrained(NOW)).isTrue(); + } + + @Test + void noRetryAttemptIsCreatedOnceDrainingBegins() { + GracefulShutdownCoordinator coordinator = + new GracefulShutdownCoordinator(Duration.ofSeconds(30)); + + assertThat(coordinator.mayCreateRetryAttempt()).isTrue(); + coordinator.beginDrain(NOW); + assertThat(coordinator.mayCreateRetryAttempt()).isFalse(); + } + + @Test + void workStillRunningAtTheDeadlineIsAbandonedUnsettled() { + GracefulShutdownCoordinator coordinator = + new GracefulShutdownCoordinator(Duration.ofSeconds(30)); + coordinator.tryBeginWork(); + coordinator.beginDrain(NOW); + + assertThat(coordinator.isDrained(NOW.plusSeconds(29))).isFalse(); + assertThat(coordinator.isDrained(NOW.plusSeconds(30))).isTrue(); + assertThat(coordinator.abandonedWorkAtDeadline(NOW.plusSeconds(30))).isTrue(); + } + + @Test + void anIdleCoordinatorIsNotDrainedBeforeShutdownStarts() { + GracefulShutdownCoordinator coordinator = + new GracefulShutdownCoordinator(Duration.ofSeconds(30)); + + assertThat(coordinator.isDrained(NOW)).isFalse(); + } + + @Test + @org.junit.jupiter.api.DisplayName("a double endWork cannot drive the in-flight count negative") + void aDoubleEndWorkCannotGoNegative() { + GracefulShutdownCoordinator shutdown = + new GracefulShutdownCoordinator(java.time.Duration.ofSeconds(5)); + assertThat(shutdown.tryBeginWork()).isTrue(); + + shutdown.endWork(); + shutdown.endWork(); + + assertThat(shutdown.inFlight()) + .as("a negative count reports the drain as finished while work is still running") + .isZero(); + } +} diff --git a/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/MessagingRuntimeRegistryTest.java b/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/MessagingRuntimeRegistryTest.java index 128d1f8..8f532c4 100644 --- a/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/MessagingRuntimeRegistryTest.java +++ b/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/MessagingRuntimeRegistryTest.java @@ -45,6 +45,30 @@ class MessagingRuntimeRegistryTest { assertThat(registry.drainingCount()).isZero(); } + @Test + void closingTheRegistryClosesCurrentAndDrainingGenerationsExactlyOnce() { + DefaultMessagingRuntimeRegistry registry = + new DefaultMessagingRuntimeRegistry(Duration.ofSeconds(30)); + CountingRuntime retired = new CountingRuntime("kafka-primary", 1); + CountingRuntime current = new CountingRuntime("kafka-primary", 2); + + registry.install(retired); + MessagingRuntimeLease leaked = registry.acquire("kafka-primary"); + registry.install(current, Instant.parse("2026-08-10T00:00:00Z")); + + registry.close(); + registry.close(); + + assertThat(current.closes()) + .as("a process that shut down without rotating left its connections to JVM exit") + .isEqualTo(1); + assertThat(retired.closes()) + .as("a second close on a broker client is where 'already closed' at shutdown comes from") + .isEqualTo(1); + + leaked.close(); + } + @Test void closingALeaseTwiceDoesNotDoubleRelease() { DefaultMessagingRuntimeRegistry registry = new DefaultMessagingRuntimeRegistry(); @@ -232,3 +256,45 @@ final class FakeTransport implements MessagingTransport { // Nothing to release. } } + +/** A runtime that counts how many times it was closed. */ +final class CountingRuntime implements MessagingRuntime { + + private final String brokerName; + private final long generation; + private int closes; + + CountingRuntime(String brokerName, long generation) { + this.brokerName = brokerName; + this.generation = generation; + } + + int closes() { + return closes; + } + + @Override + public String brokerName() { + return brokerName; + } + + @Override + public long generation() { + return generation; + } + + @Override + public MessagingTransport transport() { + return new FakeTransport(brokerName, generation); + } + + @Override + public boolean isClosed() { + return closes > 0; + } + + @Override + public void close() { + closes++; + } +} diff --git a/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/ResourceLeakGateTest.java b/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/ResourceLeakGateTest.java index fdc498b..5c07d46 100644 --- a/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/ResourceLeakGateTest.java +++ b/messaging/messaging-transport-spi/src/test/java/dev/caskeleton/messaging/transport/ResourceLeakGateTest.java @@ -56,34 +56,6 @@ class ResourceLeakGateTest { .isTrue(); } - @Test - void everyAcquiredInFlightSlotIsReleased() { - BackpressureController controller = new BackpressureController(8, 4); - - for (int i = 0; i < 1_000; i++) { - if (controller.tryAcquire("order-events")) { - controller.release("order-events"); - } - } - - assertThat(controller.inFlight("order-events")).isZero(); - assertThat(controller.globalInFlight()).isZero(); - } - - @Test - void aRefusedAcquireLeavesNoSlotBehind() { - BackpressureController controller = new BackpressureController(2, 1); - controller.tryAcquire("a"); - controller.tryAcquire("b"); - - for (int i = 0; i < 100; i++) { - controller.tryAcquire("c"); - } - - assertThat(controller.inFlight("c")).isZero(); - assertThat(controller.globalInFlight()).isEqualTo(2); - } - @Test void aDrainAlwaysTerminatesEvenWithStuckWork() { GracefulShutdownCoordinator coordinator =