fix(outbound)!: 발행·SMTP 발송·캐시 시간 상한과 S3·Mongo·HTTP 압축을 맞춘다 #37
+2
-1
@@ -171,6 +171,7 @@ APP_CACHE_REDIS_COMMAND_TIMEOUT=200ms
|
||||
APP_CACHE_REDIS_POSITIVE_SOFT_TTL=30s
|
||||
APP_CACHE_REDIS_MINIMUM_HARD_TTL=1s
|
||||
APP_CACHE_REDIS_SEMANTIC_REGION=default
|
||||
APP_CACHE_REDIS_MAXIMUM_VALUE_BYTES=1048576
|
||||
APP_MESSAGING_BROKER=kafka
|
||||
APP_MESSAGING_PRODUCER_ID=
|
||||
APP_MESSAGING_KAFKA_BROKERS=
|
||||
@@ -293,7 +294,7 @@ APP_NOTIFICATION_PLATFORM_SMTP_ENABLED=false
|
||||
APP_NOTIFICATION_PLATFORM_SMTP_PRIMARY=true
|
||||
APP_NOTIFICATION_PLATFORM_SMTP_ENVIRONMENT=local
|
||||
APP_NOTIFICATION_PLATFORM_SMTP_CREDENTIAL_PROFILE=default
|
||||
APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT=10s
|
||||
APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT=30s
|
||||
APP_NOTIFICATION_PLATFORM_SMTP_MAX_CONCURRENCY=4
|
||||
APP_NOTIFICATION_PLATFORM_SMTP_RATE_PER_SECOND=10
|
||||
APP_NOTIFICATION_PLATFORM_SMTP_SENDER_IDENTITY=no-reply@example.invalid
|
||||
|
||||
@@ -359,6 +359,15 @@
|
||||
<Method name="abortAtDeadline"/>
|
||||
</Match>
|
||||
|
||||
<!-- DLS_DEAD_LOCAL_STORE, the same disagreement: the SMTP send's queue deadline discards the
|
||||
stage `thenRun` returns into `unused`, because an abandoned send reports through its own
|
||||
result, not through that stage. -->
|
||||
<Match>
|
||||
<Bug pattern="DLS_DEAD_LOCAL_STORE"/>
|
||||
<Class name="dev.caskeleton.adapter.outbound.notification.platform.provider.smtp.SmtpNotificationProviderAdapter$PooledSend"/>
|
||||
<Method name="armQueueDeadline"/>
|
||||
</Match>
|
||||
|
||||
<!-- The servlet error-dispatch controller (re-review 2026-11-03 RR-02-02) must answer for any
|
||||
HTTP method: the original request's method is whatever failed, and the container forwards it
|
||||
unchanged to /error. CSRF protection guards a state change made with ambient credentials;
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
# root yet.
|
||||
# Update only after review with:
|
||||
# ./gradlew :adapter:outbound:persistence-jpa:updateJpaApiSurface -PapproveJpaApiSurfaceChange
|
||||
# types: 165
|
||||
# types: 168
|
||||
dev.caskeleton.adapter.outbound.persistence.api.PersistenceOperationName
|
||||
dev.caskeleton.adapter.outbound.persistence.api.capability.CapabilitySupport
|
||||
dev.caskeleton.adapter.outbound.persistence.api.capability.JpaCapability
|
||||
@@ -104,6 +104,8 @@ dev.caskeleton.adapter.outbound.persistence.idempotency.postgresql.IdempotencySc
|
||||
dev.caskeleton.adapter.outbound.persistence.idempotency.postgresql.PostgreSqlIdempotencyStore
|
||||
dev.caskeleton.adapter.outbound.persistence.liveevent.JpaLiveEventReplay
|
||||
dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventJpaRepository
|
||||
dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReaper
|
||||
dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReplaySettings
|
||||
dev.caskeleton.adapter.outbound.persistence.liveevent.entity.LiveEventEntity
|
||||
dev.caskeleton.adapter.outbound.persistence.lock.DistributedLockPersistenceConfig
|
||||
dev.caskeleton.adapter.outbound.persistence.lock.LockRegistryDistributedLock
|
||||
@@ -169,5 +171,6 @@ dev.caskeleton.adapter.outbound.persistence.springdata.SafeSortField
|
||||
dev.caskeleton.adapter.outbound.persistence.springdata.SafeSortMapper
|
||||
dev.caskeleton.adapter.outbound.persistence.springdata.SafeSortRegistry
|
||||
dev.caskeleton.adapter.outbound.persistence.springdata.ScrollPolicy
|
||||
dev.caskeleton.adapter.outbound.persistence.transaction.BatchedDeletes
|
||||
dev.caskeleton.adapter.outbound.persistence.transaction.CallerTransaction
|
||||
dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction
|
||||
|
||||
@@ -167,8 +167,8 @@ profile에는 `connect`와 `response-header`를 가장 짧은 호출자 예산
|
||||
| `redirect.max-hops` | `0` | redirect를 켜고 hop을 0으로 두면 설정 오류다 |
|
||||
| `redirect.allow-cross-origin` | `false` | 켜면 hop에서 credential을 지운다 |
|
||||
| `request.max-body-bytes` | `0` | 운영에서 필수 |
|
||||
| `request.compression` | `false` | |
|
||||
| `response.max-wire-bytes` | `5242880` | wire 위의 바이트 |
|
||||
| `request.compression` | `false` | 켜면 transport와 상관없이 `Accept-Encoding: gzip, deflate`를 보낸다. 받은 본문은 엔진이 아니라 이 플랫폼이 wire 바이트를 센 뒤 푼다(버퍼 본문과 스트리밍 다운로드 모두) |
|
||||
| `response.max-wire-bytes` | `5242880` | wire 위의 바이트. 압축을 켜도 압축된 크기로 센다 |
|
||||
| `response.max-decoded-bytes` | `10485760` | 디코딩한 뒤의 바이트. 넘을 수 없는 최대는 64 MiB |
|
||||
| `response.allowed-content-types` | JSON + problem+json | 비어 있으면 「아무것이나」 |
|
||||
|
||||
@@ -212,7 +212,7 @@ profile에는 `connect`와 `response-header`를 가장 짧은 호출자 예산
|
||||
| 속성 | 기본값 | 뜻 |
|
||||
|---|---|---|
|
||||
| `proxy.enabled` | `false` | |
|
||||
| `proxy.host` / `proxy.port` / `proxy.type` | — / `0` / `HTTP` | |
|
||||
| `proxy.host` / `proxy.port` / `proxy.type` | — / `0` / `HTTP` | 지금 transport는 모두 HTTP forward proxy만 한다. `type`의 값은 `HTTP` 하나이고, `SOCKS`는 바인딩이 거부한다 |
|
||||
| `proxy.credential-provider` | — | proxy 인증은 대상 인증과 따로다 |
|
||||
| `proxy.import-ambient-no-proxy` | `false` | 환경의 `NO_PROXY`는 검증한 profile을 넓히지 않는다 |
|
||||
| `observability.full-url-recording` | `false` | |
|
||||
|
||||
@@ -50,9 +50,9 @@
|
||||
|
||||
| 상태 | 판정 |
|
||||
|---|---|
|
||||
| 408 | 기한과 예산 안에서 재시도 |
|
||||
| 425 | 첫 시도에서만, 많아야 한 번 재시도 |
|
||||
| 429 | `Retry-After`, 기한, 예산 안에서 재시도 |
|
||||
| 408 | 안전하게 멱등인 연산만 기한과 예산 안에서 재시도. 아니면 결과 모름(upstream이 받은 뒤 답한 것일 수 있다) |
|
||||
| 425 | 안전하게 멱등인 연산만, 첫 시도에서만, 많아야 한 번 재시도. 아니면 결과 모름 |
|
||||
| 429 | 안전하게 멱등인 연산만 `Retry-After`, 기한, 예산 안에서 재시도. 아니면 거부(`notRetryableStatus`). 429는 일정 신호이지 "아무 일도 없었다"는 말이 아니다 |
|
||||
| 401 | 안전하고 재전송 가능한 연산에서만 갱신 후 재전송 한 번. credential은 재시도를 허락받은 뒤에만 갱신한다. 재전송하지 않을 401(시도 한 번, 예산 없음, 기한 없음)은 token cache를 건드리지 않는다 |
|
||||
| 500 | upstream이 일시적이라고 등록했고 **그리고** 연산이 안전하게 멱등일 때만 허락. 아니면 거부 |
|
||||
| 502, 503, 504 | 안전하게 멱등인 연산은 재시도. 아니면 결과 모름 |
|
||||
|
||||
@@ -7,7 +7,13 @@
|
||||
검증한다. 그래서 실패한 다운로드가 호출자가 따져야 하는 반쯤 읽힌 stream이 되지 않는다.
|
||||
|
||||
wire 바이트와 디코딩한 바이트는 따로 제한한다. 압축한 payload는 wire 검사를 통과한 뒤 늘어나기 때문이다. 한도는
|
||||
버퍼링한 뒤가 아니라 읽는 동안 강제한다.
|
||||
버퍼링한 뒤가 아니라 읽는 동안 강제한다. 압축(`Content-Encoding: gzip`·`deflate`)은 엔진이 아니라 응답 reader가
|
||||
푼다. 도착한 바이트를 wire 한도로 세고, 푼 바이트를 조각마다 디코딩 한도로 센다. 엔진이 먼저 풀면 wire 계수가 푼
|
||||
크기를 세게 되어, 잘 압축된 응답이 wire 위에는 없던 크기로 거부됐다(round9 리뷰 2026-11-12, RR9-01-30). 부탁하지 않은
|
||||
압축(`br` 등)과 겹친 coding은 거부한다. 스트리밍 다운로드도 같은 곳(`ContentDecoding`)을 거친다. 호출자는 푼 바이트를
|
||||
읽고, 응답 헤더에는 `Content-Encoding`·`Content-Length`가 없다. 풀기는 호출자가 읽을 때 시작하므로, 본문이 빈 gzip
|
||||
응답(HEAD·204·304)은 빈 본문이다. 어느 transport도 스스로 풀지 않는다. Apache는 압축 처리를 끄고, 실험 transport인
|
||||
Jetty HTTP/3은 시작할 때 classpath에서 등록한 decoder를 지운다.
|
||||
|
||||
## 첫 바이트 경계
|
||||
|
||||
|
||||
@@ -15,4 +15,5 @@
|
||||
{"schema": 1, "kind": "actual", "task": "w2-be-ci-perf", "transcript": "a6b6b25f-a4f7-46c2-a746-5ab20a588c78", "minutes": 663, "active_minutes": 417, "calls": 464, "output_tokens": 438927, "cache_read_tokens": 256417797, "max_context": 911512, "within": "over", "result": "partial", "commits": ["ecc14004", "6325b518", "86488256", "c7aff0f4", "7054d179", "575aa1f7", "deeb1126", "96caf997", "cfc80ae8", "000a59ed", "5a95db45", "d237bacc", "7284b581", "14053060", "7dbf11ca", "2c180b90", "24dde586", "aeb60339", "083c6910"], "why": "시간이 p80을 넘었다: desktop 공유 잠금 대기자 10여 개(flock이 순서 없이 깨움)로 측정·검증이 2시간 넘게 밀렸고, runner 1대 대기열에서 늦게 풀린 job이 맨 뒤로 가 수동 실행 결과가 몇 시간씩 늦었다. 예측에 없던 결정: Gitea needs+if Blocked 재현(일회용 workflow 셋), Trivy가 처음 읽은 lockfile 라이선스와 이미지가 jar 라이선스를 못 읽는 것에 맞춘 판정 재설계. 결과는 partial: runner 초록은 NFS(소유자 sudo)·compose(수정 뒤 runner 미확인) 빼고 달성, 속도 목표 넷은 미달(캐시 효과는 병합 뒤, PG 컨테이너 159개 공유는 후속).", "recorded_at": "2026-11-12T08:34+09:00"}
|
||||
{"schema": 1, "kind": "actual", "task": "w2-be-ci-perf", "correction": true, "transcript": "a6b6b25f-a4f7-46c2-a746-5ab20a588c78", "minutes": 1266, "active_minutes": 555, "calls": 556, "output_tokens": 534063, "cache_read_tokens": 301335415, "max_context": 963079, "within": "over", "result": "partial", "commits": ["ecc14004", "6325b518", "86488256", "c7aff0f4", "7054d179", "575aa1f7", "deeb1126", "96caf997", "cfc80ae8", "000a59ed", "5a95db45", "d237bacc", "7284b581", "14053060", "7dbf11ca", "2c180b90", "24dde586", "aeb60339", "083c6910", "335fc967", "9682c52c", "3e06a809", "d17f8bcf", "7db40c9b"], "why": "앞 actual 줄을 고친다. 그 뒤 총괄 지시 둘(새 main 위로, main.yml concurrency와 취소 증거)과 main 두 번 병합으로 작업 중 138분·호출 92가 더 들었다. 대부분 runner 대기열(PR run이 한 번에 2~4시간 뒤로 밀림)과 desktop 공유 잠금 대기였다. 결과는 여전히 partial: runner PR 130 초록·release gate 125 초록, NFS는 소유자 sudo, 속도 목표 넷은 미달(PR 13.5→12.0분/목표 8, release gate 45.6→32.5분/목표 35는 이미지 job 포함), 캐시 효과는 병합 뒤 측정.", "recorded_at": "2026-11-12T18:36+09:00"}
|
||||
{"schema": 1, "kind": "estimate", "task": "w2-be-ci-perf-2", "transcript": "a6b6b25f-a4f7-46c2-a746-5ab20a588c78", "started_at": "2026-11-12T18:45+09:00", "base": "e7be517e", "task_kind": "improve", "size": "large", "minutes": {"p30": 150, "p50": 240, "p80": 400}, "calls": {"p30": 120, "p50": 200, "p80": 320}, "output_tokens": {"p30": 100000, "p50": 170000, "p80": 280000}, "basis": "직전 라운드 w2-be-ci-perf 실측 555분(p50 144의 3.9배, 대부분 runner·desktop 잠금 대기). 이번 판단 5개(B2·NFS·C2·job 합치기·main 줄 순서), desktop main 측정 1회(30분)+pr 1회, runner nightly 1회(2시간)+release 1회+PR 2회. 대기가 시간을 정한다", "recorded_at": "2026-11-12T18:58+09:00"}
|
||||
{"schema": 1, "kind": "estimate", "task": "r9-group-b", "transcript": "0ef672ab-db3e-427a-9f53-b2e918462534", "started_at": "2026-11-13T07:00:00+09:00", "base": "d3a90479", "task_kind": "improve", "size": "large", "minutes": {"p30": 190, "p50": 286, "p80": 424}, "calls": {"p30": 378, "p50": 567, "p80": 819}, "output_tokens": {"p30": 348000, "p50": 522000, "p80": 754000}, "basis": "발견 42건·PR 3·결정 약 15·desktop 실행 약 20회(Testcontainers lane)·코드 약 3000줄; 원값 360/540/800분·600/900/1300호출·600k/900k/1300k에 보정 0.53/0.63/0.58 곱함; 선례 w2-be-ci-perf 555분", "recorded_at": "2026-11-13T06:52+09:00"}
|
||||
{"schema": 1, "kind": "estimate", "task": "r9-group-a", "transcript": "70449164-6ab3-49a9-9bcd-a4281e2a0759", "started_at": "2026-11-13T07:00+09:00", "base": "d3a90479", "task_kind": "improve", "size": "large", "minutes": {"p30": 160, "p50": 240, "p80": 370}, "calls": {"p30": 250, "p50": 380, "p80": 570}, "output_tokens": {"p30": 200000, "p50": 320000, "p80": 490000}, "basis": "발견 31건·PR 3개, 손 판단 결정 약 8(전송별 인가 전달, graphql 하위 좌표, test profile 기동 거부, broadcaster 목적지, metrics registry 맞추기 등), desktop 검증 약 8회(leaf·pr 단계); 원값 300/450/700분·400/600/900호출·350k/550k/850k에 보정 0.53·0.63·0.58을 곱함", "recorded_at": "2026-11-13T06:52+09:00"}
|
||||
|
||||
@@ -91,6 +91,17 @@ store는 자기 트랜잭션을 열지 않는다. 유스케이스가 연 트랜
|
||||
예: `DurableOperationStoreContract`(`InMemoryDurableOperationStore`, `JpaDurableOperationStoreContractTest`).
|
||||
- 여러 인스턴스가 집는 작업 → `FOR UPDATE SKIP LOCKED`는 **트랜잭션 안에서**, 그리고 lease(만료 시각)와
|
||||
fence 값을 함께 저장한다. 예: `ReconciliationJobStorePort.claimDue`(application-notification).
|
||||
- 연장·완료·되돌리기는 그 claim의 owner와 fence를 `WHERE`에 넣은 조건부 UPDATE 하나다. fence는 claim마다
|
||||
오르므로 같은 worker id가 다시 claim해도 앞 실행의 쓰기는 0행이다(예: `DurableOperationClaim`).
|
||||
- `WHERE id IN (SELECT … LIMIT n)`으로 고른 행을 바꿀 때는 subquery를 `FOR UPDATE SKIP LOCKED`로 잠가 남이 쥔 행을
|
||||
고르지 않는다. 상태 조건도 바깥 `UPDATE … WHERE`에 한 번 더 둔다. 잠금 절이 빠지면 READ COMMITTED는 잠금을 기다린
|
||||
행에 바깥 조건만 다시 보고 subquery 안 조건은 보지 않기 때문이다(예: `RecipientClaimSql.EXPIRE_OVERDUE`). 잠그지 않는
|
||||
subquery라면 바깥 조건이 유일한 방어다(예: inbox `markAllRead`).
|
||||
- 만료는 그 시각을 쓴 시계로 본다. row lease(outbox, 알림 수신자, durable operation, fileserver cleanup,
|
||||
Mongo migration lock)는 앱 `Clock`으로 쓰고 앱 `Clock`으로 비교한다. 시험이 시계를 옮길 수 있게 하려는
|
||||
것이고, 대신 인스턴스 사이 시계 차이가 가장 짧은 lease보다 훨씬 작아야 한다(NTP 동기화가 배포 조건이다).
|
||||
DB가 쓴 시각은 DB 시계로(멱등성 `expires_at`은 `clock_timestamp()`로 쓰고 `now()`로 지운다), Redis가 쓴
|
||||
시각은 Redis `TIME`으로 본다. 한 저장소 안에서 두 시계를 섞지 않는다.
|
||||
- 수정 충돌 → 버전 열(낙관적 잠금). 예: `DurableOperationEntity`(persistence-jpa)의 `@Version` 열 `row_version`.
|
||||
- **예외**: DB가 볼 수 없는 규칙(외부 시스템 상태)은 분산 락(`DistributedLockPort`)이나 멱등성 키(05장).
|
||||
- 동시성 테스트는 실 DB lane에서 동시 요청 N개 → 행 1개를 단언한다.
|
||||
|
||||
@@ -57,7 +57,10 @@ executor 조립과 실제 PostgreSQL 저장소 왕복은 `PostgreSqlIdempotencyP
|
||||
|
||||
같은 키 재전송이 무엇을 막는지는 저장소마다 다르다(정본
|
||||
`docs/web/http-contract.md` 2절). Redis 저장소에서 같은 키 재전송은 효과가 있었을 수 있는 작업을 다시 돌리지
|
||||
않는다. PostgreSQL 저장소를 템플릿처럼 쓰기 트랜잭션 하나로 쓰면 실패한 요청의 기록은 롤백되어 다음 같은 키
|
||||
않는다. 단, 그 기록이 Redis에 남아 있는 동안만이다. Sentinel·Cluster failover는 비동기 복제라 primary가 받고
|
||||
replica에 아직 넘기지 못한 기록을 잃을 수 있고, 그러면 같은 키 요청은 처음 보는 키로 다시 실행된다
|
||||
(`docs/runbooks/redis-capability-incident.md` 「Topology and persistence」). 이 창이 업무상 받아들일 수 없는
|
||||
작업은 PostgreSQL 저장소를 쓰거나 업무 데이터의 유일 제약으로 따로 막는다. PostgreSQL 저장소를 템플릿처럼 쓰기 트랜잭션 하나로 쓰면 실패한 요청의 기록은 롤백되어 다음 같은 키
|
||||
요청이 다시 실행한다. 그래서 트랜잭션 밖 효과는 따로 멱등하게 만든다. 첫 시도가 processing lease(기본 30s)보다
|
||||
오래 걸릴 때 보이는 모습도 저장소마다 다르다.
|
||||
|
||||
|
||||
@@ -41,7 +41,10 @@ email을 모든 tenant에 걸쳐 유일하게 만든다. 그러면 tenant A가
|
||||
## 2. tenant마다 database (Experimental)
|
||||
|
||||
`MongoTenantClientRegistry`는 tenant마다 client를 하나 둔다. 개수 상한이 있고 쓰지 않는 client는
|
||||
내보낸다. tenant를 어느 database에 대응시킬지는 도입하는 쪽의 코드가 정한다.
|
||||
내보낸다. tenant를 어느 database에 대응시킬지는 도입하는 쪽의 코드가 정한다. `acquire`는 `Lease`를 돌려주고,
|
||||
그 client는 lease를 모두 닫을 때까지 쓰는 중이다. 쓰는 중인 client는 얼마나 오래전에 받았든 내보내지 않는다.
|
||||
내보낸 client를 닫다가 실패하면 그 tenant 대신 `evictionCloseFailures()`가 센다. 닫기를 부른 다른 tenant의
|
||||
`acquire`는 실패하지 않는다(round9 리뷰 2026-11-12, RR9-01-27).
|
||||
|
||||
Experimental인 이유가 분명하다. 작을 때는 맞고, 클 때는 한계가 없다. tenant database마다 connection,
|
||||
file handle, 모니터링 가짓수가 든다. tenant 20개에서는 잘 돌고 2,000개에서는 쓰러진다. 기능 테스트로는
|
||||
|
||||
@@ -39,9 +39,15 @@ regex는 index를 쓰는 척하는 collection 전체 scan이고, 사용자가
|
||||
| 상한 | 이유 |
|
||||
|---|---|
|
||||
| `maxTimeMS` | 아무도 기다리지 않는 query를 서버가 멈춘다. |
|
||||
| 결과 개수 상한 | 상한 없는 결과는 단계가 몇 개 더 붙은 OOM이다. |
|
||||
| batch 크기 | 왕복 한 번의 메모리를 제한한다. |
|
||||
| 검사한 문서 수 상한 | 빠른 날에는 시간 상한만으로 숨겨지는 index 회귀를 잡는다. |
|
||||
| 결과 개수 상한(`maxResults`) | 상한 없는 결과는 단계가 몇 개 더 붙은 OOM이다. |
|
||||
| 결과 바이트 상한(`maxResultBytes`) | 문서 하나하나가 크면 개수 상한 안에서도 heap이 찬다. |
|
||||
| batch 크기(`cursorBatchSize`) | 왕복 한 번의 메모리를 제한한다. |
|
||||
|
||||
개수·시간·batch는 서버에 보내는 요청에 실린다(`PolicyAwareMongoQueryBuilder.build`가 만든 `Query`, aggregation의
|
||||
`$limit`과 옵션). 바이트는 결과를 읽는 쪽만 셀 수 있다. aggregation은 `PolicyAwareMongoAggregationExecutor`가
|
||||
cursor에서 문서를 하나씩 읽으며 encode한 크기를 더하고 넘으면 거부한다(5절). query builder는 `Query`만 돌려주므로
|
||||
query의 바이트 상한을 집행하는 템플릿 코드는 없다. 그 `Query`로 결과를 읽는 서비스의 adapter가 `maxResultBytes`를
|
||||
직접 센다. 세지 않으면 그 상한은 선언일 뿐이다.
|
||||
|
||||
`MongoBudgetPolicyRegistry`가 예산을 연산 이름에 묶는다. `MongoBudgetEnforcer`가 그것을 적용한다. 요청이
|
||||
예산을 넘으면 실행 전에 `MongoOperationRejectedException`을 던지고, 서버가 상한을 집행하면
|
||||
@@ -86,6 +92,11 @@ regex는 index를 쓰는 척하는 collection 전체 scan이고, 사용자가
|
||||
| high | `$lookup`, `$graphLookup`, `$facet`, index 없는 `$sort` | profile의 명시적 승인이 필요하다. |
|
||||
| forbidden | admin plane 밖의 `$out`·`$merge`, `$function`, `$accumulator` | 거부한다. |
|
||||
|
||||
실행은 예산 둘을 결과가 오는 동안 본다. pipeline 끝에 `$limit(maxResults + 1)`을 붙여 서버가 예산보다 한 개 넘게는
|
||||
만들지 않게 하고, 결과를 cursor에서 한 문서씩 읽으며 개수와 encode한 바이트를 더한다. 둘 중 하나라도 넘으면
|
||||
`MongoOperationRejectedException`(`aggregation.result`)이다. 예전에는 결과를 전부 메모리에 올린 뒤 개수만 셌고 바이트
|
||||
상한은 선언만 있었다(round9 리뷰 2026-11-12, RR9-01-20).
|
||||
|
||||
`allowDiskUse`는 실행 중 flag가 아니라 plan에 선언하는 속성이다. 디스크가 필요한 pipeline은 모양을 검토해야
|
||||
하는 pipeline이다.
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ support leaf의 `ModuleDefaultsEnvironmentPostProcessor`가 가장 낮은 우선
|
||||
| 속성 | 환경 변수 | 기본값 | 범위 |
|
||||
|---|---|---|---|
|
||||
| `dispatch.claim-batch-size` | `APP_NOTIFICATION_PLATFORM_CLAIM_BATCH_SIZE` | `50` | 1..1000 |
|
||||
| `dispatch.lease-duration` | `APP_NOTIFICATION_PLATFORM_LEASE_DURATION` | `2m` | 양수, 유한 |
|
||||
| `dispatch.lease-duration` | `APP_NOTIFICATION_PLATFORM_LEASE_DURATION` | `2m` | 양수, 유한. `poll-interval`과 켜진 provider profile의 `timeout`보다 길어야 기동한다. SMTP profile은 `timeout`의 두 배보다 길어야 한다(「SMTP relay」) |
|
||||
| `dispatch.poll-interval` | `APP_NOTIFICATION_PLATFORM_POLL_INTERVAL` | `1s` | 양수, 유한 |
|
||||
| `dispatch.max-global-concurrency` | `APP_NOTIFICATION_PLATFORM_MAX_CONCURRENCY` | `64` | 양수 |
|
||||
| `dispatch.max-additional-attempts` | `APP_NOTIFICATION_PLATFORM_MAX_ADDITIONAL_ATTEMPTS` | `4` | 0 이상 |
|
||||
@@ -35,8 +35,11 @@ support leaf의 `ModuleDefaultsEnvironmentPostProcessor`가 가장 낮은 우선
|
||||
|
||||
모든 값에 상한이 있다. 「무제한」은 받는 설정이 아니다.
|
||||
|
||||
lease는 provider 호출과 그 시간 상한보다 길어야 한다. 그보다 짧으면, 살아 있는 worker가 아직 기다리는 전달을 두 번째
|
||||
worker가 claim하고, 수신자는 알림을 두 번 받는다.
|
||||
lease는 provider 호출 한 번이 delivery를 쥐는 시간보다 길어야 한다. 그보다 짧으면 아직 기다리는 호출이 멈춘 lease로
|
||||
보인다. lease가 끝난 뒤 lease만큼 더 지나도 호출이 끝나지 않았으면, lease 복구가 그 delivery를 버려진 것으로 보고 맞추기로
|
||||
넘긴다. 기본 제공 provider(SES·SMTP)는 결과 조회가 없어 운영자 확인(`RECONCILIATION_REQUIRED`)으로 넘어간다. 그 뒤에
|
||||
끝난 호출도 자기 결과를 기록한다(owner와 fence가 그대로다). 두 번째 worker가 다시 보내지는 않는다. claim은
|
||||
`DISPATCHING`인 delivery를 가져가지 않는다.
|
||||
|
||||
## 콜백
|
||||
|
||||
@@ -131,7 +134,7 @@ Spring이 이미 그것을 가지고 있고, 두 번째 표기는 맞춰 둘 것
|
||||
| `providers.smtp.primary-for-channel` | `APP_NOTIFICATION_PLATFORM_SMTP_PRIMARY` | `true` | boolean. 채널마다 primary는 정확히 하나 |
|
||||
| `providers.smtp.environment` | `APP_NOTIFICATION_PLATFORM_SMTP_ENVIRONMENT` | `local` | 켜면 필요하다 |
|
||||
| `providers.smtp.credential-profile` | `APP_NOTIFICATION_PLATFORM_SMTP_CREDENTIAL_PROFILE` | `default` | `SecretMaterialProvider`로 푼다. 값을 직접 적지 않는다 |
|
||||
| `providers.smtp.timeout` | `APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT` | `10s` | 양수, 유한 |
|
||||
| `providers.smtp.timeout` | `APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT` | `30s` | 양수, 유한. `mail.smtp.connectiontimeout`에 `mail.smtp.timeout`·`mail.smtp.writetimeout` 중 긴 것을 더한 값보다 길어야 기동한다 |
|
||||
| `providers.smtp.max-concurrency` | `APP_NOTIFICATION_PLATFORM_SMTP_MAX_CONCURRENCY` | `4` | 양수 |
|
||||
| `providers.smtp.rate-per-second` | `APP_NOTIFICATION_PLATFORM_SMTP_RATE_PER_SECOND` | `10` | 양수 |
|
||||
|
||||
@@ -140,11 +143,40 @@ Spring이 이미 그것을 가지고 있고, 두 번째 표기는 맞춰 둘 것
|
||||
| `smtp.sender-identity` | `APP_NOTIFICATION_PLATFORM_SMTP_SENDER_IDENTITY` | `no-reply@example.invalid` | 주소 |
|
||||
| `smtp.dispatch-concurrency` | `APP_NOTIFICATION_PLATFORM_SMTP_DISPATCH_CONCURRENCY` | `4` | 양수 |
|
||||
|
||||
`smtp.dispatch-concurrency`는 SMTP 발송 pool의 thread 수다. 첫 발송부터 그만큼 함께 돌고, 대기열은 그 네 배다. 모든 SMTP
|
||||
profile이 이 pool 하나를 같이 쓴다. 켜진 SMTP profile들의 `max-concurrency` 합이 thread와 대기열을 합친 것보다 크면 SMTP
|
||||
조립이 기동 때 실패한다. pool이 가득 차서 거절한 발송은 relay에 닿지 않은 것으로 기록되어 다시 시도된다(결과 불명이
|
||||
아니다).
|
||||
|
||||
SMTP 전송 설정의 소유자는 Spring Mail 하나다. 실제 `JavaMailSenderImpl`은 필수 STARTTLS
|
||||
(`mail.smtp.starttls.enable=true`와 `mail.smtp.starttls.required=true`)나 암묵적 TLS(`mail.smtp.ssl.enable=true`)를
|
||||
써야 한다. `mail.smtp.connectiontimeout`, `mail.smtp.timeout`, `mail.smtp.writetimeout`도 양수여야 한다. 이 속성이
|
||||
없거나 틀리면 SMTP 조립이 트래픽을 받기 전에 실패한다.
|
||||
|
||||
이 socket timeout은 SMTP 명령 하나씩을 묶을 뿐 대화 전체를 묶지 않는다. 발송 한 번 전체는 provider profile의 `timeout`이
|
||||
두 번 묶는다(2026-11-13).
|
||||
|
||||
- pool 대기: 발송이 send pool에서 `timeout`이 지나도록 시작하지 못하면 버려지고 relay에 가지 않는다. relay에 닿지 않은
|
||||
것(`PROVIDER_UNAVAILABLE`, `CAPACITY_REJECTED`)으로 기록되어 다시 시도된다.
|
||||
- 발송: pool thread가 잡은 때부터 `timeout`이 지나면 결과 불명(`PROVIDER_RESPONSE_LOST`)으로 기록되고 자동으로 다시 보내지
|
||||
않는다. DATA가 이미 넘어갔을 수 있기 때문이다.
|
||||
|
||||
그래서 한 발송이 delivery lease를 쓰는 시간은 많아야 `timeout`의 두 배다. 기동 때 두 가지를 본다.
|
||||
|
||||
- `dispatch.lease-duration`은 SMTP `timeout`의 두 배보다 길어야 한다. 모든 provider에 거는 조건(lease가 `timeout`보다
|
||||
길다)보다 엄격하다. 짧으면 생기는 일은 위 「발송」 절의 lease 문단과 같다. 기본값(lease 2분, timeout 30초)은 맞는다.
|
||||
- profile `timeout`은 connect timeout에 명령 timeout 하나를 더한 값보다 길어야 한다. 그보다 짧으면 인사말을 주지 않는
|
||||
relay처럼 아무것도 보내지 않은 실패가, socket timeout이 알려 주기 전에 결과 불명으로 기록된다. 기본값(30초, connect 5초 +
|
||||
명령 10초)은 맞는다.
|
||||
|
||||
`dispatch.shutdown-grace`(기본 20초)가 `timeout`의 두 배보다 짧으면, 종료하는 동안 끝나지 않은 발송은 attempt가
|
||||
`DISPATCHING`으로 남고, 위 「발송」 절의 lease 문단대로 운영자 확인으로 넘어간다(종료된 발송은 결과를 기록하지 못한다).
|
||||
느린 relay에서 이것 없이 끝내려면 pod 종료 유예와 함께 grace를 늘린다.
|
||||
|
||||
send pool(`notificationSmtpExecutor` bean)을 바꿔 끼우면 발송을 부른 thread에서 돌리지 않는 executor여야 한다
|
||||
(`CallerRunsPolicy`나 직접 실행 executor는 안 된다). 부른 thread에서 돌면 두 시한이 dispatch thread를 풀지 못한다.
|
||||
`ThreadPoolExecutor`가 아니면 위의 pool 합 검사도 하지 않는다.
|
||||
|
||||
환경 변수 계약은 Spring 그대로다. `SPRING_MAIL_HOST`, `SPRING_MAIL_PORT`, 선택인 `SPRING_MAIL_USERNAME`과 secret
|
||||
`SPRING_MAIL_PASSWORD`, 그리고 `.env.example`에 있는 `SPRING_MAIL_PROPERTIES_MAIL_SMTP_*` TLS·시간 상한 키다. Spring
|
||||
Boot 자신이 `spring.mail.*`를 가지고 읽으므로 이름을 Spring 그대로 둔다. 알림 모듈은 같은 relay에 두 번째 별칭을
|
||||
|
||||
@@ -2688,10 +2688,8 @@ env_keys:
|
||||
required_test: cache-contract:redis-semantic-region-bound
|
||||
|
||||
- name: APP_CACHE_REDIS_MAXIMUM_VALUE_BYTES
|
||||
# 폐기 2026-08-10: 이 속성을 바인딩하던 settings class를 옛 Redis 세대와 함께 지워서 이 키는 어디에도
|
||||
# 닿지 않는다. 아직 이 키를 쓰는 배포에 무엇이 대신하는지 알려 주려고 행을 지우지 않고 둔다.
|
||||
deprecated_orphaned: true
|
||||
removal_deadline: 2026-11-30
|
||||
# 2026-08-10에 옛 Redis 세대와 함께 폐기했다가, 2026-11-13 cache-redis 모듈 기본 설정이
|
||||
# app.capabilities.cache.maximum-value-bytes에 다시 묶었다. 상한을 넘는 값은 기록하지 않는다(round9 리뷰 RR9-01-18).
|
||||
type: int
|
||||
default: 1048576
|
||||
allowed_values: null
|
||||
@@ -4780,9 +4778,10 @@ env_keys:
|
||||
required_test: adapter-contract:notification-smtp-provider-assembled
|
||||
|
||||
- name: APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT
|
||||
# source: NTF-INT-001 — 이 profile의 시도마다 provider timeout.
|
||||
# source: NTF-INT-001 — 이 profile의 발송 한 번 상한: pool 대기와 발송이 각각 이 시간을 넘지 않는다.
|
||||
# 2026-11-13 10s에서 30s로 올렸다. 기동 때 mail.smtp.connectiontimeout + 명령 timeout보다 길어야 한다.
|
||||
type: duration
|
||||
default: 10s
|
||||
default: 30s
|
||||
allowed_values: null
|
||||
classification: public-config
|
||||
required: false
|
||||
@@ -5063,7 +5062,8 @@ env_keys:
|
||||
required_test: adapter-contract:notification-dispatch-bounds
|
||||
|
||||
- name: APP_NOTIFICATION_PLATFORM_LEASE_DURATION
|
||||
# source: NTF-025 — provider 호출과 그 timeout보다 길어야 한다. 아니면 살아 있는 worker의 전달을 다른 worker가 claim한다
|
||||
# source: NTF-025 — poll-interval과 켜진 provider profile의 timeout보다 길어야 기동한다(2026-11-13부터 검사).
|
||||
# 짧으면 살아 있는 worker의 전달을 다른 worker가 claim한다. 넉넉히 길게 둔다: lease는 claim 때 시작한다
|
||||
type: duration
|
||||
default: 2m
|
||||
allowed_values: null
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# 업그레이드 안내 — 2026-11 claim·fence·트랜잭션 시한·정리
|
||||
|
||||
- 범위: durable operation·알림 수신자·fileserver cleanup·멱등성 기록·live event·분산 락·stateless runner의 lease,
|
||||
fence, 조건부 쓰기, 정리, 시간 상한을 고친 변경. 포트 시그니처와 기동 조건이 바뀐 곳이 있다. 아래 표가 그 목록이다.
|
||||
- 근거: 2026-11-12 전체 리뷰(`docs/reviews/2026-11-12-round9-review/`)의 claim·fence(B8)와 트랜잭션 시한·정리(B9) 발견
|
||||
|
||||
## 1. 누가 읽나
|
||||
|
||||
이 템플릿을 `upstream` remote로 두고 merge해 받는 파생 서비스가 읽는다. 특히 durable operation worker를 직접 쓴 서비스,
|
||||
알림 recipient store를 직접 구현한 서비스, `app.lock.wait-time`이나 알림 provider timeout을 설정한 서비스, experimental
|
||||
tenant pool 예산을 쓰는 서비스다.
|
||||
|
||||
## 2. 할 일
|
||||
|
||||
| # | 할 일 | 안 하면 | 해당 서비스 |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | durable operation worker는 `claimNext`가 돌려주는 `DurableOperationClaim`을 들고 있다가 `heartbeat(claim, …)`·`reportProgress(claim, …)`·`succeed(claim, …)`·`fail(claim, …)`에 넘긴다. 예전에는 `(operationId, workerId, …)`였다 | 컴파일이 실패한다 | durable operation worker를 쓴 서비스 |
|
||||
| 2 | `DurableOperationStorePort`를 직접 구현했으면 claim마다 오르는 fence를 저장하고, 네 쓰기가 owner와 fence를 함께 확인하게 한다. `DurableOperationStoreContract`의 새 사례(같은 worker id가 다시 claim한 뒤 앞 실행의 쓰기는 거절)로 확인한다 | 재시작한 worker나 한 프로세스의 두 실행이 같은 id를 쓰면 앞 실행의 결과가 넘겨받은 실행 위에 기록된다 | 자기 durable operation store를 둔 서비스 |
|
||||
| 3 | persistence-jpa의 `V23__durable_operation_lease_fence.sql`(`durable_operation.lease_fence BIGINT NOT NULL DEFAULT 0`)이 기본 stream에 더해진다. 같은 번호의 서비스 마이그레이션이 있으면 번호를 옮긴다. `durable_operation`을 건드린 트랜잭션이 열려 있으면 5초 뒤 실패하고 V22에 남는다(다시 배포) | Flyway가 번호 충돌로 기동하지 않는다 | persistence-jpa를 쓰는 모든 서비스 |
|
||||
| 4 | 알림 stream에 `V14__notification_provider_event_open_index.sql`(부분 인덱스 `ix_notification_provider_event_open`)이 더해진다. `notification_provider_event`가 이미 큰 운영 DB는 배포 전에 같은 인덱스를 `CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_notification_provider_event_open ON notification_provider_event (received_at) WHERE projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL;`로 먼저 만든다. 미리 만든 인덱스가 실패해 남았으면(`SELECT indisvalid FROM pg_index WHERE indexrelid = 'ix_notification_provider_event_open'::regclass`가 false) `DROP INDEX CONCURRENTLY ix_notification_provider_event_open` 뒤 다시 만든다. `IF NOT EXISTS`는 쓸 수 없는 인덱스도 있는 것으로 보고 넘어간다 | 마이그레이션이 인덱스를 만드는 동안 provider callback 저장이 기다린다(provider가 callback을 다시 보낸다) | 알림 capability를 켠 서비스 |
|
||||
| 5 | `RecipientLeaseStorePort.expiredDispatching`이 `List<RecipientLease>`(버려질 때의 owner·fence·만료 시각)를 돌려주고, `RecipientDeliveryStorePort`에 `requeueAbandoned(lease, nextDispatchAt)`가 생겼다. 직접 구현했으면 상태가 `DISPATCHING`이고 owner·fence가 같고 lease가 늘지 않았을 때만 `READY_TO_DISPATCH`로 바꾼다(`RecipientDeliveryStoreContract` 새 사례). owner가 없거나 fence가 0인 `DISPATCHING` 행(notification V4 전에 남은 행)은 복구가 더는 고르지 않는다. 그런 행은 `SELECT id FROM notification_recipient_delivery WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 찾는다. 배포 전에 `UPDATE notification_recipient_delivery SET lease_owner = COALESCE(lease_owner, 'pre-v4'), lease_fence = lease_fence + 1, lease_until = now() - interval '1 day', version = version + 1, updated_at = now() WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 owner를 채우고 fence를 올려 복구 패스에 넘긴다. 복구는 attempt를 보고 정한다: attempt가 없으면 다시 보낼 차례로 돌리고, 끝나지 않은 attempt가 있으면 맞추기로 넘긴다(그 뒤는 `docs/notification/configuration-reference.md` 「발송」의 lease 문단). 손으로 `READY_TO_DISPATCH`로 바꾸지 않는다. 발송 중에 멈춘 행이면 같은 알림이 한 번 더 나간다. owner가 없는 행은 stuck lease 지표에도 잡히지 않는다 | 컴파일이 실패한다. 조건 없이 바꾸면 끝났거나 다시 claim된 delivery가 다시 발송된다 | 알림 store를 직접 구현한 서비스 |
|
||||
| 6 | 알림 provider profile의 `timeout`이 `app.notification.platform.dispatch.lease-duration`보다 짧은지 확인한다. 짧은 것은 필요조건일 뿐이다: lease는 claim 때 시작해서 attempt 기록·렌더링·provider 허가 대기도 lease를 쓴다. lease는 timeout보다 넉넉히 길게 둔다. 이 timeout은 SES에서는 API 호출 하나의 상한이다. SMTP의 추가 조건은 `docs/upgrade/2026-11-round9-outbound.md` 9행이다 | 짧지 않으면 기동이 실패한다(`notification provider profile '…': timeout (…) must be shorter than …lease-duration`). 전에는 provider 호출 하나가 lease보다 길어도 기동했다. 그때 생기는 일은 `docs/notification/configuration-reference.md` 「발송」의 lease 문단이다 | 알림 capability를 켠 서비스 |
|
||||
| 7 | `app.lock.wait-time`을 설정에서 지운다. 기다릴 시간은 `tryAcquire(key, waitTime, leaseTtl)`의 인자다 | `app.lock` 아래 모르는 키라 기동이 실패한다. 전에도 읽는 곳이 없었다 | `app.lock.wait-time`을 설정한 서비스 |
|
||||
| 8 | `JpaLiveEventReplay.sweepExpired()`를 부르던 코드를 지운다. replay를 켜면 `LiveEventReaper`가 `app.realtime.replay.reaper-interval`(기본 `PT10M`)마다 `retained_until`이 지난 행을 1000행씩 지운다. `app.realtime.replay.*`는 이제 `LiveEventReplaySettings`가 엄격하게 묶는다: 그 아래 모르는 키, 0 이하의 `retention`·`reaper-interval`은 기동을 실패시킨다 | 컴파일이 실패한다. 잘못 쓴 키는 기동이 실패한다. 전에는 지우는 곳이 없어 `live_event_log`가 끝없이 자랐다 | `app.realtime.replay.enabled=true`인 서비스 |
|
||||
| 9 | `IdempotencyReaper`를 직접 만들던 코드는 `new IdempotencyReaper(repository)`로 바꾼다(`Clock` 인자 삭제). 만료 비교는 DB 시계(`now()`)이고 1000행씩 지운다 | 컴파일이 실패한다 | reaper를 직접 조립한 서비스 |
|
||||
| 10 | `HibernateStatelessSessionRunner`의 트랜잭션은 이제 30초 상한이 있다(`DEFAULT_TIMEOUT`). 더 긴 일이면 셋째 인자로 상한을 넘기거나 일을 나눈다 | 30초를 넘는 문장이 취소되고 일 전체가 롤백된다. 전에는 상한이 없었다 | stateless runner를 쓰는 서비스 |
|
||||
| 11 | experimental `TenantPoolBudget.requireCapacity`·`hasCapacity`가 새 pool의 연결 수를 셋째 인자로 받는다. pool 수만 볼 때는 `requirePoolSlot(openPools)`를 쓴다. `TenantDataSourceRegistry`의 둘째 인자는 이제 열린 pool의 크기를 재는 함수가 아니라 tenant의 계획된 pool 크기(`ToIntFunction<TenantId>`)다. 예산은 pool을 열기 전에 그 크기로 본다 | 컴파일이 실패한다. 전에는 새 pool 크기를 모른 채 열어 연결 합계가 상한을 넘었다 | experimental tenant pool을 쓰는 서비스 |
|
||||
| 12 | fileserver cleanup 묶음의 `max-items`를 정할 때, 한 묶음이 claim lease(10분)의 첫 8분 안에 시작할 수 있는 만큼만 처리된다는 것을 안다. 나머지는 시도하지 않은 채 queue에 돌아가 다음 묶음이 다시 claim한다. 시도 횟수는 그대로다(바이트 예산이 바닥나 돌려준 항목도 이제 같다. 전에는 시도 한 번으로 세어 8번째에 보류됐다). 직접 구현한 `CleanupQueuePort`는 새 연산 `release`를 구현한다(`CleanupQueuePortContract` 새 사례) | 느린 저장소에서 큰 `max-items`는 그 사유로 도는 항목이 늘어난다. 전에는 lease가 끝난 뒤에도 남은 항목을 처리해 다른 worker와 겹쳤다 | fileserver capability를 켠 서비스 |
|
||||
|
||||
## 3. 바뀐 동작 (따라 할 일 없음)
|
||||
|
||||
- reaper 셋(outbox·멱등성·live event)의 트랜잭션은 `SpringTransaction.DEFAULT_TIMEOUT_SECONDS`(30초)로 묶인다.
|
||||
`app.persistence-jpa.transaction.*`를 바꿔도 reaper 시한은 그대로다. 한 트랜잭션이 1000행 한 묶음만 지운다.
|
||||
- 저장소가 행에서 만드는 `DurableOperationClaim`·`OperationLease`·`RecipientLease`·`OutboxLease`·`ClaimedOutboxEvent`·
|
||||
`ReconciliationJob`·`AdminOperationClaim`과 stateless runner의 timeout이 값을 거절할 때 `IllegalStateException`을 던진다
|
||||
(전에는 `IllegalArgumentException`). 요청에서 오는 값이 아니라 서버 결함이다. 이 IAE를 잡던 코드는 ISE를 잡는다.
|
||||
- 알림 lease 회복은 버려질 때의 lease 그대로일 때만 delivery를 다시 넣는다. 다른 인스턴스가 이미 다시 넣었거나 holder가
|
||||
끝냈으면 아무것도 쓰지 않고 처리 수에 세지 않는다.
|
||||
- 알림 만료 sweep(`EXPIRE_OVERDUE`)은 다른 트랜잭션이 쥔 행을 건너뛰고, 바깥 `UPDATE`에서도 상태·만료를 다시 본다.
|
||||
inbox 전체 읽음(`markAllRead`)도 바깥 `UPDATE`에서 상태를 다시 본다. 기다리는 사이 claim된 delivery나 보관된 inbox 항목을
|
||||
바꾸지 않는다.
|
||||
- 멱등성 reaper는 다른 트랜잭션이 쥔 기록을 건너뛰고, 바깥 `DELETE`에서도 만료를 다시 본다. 같은 키 요청이 만료된 기록을
|
||||
되살리는 동안 돈 reap이 그 claim을 지우지 않는다.
|
||||
- fileserver recovery 대기열 enqueue는 `INSERT … ON CONFLICT` 한 문장이다. 같은 파일을 동시에 넣어도 25P02로 실패하지 않는다.
|
||||
- 알림 scheduler는 poll마다 backlog 한 문장만 읽는다(전에는 넷). health snapshot의 projection 지연은 열린 provider event만
|
||||
읽는다.
|
||||
- 알림 backlog(`backlogDepth`, scheduler의 queue-depth gauge, health 임계값)는 claim이 지금 가져갈 delivery만 센다. claim과
|
||||
같은 SQL 조각이다. 전에는 창이 닫힌(`expires_at`이 지난) delivery도 셌고, lease가 지금 막 끝나는 행(`lease_until = now`)도
|
||||
셌다. 만료가 쌓인 플랫폼은 배포 직후 backlog 수치가 내려간다. 보낼 일이 줄어든 것이 아니라 보낼 수 없는 행을 빼고 센 것이다.
|
||||
@@ -0,0 +1,41 @@
|
||||
# 업그레이드 안내 — 2026-11 outbound 발행·발송·캐시·S3·MongoDB·HTTP client
|
||||
|
||||
- 범위: outbox 발행 대기, SMTP 발송 pool, Redis 캐시 시간·크기 상한, S3 조건부 쓰기, MongoDB 집계 예산·tenant
|
||||
client·migration lease, HTTP client 압축·proxy를 고친 변경. 기동 조건과 공개 타입이 바뀐 곳이 있다. 아래 표가 그
|
||||
목록이다.
|
||||
- 근거: 2026-11-12 전체 리뷰(`docs/reviews/2026-11-12-round9-review/`)의 발행·발송·캐시(B10a)와 S3·Mongo·httpclient
|
||||
(B10b) 발견
|
||||
|
||||
## 1. 누가 읽나
|
||||
|
||||
이 템플릿을 `upstream` remote로 두고 merge해 받는 파생 서비스가 읽는다. 특히 SMTP 알림, Redis 캐시, MongoDB
|
||||
집계·tenant별 database, HTTP client의 압축이나 proxy를 쓰는 서비스다.
|
||||
|
||||
## 2. 할 일
|
||||
|
||||
| # | 할 일 | 안 하면 | 해당 서비스 |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | SMTP provider profile들의 `max-concurrency` 합이 발송 pool이 담는 수(`app.notification.platform.smtp.dispatch-concurrency`개 thread와 그 네 배의 대기열) 이하인지 확인한다. 모든 SMTP profile이 pool 하나를 같이 쓴다. 기본값(profile 하나, 4와 4)은 맞는다 | 넘으면 넘게 만든 profile의 조립이 기동 때 실패한다. 전에는 넘친 발송이 대기열에서 거절되어 결과 불명으로 남았다 | SMTP 알림을 켠 서비스 |
|
||||
| 2 | Redis 캐시의 `command-timeout`이 이제 한 캐시 연산 전체(연결 대기와 응답 전부)의 상한이다. 연결 대기가 길던 배포는 값을 다시 본다 | 상한 안에 못 끝난 연산이 캐시 없음(`DEGRADED_UNAVAILABLE`·`Unavailable`)으로 끝나는 일이 늘 수 있다. 전에는 lane 대기 2초가 더해졌다 | Redis 캐시를 켠 서비스 |
|
||||
| 3 | encode한 값이 1 MiB(`RedisCacheRegionAdapter.DEFAULT_MAXIMUM_VALUE_BYTES`)를 넘는 캐시는 그 값을 기록하지 않는다(`NOT_RECORDED_PROVIDER_POLICY`). 더 큰 값이 필요하면 `app.capabilities.cache.maximum-value-bytes`(`APP_CACHE_REDIS_MAXIMUM_VALUE_BYTES`, 1..16777216)로 상한을 올린다. 2026-08-10에 폐기했던 그 env 키가 다시 묶였다 | 그 값은 매번 source에서 읽힌다 | 큰 값을 캐시하던 서비스 |
|
||||
| 4 | MongoDB `PolicyAwareMongoAggregationExecutor`는 등록 예산의 `maxResultBytes`를 집행하고, pipeline 끝에 `$limit(maxResults + 1)`을 붙인다. 결과가 큰 집계의 예산을 다시 본다. 집계 결과는 template의 converter로만 바뀌어 `AfterLoadEvent`·`AfterConvertEvent`와 entity callback이 돌지 않는다. 그것에 기대던 listener는 돌려받은 타입에서 직접 한다 | 바이트 예산을 넘는 집계가 `MongoOperationRejectedException`(`aggregation.result`)으로 실패한다. 전에는 결과를 전부 메모리에 올렸다 | MongoDB 집계를 쓰는 서비스 |
|
||||
| 5 | experimental `MongoTenantClientRegistry.acquire`가 `Lease`(interface)를 돌려준다. client를 다 쓴 뒤 `lease.close()`(또는 try-with-resources)를 부른다. 쓰는 중인 tenant의 `release`는 거부된다. idle 시간은 마지막 lease를 닫은 때부터 잰다. 열린 lease 수는 `openLeases()`다. 시각을 인자로 받던 `acquire(tenant, Instant)`는 없어졌다. 시계는 생성자의 `Clock` 하나다 | 반환값을 버리는 호출은 컴파일되지만, 닫지 않은 lease의 client는 쓰는 중으로 남아 내보내지지 않고 cap 한 칸을 계속 차지한다. `acquire(tenant, Instant)`를 부른 코드는 컴파일이 실패한다 | tenant별 database를 쓰는 서비스 |
|
||||
| 6 | MongoDB migration lease를 잃은 뒤의 `refresh`는 `MongoOperationRejectedException`(`migration.lock`)을 던진다. `IllegalStateException`을 잡던 코드를 바꾼다. collection을 직접 넘겨 `MongoCollectionMigrationLedger`를 만들던 코드는 셋째 인자로 lock collection(`MongoCollectionMigrationLock.DEFAULT_COLLECTION`)을 넘긴다. 옛 runner의 `recordApplied`는 새 runner가 lease를 잡은 뒤면(checkpoint를 쓰기 전이어도) 거부된다 | 그 예외를 잡지 못한다. 두 인자 생성자는 컴파일이 실패한다 | MongoDB migration runner를 직접 쓰는 서비스 |
|
||||
| 7 | HTTP client profile의 `proxy.type: SOCKS`를 지우거나 HTTP proxy로 바꾼다. `ProxyType`에는 이제 `HTTP`만 있다 | 설정 바인딩이 실패해 기동하지 않는다. 전에는 SOCKS port에 HTTP proxy로 붙어 첫 요청에서 실패했다 | SOCKS proxy를 설정한 서비스 |
|
||||
| 8 | `request.compression: true`인 HTTP client는 transport와 상관없이 `Accept-Encoding: gzip, deflate`를 보내고, 버퍼 본문과 스트리밍 다운로드 모두 이 플랫폼이 푼다(JDK transport는 전에 이 설정을 읽지 않았다). 압축을 켰든 껐든, 그 밖의 `Content-Encoding`(`br` 등)이나 겹친 coding으로 온 응답은 `HttpTargetRejectedException`(`TARGET_REJECTED`, 증거 `RESPONSE_RECEIVED`)으로 거부된다. 본문이 빈 gzip 응답(HEAD·204·304)은 빈 본문이다 | — 동작 변경. `response.max-wire-bytes`가 이제 압축된 크기로 센다. 스트리밍 다운로드도 identity 본문까지 `max-decoded-bytes`로 센다. 실험 transport인 Jetty HTTP/3도 이제 스스로 풀지 않는다 | HTTP client를 쓰는 서비스 |
|
||||
| 9 | SMTP provider profile의 `timeout`(기본이 `10s`에서 `30s`로 바뀌었다)이 발송 한 번을 두 번 묶는다. 발송이 send pool에서 그 시간이 지나도록 시작하지 못하면 relay에 보내지 않고 시작 전 실패(`PROVIDER_UNAVAILABLE`, 재시도)로 기록한다. 시작한 뒤 그 시간이 지나면 결과 불명(`PROVIDER_RESPONSE_LOST`)으로 기록하고 자동으로 다시 보내지 않는다. 값은 `mail.smtp.connectiontimeout`에 `mail.smtp.timeout`·`mail.smtp.writetimeout` 중 긴 것을 더한 값보다 길어야 하고, `dispatch.lease-duration`은 그 두 배보다 길어야 한다. `dispatch.shutdown-grace`가 그 두 배보다 짧으면 종료 중에 끝나지 않은 발송이 결과 불명으로 남는다(`docs/notification/configuration-reference.md` 「SMTP relay」). `SmtpNotificationProviderAdapter`를 직접 만들던 코드는 마지막 인자로 이 timeout을 넘긴다. 직접 만든 `ProviderRuntimeAssembler`는 새 default 메서드 `requireTogether(profiles, deliveryLease)`로 한 family의 켜진 profile 전체에 걸린 검사를 할 수 있다(`NotificationProviderAssembly`가 SERVING일 때 family마다 한 번 부른다). `assemble`을 직접 부르는 코드는 그 전에 `requireTogether`를 부른다 | timeout이 socket timeout 합 이하이거나 lease가 timeout 두 배 이하면 SMTP 조립이 기동 때 실패한다. 상한 안에 끝나지 않는 발송이 결과 불명으로 남는다. 전에는 socket timeout이 명령마다만 묶어, 느린 relay에서 발송이 lease를 넘으면 lease 복구가 아직 도는 발송을 운영자 확인으로 넘겼다. | SMTP 알림을 켠 서비스 |
|
||||
|
||||
## 3. 바뀐 동작 (따라 할 일 없음)
|
||||
|
||||
- outbox relay의 발행 대기는 `publish()`를 부른 때부터 publisher의 `longestAttempt()`까지다. 넘으면 결과 불명(`PUBLISH_WAIT_EXCEEDED`)으로 기록하고
|
||||
다음 claim에서 다시 본다. 직접 만든 `IntegrationEventPublishPort`는 `longestAttempt()`를 실제 최장 시간으로 답한다. 0이나
|
||||
음수를 답하면 그 행은 보내지 않고 `REJECTED_RETRYABLE`(`PUBLISH_BOUND_NOT_POSITIVE`)로 backoff 뒤에 다시 본다.
|
||||
- 이미 interrupt된 relay thread는 그 행을 발행하지 않고 `REJECTED_RETRYABLE`(`PUBLISH_INTERRUPTED_BEFORE_SEND`)로 돌려준다.
|
||||
relay가 그 행을 backoff 뒤로 미뤘다가 발행한다. 기다리는 중의 interrupt는 전처럼 결과 불명(`PUBLISH_WAIT_INTERRUPTED`)이다.
|
||||
- SMTP 발송 pool은 첫 발송부터 `dispatch-concurrency`개 thread로 돈다(전에는 대기열이 찰 때까지 thread 하나였다).
|
||||
pool이 거절한 발송은 relay에 닿지 않은 것(`PROVIDER_UNAVAILABLE`, `CAPACITY_REJECTED`)으로 기록되어 재시도된다.
|
||||
- S3 control 저장소의 조건부 `PUT`이 412를 받으면 key를 다시 읽는다. 이 호출이 쓴 record 그대로면 성공이다(SDK가 응답을
|
||||
잃은 첫 쓰기를 다시 보낸 경우). 다른 내용이나 없음이면 전처럼 충돌이다. 「그대로」는 같은 값이라는 뜻이라, 두 writer가
|
||||
같은 버전에서 같은 record를 쓰면 둘 다 성공으로 본다. 그럴 수 있는 호출자는 record에 자기 토큰을 싣는다.
|
||||
- MongoDB tenant client를 내보내다 닫기에 실패하면 그 tenant 대신 `evictionCloseFailures()`가 센다. 닫기를 부른 다른
|
||||
tenant의 `acquire`는 실패하지 않는다.
|
||||
@@ -125,7 +125,9 @@ Envelope가 아닌 응답은 두 곳뿐이다. fileserver의 프로토콜 예외
|
||||
`IDEMPOTENT_IN_FLIGHT`, lease가 지났으면 500 `IDEMPOTENCY_RECOVERY_REQUIRED`다. 이것은 Redis 저장소의
|
||||
판정이다. PostgreSQL 저장소는 아래처럼 첫 요청이 끝날 때까지 기다린 뒤 판정한다. 새 키로 보내면 서버는
|
||||
다른 요청으로 보고 막지 못한다(재리뷰 2026-11-03 README 4절 결정 RR-02-03, 2026-11-04 9.1절 결정
|
||||
RR-01-11).
|
||||
RR-01-11). Redis 저장소의 판정은 그 기록이 Redis에 남아 있는 동안만이다. Sentinel·Cluster failover는 비동기
|
||||
복제라 replica에 아직 넘기지 못한 기록을 잃을 수 있고, 그러면 같은 키 요청은 처음 보는 키로 다시 실행된다
|
||||
(`docs/manual/05-idempotency.md` 「클라이언트에게 무엇을 약속하나」).
|
||||
- **첫 시도가 lease보다 오래 걸릴 때 — 저장소마다 다르다.** executor는 lease를 늘리지 않으므로 30s를 넘는
|
||||
정상 요청도 lease가 지난다(재리뷰 2026-11-04 RR5-01-05).
|
||||
- Redis: lease가 지난 실행 기록을 다음 요청이 만나면 500으로 답하고 기록은 건드리지 않는다. 첫 시도가
|
||||
|
||||
@@ -132,7 +132,12 @@ credential이 깨지면 regular lane PING이 UP을 알리게 두지 않고 pod
|
||||
- 부정 항목은 더 짧은 부정 TTL을 쓴다;
|
||||
- Redis 물리 TTL은 hard 만료다;
|
||||
- cache 장애는 애플리케이션의 타입 있는 unavailable·degraded 결과로 떨어진다;
|
||||
- 식별자는 keyspace에 들어가기 전에 키 있는 digest를 거친다.
|
||||
- 식별자는 keyspace에 들어가기 전에 키 있는 digest를 거친다;
|
||||
- `app.capabilities.cache.command-timeout`은 한 cache 연산 전체(lane 연결 대기와 응답 전부)의 상한이다. rate limit과
|
||||
같은 방식이다. 2026-11-13 전에는 lane 대기 2초가 그 위에 더해졌다. 상한 안에 못 끝난 연산은 `DEGRADED_UNAVAILABLE`이나
|
||||
`Unavailable`이다;
|
||||
- encode한 값이 `app.capabilities.cache.maximum-value-bytes`(`APP_CACHE_REDIS_MAXIMUM_VALUE_BYTES`, 기본 1 MiB,
|
||||
1..16777216)를 넘으면 기록하지 않는다(`NOT_RECORDED_PROVIDER_POLICY`, 보낸 것 없음). 그 값은 매번 source에서 읽힌다.
|
||||
|
||||
### 조건부 바꾸기
|
||||
|
||||
|
||||
+24
@@ -116,6 +116,30 @@ public final class SemanticRedisCall {
|
||||
return call(operation, null, body, notSent, unknown, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a call that decides for itself when its effect was handed to the driver, and must finish
|
||||
* within {@code budget}.
|
||||
*
|
||||
* <p>The budget bounds the wait for a lane connection and every awaited reply together, as {@link
|
||||
* #effectWithin} does. Without it each await gets the frame's timeout of its own and the lane
|
||||
* wait comes on top, so a caller that called its timeout the ceiling of one operation waited the
|
||||
* lane's acquire timeout plus one timeout per reply (round9 review 2026-11-12, RR9-01-17).
|
||||
*
|
||||
* @param operation the semantic operation, for the failure log
|
||||
* @param budget the ceiling on the whole call
|
||||
* @param body what to send, and how to read the reply; calls {@link Session#markSent()} before
|
||||
* the command whose loss would leave the outcome unknown
|
||||
* @param notSent the answer when nothing can have happened
|
||||
* @param unknown the answer when the command may have run
|
||||
* @param <R> the answer type
|
||||
* @return the body's answer, or one of the two failure answers
|
||||
*/
|
||||
public <R> R runWithin(
|
||||
String operation, Duration budget, Body<R> body, Supplier<R> notSent, Supplier<R> unknown) {
|
||||
Objects.requireNonNull(budget, "budget must be non-null");
|
||||
return call(operation, budget, body, notSent, unknown, false);
|
||||
}
|
||||
|
||||
private <R> R call(
|
||||
String operation,
|
||||
Duration budget,
|
||||
|
||||
+67
-6
@@ -91,6 +91,20 @@ public final class RedisCacheRegionAdapter<K, V> implements CacheRegionPort<K, V
|
||||
|
||||
private final Duration negativeTtl;
|
||||
|
||||
private final Duration commandTimeout;
|
||||
|
||||
private final int maximumValueBytes;
|
||||
|
||||
/**
|
||||
* The largest encoded value one entry carries by default: 1 MiB, the default the old {@code
|
||||
* APP_CACHE_REDIS_MAXIMUM_VALUE_BYTES} had before that key stopped reaching anything.
|
||||
*
|
||||
* <p>Without a ceiling a single oversized value — a list that grew, a payload nobody sized — was
|
||||
* written whole: one {@code SET} that holds the connection and the server's single thread while
|
||||
* it is copied, then a key every reader fetches whole (round9 review 2026-11-12, RR9-01-18).
|
||||
*/
|
||||
public static final int DEFAULT_MAXIMUM_VALUE_BYTES = 1_048_576;
|
||||
|
||||
private final CacheScripts scripts = new CacheScripts();
|
||||
|
||||
private static final String ABSENT = "absent";
|
||||
@@ -115,7 +129,8 @@ public final class RedisCacheRegionAdapter<K, V> implements CacheRegionPort<K, V
|
||||
* @param softTtl how long an entry stays fresh
|
||||
* @param hardTtl how long an entry stays usable
|
||||
* @param negativeTtl how long an authoritative absence is cached
|
||||
* @param commandTimeout the ceiling on one cache operation
|
||||
* @param commandTimeout the ceiling on one cache operation, the lane wait and every reply
|
||||
* together
|
||||
*/
|
||||
public RedisCacheRegionAdapter(
|
||||
RedisRuntimeOwner owner,
|
||||
@@ -128,6 +143,38 @@ public final class RedisCacheRegionAdapter<K, V> implements CacheRegionPort<K, V
|
||||
Duration hardTtl,
|
||||
Duration negativeTtl,
|
||||
Duration commandTimeout) {
|
||||
this(
|
||||
owner,
|
||||
keys,
|
||||
keyDigest,
|
||||
encoder,
|
||||
decoder,
|
||||
clock,
|
||||
softTtl,
|
||||
hardTtl,
|
||||
negativeTtl,
|
||||
commandTimeout,
|
||||
DEFAULT_MAXIMUM_VALUE_BYTES);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the adapter with its own ceiling on an encoded value.
|
||||
*
|
||||
* @param maximumValueBytes the largest encoded value one entry may carry; a larger one is not
|
||||
* recorded ({@link CacheRecordOutcome#NOT_RECORDED_PROVIDER_POLICY})
|
||||
*/
|
||||
public RedisCacheRegionAdapter(
|
||||
RedisRuntimeOwner owner,
|
||||
CacheKeys keys,
|
||||
Function<K, String> keyDigest,
|
||||
Function<V, byte[]> encoder,
|
||||
Function<byte[], V> decoder,
|
||||
Clock clock,
|
||||
Duration softTtl,
|
||||
Duration hardTtl,
|
||||
Duration negativeTtl,
|
||||
Duration commandTimeout,
|
||||
int maximumValueBytes) {
|
||||
this.call =
|
||||
new SemanticRedisCall(
|
||||
owner,
|
||||
@@ -144,6 +191,11 @@ public final class RedisCacheRegionAdapter<K, V> implements CacheRegionPort<K, V
|
||||
this.softTtl = Objects.requireNonNull(softTtl, "soft TTL must be non-null");
|
||||
this.hardTtl = Objects.requireNonNull(hardTtl, "hard TTL must be non-null");
|
||||
this.negativeTtl = Objects.requireNonNull(negativeTtl, "negative TTL must be non-null");
|
||||
this.commandTimeout = commandTimeout;
|
||||
if (maximumValueBytes < 1) {
|
||||
throw new IllegalStateException("the maximum value size must be at least one byte");
|
||||
}
|
||||
this.maximumValueBytes = maximumValueBytes;
|
||||
if (softTtl.compareTo(hardTtl) > 0) {
|
||||
throw new IllegalArgumentException(
|
||||
"the soft TTL must not exceed the hard TTL: an entry cannot stop being fresh after it has"
|
||||
@@ -158,8 +210,9 @@ public final class RedisCacheRegionAdapter<K, V> implements CacheRegionPort<K, V
|
||||
// A read changed nothing, whether or not it reached the server: degraded, not broken. The
|
||||
// caller
|
||||
// loads from the source and carries on.
|
||||
return call.run(
|
||||
return call.runWithin(
|
||||
"lookup",
|
||||
commandTimeout,
|
||||
session -> {
|
||||
resolveGeneration(session, now);
|
||||
byte[] stored =
|
||||
@@ -233,7 +286,12 @@ public final class RedisCacheRegionAdapter<K, V> implements CacheRegionPort<K, V
|
||||
@Override
|
||||
public CacheRecordOutcome record(K key, V value, CacheRecordMetadata metadata) {
|
||||
Objects.requireNonNull(value, "value must be non-null");
|
||||
return write(key, encoder.apply(value), "", metadata, hardTtl);
|
||||
byte[] payload = encoder.apply(value);
|
||||
if (payload.length > maximumValueBytes) {
|
||||
// Refused before any round trip; the caller serves the value it loaded and carries on.
|
||||
return CacheRecordOutcome.NOT_RECORDED_PROVIDER_POLICY;
|
||||
}
|
||||
return write(key, payload, "", metadata, hardTtl);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -268,8 +326,9 @@ public final class RedisCacheRegionAdapter<K, V> implements CacheRegionPort<K, V
|
||||
// The write is marked sent only when the write command itself is handed to the driver. A
|
||||
// failure before that point — lane refusal, connection, the generation read — wrote nothing.
|
||||
return (scripted ? scriptCall : call)
|
||||
.run(
|
||||
.runWithin(
|
||||
"record",
|
||||
commandTimeout,
|
||||
session -> {
|
||||
resolveGeneration(session, now);
|
||||
CacheEnvelope envelope =
|
||||
@@ -343,8 +402,9 @@ public final class RedisCacheRegionAdapter<K, V> implements CacheRegionPort<K, V
|
||||
@Override
|
||||
public CacheInvalidationOutcome invalidate(K key) {
|
||||
Objects.requireNonNull(key, "key must be non-null");
|
||||
return call.run(
|
||||
return call.runWithin(
|
||||
"invalidate",
|
||||
commandTimeout,
|
||||
session -> {
|
||||
byte[] physicalKey = keys.entryKey(keyDigest.apply(key));
|
||||
// A marker, not a delete. A delete leaves the key absent, and a source load that read the
|
||||
@@ -371,8 +431,9 @@ public final class RedisCacheRegionAdapter<K, V> implements CacheRegionPort<K, V
|
||||
// a server that is answering everything else at the same time, and the SDK blocks KEYS for
|
||||
// exactly that reason. Bumping the generation makes every older entry fail the generation
|
||||
// check on read and expire on its own schedule.
|
||||
return call.run(
|
||||
return call.runWithin(
|
||||
"invalidate-region",
|
||||
commandTimeout,
|
||||
session -> {
|
||||
session.markSent();
|
||||
Long generation = session.await(session.commands().incrementBy(keys.generationKey(), 1));
|
||||
|
||||
+2
-1
@@ -56,7 +56,8 @@ class RedisCacheConfig {
|
||||
cache.getPositiveSoftTtl(),
|
||||
cache.getPositiveHardTtl(),
|
||||
cache.getNegativeTtl(),
|
||||
cache.getCommandTimeout());
|
||||
cache.getCommandTimeout(),
|
||||
cache.getMaximumValueBytes());
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+26
@@ -1,5 +1,6 @@
|
||||
package dev.caskeleton.adapter.outbound.cache.redis.capability;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.cache.redis.cache.RedisCacheRegionAdapter;
|
||||
import dev.caskeleton.adapter.outbound.support.CapabilitySelectors;
|
||||
import dev.caskeleton.adapter.outbound.support.CapabilityStartupValidationException;
|
||||
import dev.caskeleton.adapter.outbound.support.StrictSettingsBinder;
|
||||
@@ -87,6 +88,12 @@ public class RedisCapabilitySettings {
|
||||
private Duration negativeTtl = Duration.ofSeconds(10);
|
||||
private Duration minimumHardTtl = Duration.ofSeconds(1);
|
||||
private int maximumScriptConnections = 4;
|
||||
private int maximumValueBytes = RedisCacheRegionAdapter.DEFAULT_MAXIMUM_VALUE_BYTES;
|
||||
|
||||
/**
|
||||
* The largest ceiling a deployment may set: sixteen mebibytes, as the env registry bounds it.
|
||||
*/
|
||||
static final int MAXIMUM_VALUE_BYTES_CEILING = 16_777_216;
|
||||
|
||||
/**
|
||||
* Reports whether the default binding selected Redis.
|
||||
@@ -124,6 +131,10 @@ public class RedisCapabilitySettings {
|
||||
throw new CapabilityStartupValidationException(
|
||||
"the cache script lane needs at least one connection");
|
||||
}
|
||||
if (maximumValueBytes < 1 || maximumValueBytes > MAXIMUM_VALUE_BYTES_CEILING) {
|
||||
throw new CapabilityStartupValidationException(
|
||||
"the cache value ceiling must be 1.." + MAXIMUM_VALUE_BYTES_CEILING + " bytes");
|
||||
}
|
||||
}
|
||||
|
||||
public Map<String, String> getBindings() {
|
||||
@@ -214,6 +225,21 @@ public class RedisCapabilitySettings {
|
||||
public void setMaximumScriptConnections(int maximumScriptConnections) {
|
||||
this.maximumScriptConnections = maximumScriptConnections;
|
||||
}
|
||||
|
||||
/**
|
||||
* The largest value the region records, in encoded bytes. A larger one is not written and the
|
||||
* read path falls back to the source, so one oversized entry cannot fill the instance or hold a
|
||||
* connection for the whole transfer.
|
||||
*
|
||||
* @return the ceiling, 1 to 16 MiB
|
||||
*/
|
||||
public int getMaximumValueBytes() {
|
||||
return maximumValueBytes;
|
||||
}
|
||||
|
||||
public void setMaximumValueBytes(int maximumValueBytes) {
|
||||
this.maximumValueBytes = maximumValueBytes;
|
||||
}
|
||||
}
|
||||
|
||||
/** Edge rate-limit provider composition. */
|
||||
|
||||
+3
@@ -43,6 +43,9 @@ app:
|
||||
# limiting and leases share (app.redis.capacity.maximum-script-connections). A cache under
|
||||
# load runs out of these and degrades; it cannot take the correctness features' connections.
|
||||
maximum-script-connections: ${APP_CACHE_REDIS_MAXIMUM_SCRIPT_CONNECTIONS:4}
|
||||
# The largest value the region records, in encoded bytes (1..16777216). A larger one is not
|
||||
# written; the read falls back to the source.
|
||||
maximum-value-bytes: ${APP_CACHE_REDIS_MAXIMUM_VALUE_BYTES:1048576}
|
||||
idempotency:
|
||||
# Redis store tuning only; the provider selector and lifecycle policy stay with the
|
||||
# application's idempotency executor.
|
||||
|
||||
+122
-5
@@ -1,12 +1,14 @@
|
||||
package dev.caskeleton.adapter.outbound.cache.redis.cache;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.cache.redis.runtime.RedisDeploymentMode;
|
||||
import dev.caskeleton.adapter.outbound.cache.redis.runtime.command.InMemorySemanticRedisCommands;
|
||||
import dev.caskeleton.adapter.outbound.cache.redis.runtime.command.SemanticRedisCommands;
|
||||
import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisConnectionKind;
|
||||
import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLane;
|
||||
import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLease;
|
||||
import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisRuntimeClient;
|
||||
import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisRuntimeOwner;
|
||||
import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.StubRedisRuntimeClient;
|
||||
@@ -23,7 +25,10 @@ import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -56,7 +61,7 @@ class RedisCacheRegionAdapterTest {
|
||||
|
||||
private RedisCacheRegionAdapter<String, String> cache(
|
||||
Instant now, RedisRuntimeClient client, RedisCacheRegionAdapter.CacheKeys cacheKeys) {
|
||||
Map<RedisLane, Integer> limits = new java.util.LinkedHashMap<>();
|
||||
Map<RedisLane, Integer> limits = new LinkedHashMap<>();
|
||||
for (RedisConnectionKind kind : RedisConnectionKind.values()) {
|
||||
limits.put(kind, 4);
|
||||
}
|
||||
@@ -82,6 +87,119 @@ class RedisCacheRegionAdapterTest {
|
||||
Duration.ofSeconds(2));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aLookupOnAFullLaneGivesUpAtTheCommandTimeoutNotTheLaneWait() {
|
||||
// given
|
||||
// One connection, already borrowed; a borrow would wait five seconds for it. The cache's
|
||||
// command timeout is 100 ms and is the ceiling on the whole operation, lane wait included.
|
||||
Map<RedisLane, Integer> limits = new LinkedHashMap<>();
|
||||
for (RedisConnectionKind kind : RedisConnectionKind.values()) {
|
||||
limits.put(kind, 1);
|
||||
}
|
||||
limits.put(CacheScriptLane.CACHE_SCRIPT, 1);
|
||||
RedisRuntimeOwner owner =
|
||||
new RedisRuntimeOwner(
|
||||
StubRedisRuntimeClient.serving(gateway),
|
||||
limits,
|
||||
Duration.ofSeconds(5),
|
||||
Duration.ofSeconds(1));
|
||||
RedisCacheRegionAdapter<String, String> cache =
|
||||
new RedisCacheRegionAdapter<>(
|
||||
owner,
|
||||
keys,
|
||||
key -> "digest-" + key,
|
||||
value -> value.getBytes(StandardCharsets.UTF_8),
|
||||
bytes -> new String(bytes, StandardCharsets.UTF_8),
|
||||
Clock.fixed(T0, ZoneOffset.UTC),
|
||||
Duration.ofSeconds(30),
|
||||
Duration.ofSeconds(120),
|
||||
Duration.ofSeconds(10),
|
||||
Duration.ofMillis(100));
|
||||
|
||||
// when
|
||||
long started = System.nanoTime();
|
||||
CacheLookup<String> lookup;
|
||||
RedisLease held = owner.borrow(RedisConnectionKind.REGULAR);
|
||||
try {
|
||||
lookup = cache.lookup("order-1");
|
||||
} finally {
|
||||
held.close();
|
||||
}
|
||||
Duration waited = Duration.ofNanos(System.nanoTime() - started);
|
||||
|
||||
// then
|
||||
assertThat(
|
||||
List.of(
|
||||
lookup instanceof CacheLookup.Unavailable,
|
||||
waited.compareTo(Duration.ofSeconds(3)) < 0))
|
||||
.as("waited %s on a lane that would have held the caller five seconds", waited)
|
||||
.containsExactly(true, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aValueLargerThanTheCeilingIsNotRecorded() {
|
||||
// given
|
||||
RedisCacheRegionAdapter<String, String> cache = withCeiling(8);
|
||||
|
||||
// when
|
||||
CacheRecordOutcome outcome = cache.record("order-1", "nine bytes", upsert());
|
||||
|
||||
// then
|
||||
assertThat(outcome).isEqualTo(CacheRecordOutcome.NOT_RECORDED_PROVIDER_POLICY);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aValueExactlyAtTheCeilingIsRecorded() {
|
||||
// given
|
||||
RedisCacheRegionAdapter<String, String> cache = withCeiling(8);
|
||||
|
||||
// when
|
||||
CacheRecordOutcome outcome = cache.record("order-1", "eight by", upsert());
|
||||
|
||||
// then
|
||||
assertThat(outcome).isEqualTo(CacheRecordOutcome.RECORDED);
|
||||
}
|
||||
|
||||
private RedisCacheRegionAdapter<String, String> withCeiling(int maximumValueBytes) {
|
||||
return new RedisCacheRegionAdapter<>(
|
||||
new RedisRuntimeOwner(
|
||||
StubRedisRuntimeClient.serving(gateway),
|
||||
lanesOf(4),
|
||||
Duration.ofSeconds(1),
|
||||
Duration.ofSeconds(1)),
|
||||
keys,
|
||||
key -> "digest-" + key,
|
||||
value -> value.getBytes(StandardCharsets.UTF_8),
|
||||
bytes -> new String(bytes, StandardCharsets.UTF_8),
|
||||
Clock.fixed(T0, ZoneOffset.UTC),
|
||||
Duration.ofSeconds(30),
|
||||
Duration.ofSeconds(120),
|
||||
Duration.ofSeconds(10),
|
||||
Duration.ofSeconds(2),
|
||||
maximumValueBytes);
|
||||
}
|
||||
|
||||
@Test
|
||||
void theDefaultCeilingIsTheOneMebibyteTheRetiredSettingHad() {
|
||||
// given
|
||||
String justOver = "x".repeat(RedisCacheRegionAdapter.DEFAULT_MAXIMUM_VALUE_BYTES + 1);
|
||||
|
||||
// when
|
||||
CacheRecordOutcome outcome = cache(T0).record("order-1", justOver, upsert());
|
||||
|
||||
// then
|
||||
assertThat(outcome).isEqualTo(CacheRecordOutcome.NOT_RECORDED_PROVIDER_POLICY);
|
||||
}
|
||||
|
||||
private static Map<RedisLane, Integer> lanesOf(int limit) {
|
||||
Map<RedisLane, Integer> limits = new LinkedHashMap<>();
|
||||
for (RedisConnectionKind kind : RedisConnectionKind.values()) {
|
||||
limits.put(kind, limit);
|
||||
}
|
||||
limits.put(CacheScriptLane.CACHE_SCRIPT, limit);
|
||||
return limits;
|
||||
}
|
||||
|
||||
private static CacheRecordMetadata upsert() {
|
||||
return new CacheRecordMetadata("rev-1", CacheRecordIntent.UPSERT);
|
||||
}
|
||||
@@ -428,8 +546,7 @@ class RedisCacheRegionAdapterTest {
|
||||
|
||||
// when
|
||||
Throwable refused =
|
||||
org.assertj.core.api.Assertions.catchThrowable(
|
||||
() -> cache.record("order-1", "payload", newerThan("rev-2")));
|
||||
catchThrowable(() -> cache.record("order-1", "payload", newerThan("rev-2")));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
@@ -441,7 +558,7 @@ class RedisCacheRegionAdapterTest {
|
||||
@DisplayName("멱등성·lease가 SCRIPT lane을 모두 잡고 있어도 캐시 조건부 기록은 캐시 전용 lane에서 기록된다")
|
||||
void aConditionalCacheWriteDoesNotWaitForTheCorrectnessScriptLane() {
|
||||
// given
|
||||
Map<RedisLane, Integer> limits = new java.util.LinkedHashMap<>();
|
||||
Map<RedisLane, Integer> limits = new LinkedHashMap<>();
|
||||
limits.put(RedisConnectionKind.REGULAR, 4);
|
||||
limits.put(RedisConnectionKind.SCRIPT, 1);
|
||||
limits.put(CacheScriptLane.CACHE_SCRIPT, 1);
|
||||
@@ -584,7 +701,7 @@ class RedisCacheRegionAdapterTest {
|
||||
}
|
||||
|
||||
@Override
|
||||
public RedisLaneConnection openLane(RedisLane kind, java.util.Optional<byte[]> routingKey) {
|
||||
public RedisLaneConnection openLane(RedisLane kind, Optional<byte[]> routingKey) {
|
||||
throw new IllegalStateException("the server is unreachable");
|
||||
}
|
||||
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package dev.caskeleton.adapter.outbound.cache.redis.capability;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.cache.redis.cache.RedisCacheRegionAdapter;
|
||||
import dev.caskeleton.adapter.outbound.support.CapabilityStartupValidationException;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* The cache value ceiling is a deployment's setting, {@code app.capabilities.cache
|
||||
* .maximum-value-bytes} (round9 review 2026-11-12, RR9-01-18): the region refused larger values
|
||||
* with no way for the shipped composition to change the bound.
|
||||
*/
|
||||
class RedisCacheValueCeilingSettingTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("설정하지 않으면 상한은 1 MiB다")
|
||||
void anUnsetCeilingIsOneMebibyte() {
|
||||
assertThat(new RedisCapabilitySettings().getCache().getMaximumValueBytes())
|
||||
.isEqualTo(RedisCacheRegionAdapter.DEFAULT_MAXIMUM_VALUE_BYTES);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("16 MiB를 넘는 상한은 기동 때 거절한다")
|
||||
void aCeilingPastSixteenMebibytesFailsStartup() {
|
||||
// given
|
||||
RedisCapabilitySettings.Cache cache = new RedisCapabilitySettings().getCache();
|
||||
cache.setMaximumValueBytes(16_777_217);
|
||||
|
||||
// when
|
||||
Throwable refused = catchThrowable(cache::validate);
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.isInstanceOf(CapabilityStartupValidationException.class)
|
||||
.hasMessageContaining("the cache value ceiling must be 1..16777216 bytes");
|
||||
}
|
||||
}
|
||||
+12
-9
@@ -111,15 +111,17 @@ public final class ApacheClientFactory {
|
||||
.evictIdleConnections(TimeValue.ofMilliseconds(profile.pool().maxIdleTime().toMillis()))
|
||||
.setDefaultRequestConfig(requestConfig(profile));
|
||||
|
||||
if (!profile.request().compression()) {
|
||||
clientBuilder.disableContentCompression();
|
||||
}
|
||||
// Apache never decodes the body. With its content compression on, the stream the response
|
||||
// reader counted as "wire" bytes was already decompressed, so a profile's wire limit was a
|
||||
// second decoded limit and a well-compressed response under it was refused for its decoded size
|
||||
// (round9 review 2026-11-12, RR9-01-30). The reader counts the bytes as they arrive and decodes
|
||||
// them itself. Whether to ask for compression is the profile's, sent by the RestClient for
|
||||
// every transport.
|
||||
clientBuilder.disableContentCompression();
|
||||
if (profile.proxy().enabled()) {
|
||||
clientBuilder.setProxy(
|
||||
new HttpHost(
|
||||
profile.proxy().type().name().toLowerCase(java.util.Locale.ROOT),
|
||||
profile.proxy().host(),
|
||||
profile.proxy().port()));
|
||||
// An HTTP forward proxy, the only kind ProxyType has: SOCKS used to bind and then become an
|
||||
// HTTP proxy at the SOCKS port.
|
||||
clientBuilder.setProxy(new HttpHost("http", profile.proxy().host(), profile.proxy().port()));
|
||||
}
|
||||
return new ApacheRuntime(clientBuilder.build(), pool);
|
||||
}
|
||||
@@ -131,7 +133,8 @@ public final class ApacheClientFactory {
|
||||
.setRedirectsEnabled(false)
|
||||
.setMaxRedirects(0)
|
||||
.setCircularRedirectsAllowed(false)
|
||||
.setContentCompressionEnabled(profile.request().compression())
|
||||
// Decoding is the response reader's, after it has counted the wire bytes; see create().
|
||||
.setContentCompressionEnabled(false)
|
||||
// Hard cancellation lets a deadline abort actually abort the socket instead of leaving a
|
||||
// request in flight after the caller has given up.
|
||||
.setHardCancellationEnabled(true)
|
||||
|
||||
+13
@@ -22,6 +22,7 @@ import org.eclipse.jetty.http3.client.HTTP3Client;
|
||||
import org.eclipse.jetty.http3.client.transport.HttpClientTransportOverHTTP3;
|
||||
import org.eclipse.jetty.io.Transport;
|
||||
import org.eclipse.jetty.quic.client.ClientQuicConfiguration;
|
||||
import org.eclipse.jetty.util.component.LifeCycle;
|
||||
import org.springframework.http.client.ClientHttpRequestFactory;
|
||||
import org.springframework.http.client.JettyClientHttpRequestFactory;
|
||||
|
||||
@@ -132,6 +133,18 @@ public final class JettyHttp3TransportProvider implements BlockingTransportProvi
|
||||
.getRequestListeners()
|
||||
.addQueuedListener(
|
||||
request -> AttemptDeadline.abortOnExpiry(() -> abortAtDeadline(request)));
|
||||
// Jetty registers its gzip decoder from the classpath when it starts, then asks for gzip on
|
||||
// every request, decodes the response itself and drops Content-Encoding — so the reader's wire
|
||||
// count saw the decoded size, whatever the profile said (round9 review 2026-11-12,
|
||||
// RR9-01-30). Cleared once started, because an empty list before start is what tells Jetty to
|
||||
// load them; the coded bytes then reach the reader, which counts and decodes them.
|
||||
client.addEventListener(
|
||||
new LifeCycle.Listener() {
|
||||
@Override
|
||||
public void lifeCycleStarted(LifeCycle started) {
|
||||
client.getContentDecoderFactories().clear();
|
||||
}
|
||||
});
|
||||
return client;
|
||||
}
|
||||
|
||||
|
||||
+9
-3
@@ -1,7 +1,13 @@
|
||||
package dev.caskeleton.adapter.outbound.httpclient.profile;
|
||||
|
||||
/** Forward proxy kind (design §24.3). */
|
||||
/**
|
||||
* Forward proxy kind (design §24.3).
|
||||
*
|
||||
* <p>HTTP only. Every shipped transport speaks an HTTP forward proxy and none speaks SOCKS; a
|
||||
* {@code SOCKS} value used to bind and then be wired as an HTTP proxy at the SOCKS port, and a
|
||||
* value whose only possible effect is a failure is not kept as an option ({@code
|
||||
* docs/manual/10-capabilities.md} standard 3). {@code proxy.type: SOCKS} now fails to bind.
|
||||
*/
|
||||
public enum ProxyType {
|
||||
HTTP,
|
||||
SOCKS
|
||||
HTTP
|
||||
}
|
||||
|
||||
+55
-7
@@ -5,6 +5,7 @@ import dev.caskeleton.adapter.outbound.httpclient.api.HttpStatus;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpDeadlineExceededException;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpFailureMetadata;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpRemoteErrorException;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpTargetRejectedException;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.operation.AttemptStage;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.operation.ExecutionEvidence;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.operation.HttpOperation;
|
||||
@@ -109,10 +110,38 @@ public final class BlockingStreamingGateway {
|
||||
// exceptions a buffered call throws, classified by the transport's own
|
||||
// classifier at the body stage — not as the engine's bare IOException, which
|
||||
// carried no category, no evidence and nothing to log (RR-04-03).
|
||||
// Decoded here as the buffered reader decodes: the transport hands over the coded
|
||||
// bytes, counted on the wire as they arrive, and what they decode to is counted
|
||||
// against the decoded budget as the caller reads it. A streamed download used to
|
||||
// reach the caller still compressed once the transport stopped decoding.
|
||||
String coding;
|
||||
try {
|
||||
coding = ContentDecoding.codingOf(rawResponse.getHeaders());
|
||||
} catch (ContentDecoding.RefusedCodingException refused) {
|
||||
// A coding this client did not ask for: nothing is handed over, and the
|
||||
// connection is not left open behind the refusal.
|
||||
rawResponse.close();
|
||||
throw new HttpTargetRejectedException(
|
||||
"response content coding is not one this client asked for",
|
||||
metadata
|
||||
.withStatus(new HttpStatus(status))
|
||||
.withEvidence(ExecutionEvidence.RESPONSE_RECEIVED),
|
||||
refused);
|
||||
}
|
||||
InputStream wire =
|
||||
new CountingBoundedInputStream(
|
||||
rawResponse.getBody(), limiter::recordWireBytes);
|
||||
// Counted against the decoded budget whatever the coding, identity included, as
|
||||
// the buffered reader counts it: an identity body is its own decoding.
|
||||
InputStream body =
|
||||
new CountingBoundedInputStream(
|
||||
ContentDecoding.decoding(coding, wire), limiter::recordDecodedBytes);
|
||||
if (!coding.isEmpty()) {
|
||||
ContentDecoding.dropCodingHeaders(responseHeaders);
|
||||
}
|
||||
InputStream bounded =
|
||||
new ClassifiedBodyInputStream(
|
||||
new CountingBoundedInputStream(
|
||||
rawResponse.getBody(), limiter::recordWireBytes),
|
||||
body,
|
||||
bodyFailure ->
|
||||
runtime
|
||||
.support()
|
||||
@@ -129,11 +158,7 @@ public final class BlockingStreamingGateway {
|
||||
false);
|
||||
} catch (RuntimeException failure) {
|
||||
closeQuietly(admission);
|
||||
if (setupDeadline.expired() && !(failure instanceof HttpRemoteErrorException)) {
|
||||
throw new HttpDeadlineExceededException(
|
||||
"effective deadline reached before the stream was set up", metadata, failure);
|
||||
}
|
||||
throw failure;
|
||||
throw setupFailure(failure, setupDeadline.expired(), metadata);
|
||||
}
|
||||
|
||||
BlockingStreamingResponse wrapped =
|
||||
@@ -155,6 +180,29 @@ public final class BlockingStreamingGateway {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What a failure while setting up the stream reaches the caller as.
|
||||
*
|
||||
* <p>Once the setup budget is spent the failure is the deadline, whatever the transport said. A
|
||||
* response that arrived and was refused, or a remote error, stays what it is: its evidence is
|
||||
* that the server answered.
|
||||
*
|
||||
* @param failure what setting up the stream threw
|
||||
* @param budgetSpent whether the setup deadline had expired when it did
|
||||
* @param metadata the call's failure metadata
|
||||
* @return the exception to throw
|
||||
*/
|
||||
static RuntimeException setupFailure(
|
||||
RuntimeException failure, boolean budgetSpent, HttpFailureMetadata metadata) {
|
||||
if (budgetSpent
|
||||
&& !(failure instanceof HttpRemoteErrorException)
|
||||
&& !(failure instanceof HttpTargetRejectedException)) {
|
||||
return new HttpDeadlineExceededException(
|
||||
"effective deadline reached before the stream was set up", metadata, failure);
|
||||
}
|
||||
return failure;
|
||||
}
|
||||
|
||||
/** A permit release must not mask the failure that is already propagating. */
|
||||
private static void closeQuietly(AutoCloseable closeable) {
|
||||
try {
|
||||
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
package dev.caskeleton.adapter.outbound.httpclient.restclient;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.PushbackInputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
import java.util.zip.InflaterInputStream;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
|
||||
/**
|
||||
* Undoes a response's content coding, for the buffered reader and the streaming download alike.
|
||||
*
|
||||
* <p>The transport does not decode (round9 review 2026-11-12, RR9-01-30): the bytes that arrive are
|
||||
* counted against the wire budget, and what they decode to against the decoded budget. Both read
|
||||
* paths come through here, so a streamed download is decoded the way a buffered body is rather than
|
||||
* handed over compressed.
|
||||
*/
|
||||
final class ContentDecoding {
|
||||
|
||||
/**
|
||||
* The {@code Accept-Encoding} value a profile with compression on sends: the codings {@link
|
||||
* #decoding} undoes. Kept beside the decoder so the request never asks for a coding the reader
|
||||
* would hand over still compressed.
|
||||
*/
|
||||
static final String ACCEPTED = "gzip, deflate";
|
||||
|
||||
/** How much a decoder reads from the wire at a time. */
|
||||
static final int CHUNK = 8 * 1024;
|
||||
|
||||
private ContentDecoding() {}
|
||||
|
||||
/** The codings {@link #decoding} undoes; {@code x-gzip} is the old name of {@code gzip}. */
|
||||
private static final Set<String> DECODABLE = Set.of("gzip", "x-gzip", "deflate");
|
||||
|
||||
/**
|
||||
* A response coded in a way this client did not ask for, refused before its body is read.
|
||||
*
|
||||
* <p>Its own type so the gateways can report it as what it is — a response that arrived and was
|
||||
* refused — rather than as the bare {@link IOException} a transport classifier reads as a call
|
||||
* that got no response.
|
||||
*/
|
||||
static final class RefusedCodingException extends IOException {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private RefusedCodingException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The response's content coding, lower case; empty for none or {@code identity}.
|
||||
*
|
||||
* @throws RefusedCodingException when the response names more than one coding — stacked codings
|
||||
* are not something this client asks for, and decoding only the first would hand the rest
|
||||
* over as the body — or a coding {@link #decoding} cannot undo
|
||||
*/
|
||||
static String codingOf(HttpHeaders headers) throws RefusedCodingException {
|
||||
List<String> codings = new ArrayList<>();
|
||||
for (String value : headers.getOrEmpty(HttpHeaders.CONTENT_ENCODING)) {
|
||||
// Limit -1: every element, an empty trailing one included, which the check below skips.
|
||||
for (String part : value.split(",", -1)) {
|
||||
String coding = part.trim().toLowerCase(Locale.ROOT);
|
||||
if (!coding.isEmpty() && !coding.equals("identity")) {
|
||||
codings.add(coding);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (codings.size() > 1) {
|
||||
throw new RefusedCodingException(
|
||||
"the response stacked content codings this client did not ask for: " + codings);
|
||||
}
|
||||
String coding = codings.isEmpty() ? "" : codings.get(0);
|
||||
if (!coding.isEmpty() && !DECODABLE.contains(coding)) {
|
||||
throw new RefusedCodingException(
|
||||
"the response used a content coding this client did not ask for: " + coding);
|
||||
}
|
||||
return coding;
|
||||
}
|
||||
|
||||
/**
|
||||
* The decoded body, read lazily from {@code wire}.
|
||||
*
|
||||
* <p>Lazily, because a gzip stream reads its header when it is built: built eagerly, an empty
|
||||
* body under {@code Content-Encoding: gzip} — a {@code HEAD}, a 204 or a 304 — failed the whole
|
||||
* response, and a streamed download read from the network before it was handed over. Here nothing
|
||||
* is read until the caller reads, and a body that turns out empty decodes to nothing.
|
||||
*
|
||||
* @param coding a coding {@link #codingOf} returned
|
||||
* @param wire the counted wire stream
|
||||
*/
|
||||
static InputStream decoding(String coding, InputStream wire) {
|
||||
return coding.isEmpty() ? wire : new LazilyDecoded(coding, wire);
|
||||
}
|
||||
|
||||
/** {@code headers} without the two that describe the coded body rather than the decoded one. */
|
||||
static void dropCodingHeaders(Map<String, List<String>> headers) {
|
||||
headers
|
||||
.keySet()
|
||||
.removeIf(
|
||||
name ->
|
||||
name.equalsIgnoreCase(HttpHeaders.CONTENT_ENCODING)
|
||||
|| name.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH));
|
||||
}
|
||||
|
||||
private static final class LazilyDecoded extends InputStream {
|
||||
|
||||
private final String coding;
|
||||
private final InputStream wire;
|
||||
private InputStream decoded;
|
||||
|
||||
private LazilyDecoded(String coding, InputStream wire) {
|
||||
this.coding = coding;
|
||||
this.wire = wire;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
return open().read();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(byte[] buffer, int offset, int length) throws IOException {
|
||||
return open().read(buffer, offset, length);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
try {
|
||||
if (decoded != null) {
|
||||
decoded.close();
|
||||
}
|
||||
} finally {
|
||||
wire.close();
|
||||
}
|
||||
}
|
||||
|
||||
private InputStream open() throws IOException {
|
||||
if (decoded == null) {
|
||||
PushbackInputStream peeked = new PushbackInputStream(wire, 1);
|
||||
int first = peeked.read();
|
||||
if (first == -1) {
|
||||
decoded = InputStream.nullInputStream();
|
||||
} else {
|
||||
peeked.unread(first);
|
||||
// deflate is the zlib-wrapped form RFC 9110 names; a server that sends raw deflate under
|
||||
// this name fails to decode, which is a failed response rather than a guessed one.
|
||||
decoded =
|
||||
coding.equals("deflate")
|
||||
? new InflaterInputStream(peeked)
|
||||
: new GZIPInputStream(peeked, CHUNK);
|
||||
}
|
||||
}
|
||||
return decoded;
|
||||
}
|
||||
}
|
||||
}
|
||||
+5
@@ -36,6 +36,11 @@ public final class ResponseSizeLimiter {
|
||||
this.metadata = metadata;
|
||||
}
|
||||
|
||||
/** What a failure of the response this limiter counts says about the call. */
|
||||
HttpFailureMetadata failureMetadata() {
|
||||
return metadata;
|
||||
}
|
||||
|
||||
public void recordWireBytes(long additional) {
|
||||
wireBytes += additional;
|
||||
if (wireBytes > maxWireBytes) {
|
||||
|
||||
+43
-9
@@ -1,8 +1,10 @@
|
||||
package dev.caskeleton.adapter.outbound.httpclient.restclient;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.HttpStatus;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpFailureMetadata;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpSerializationException;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpTargetRejectedException;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.operation.ExecutionEvidence;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.result.ByteArrayResponseType;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.result.ClassResponseType;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.result.EmptyResponseType;
|
||||
@@ -10,6 +12,7 @@ import dev.caskeleton.adapter.outbound.httpclient.api.result.GenericResponseType
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.result.ResponseType;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.profile.ResponseLimits;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.lang.reflect.Type;
|
||||
@@ -34,6 +37,11 @@ import org.springframework.http.converter.json.JacksonJsonHttpMessageConverter;
|
||||
* <p>The body is bounded while it is read and then decoded from memory by a Spring message
|
||||
* converter. Doing the bounding first is what makes a decompression bomb or an unexpectedly huge
|
||||
* payload a rejected request instead of an out-of-memory error.
|
||||
*
|
||||
* <p>Content coding is undone here, not by the transport: the bytes that arrive are counted against
|
||||
* the wire budget as they arrive, and the bytes they decode to are counted against the decoded
|
||||
* budget as they are produced — a chunk at a time, so a decompression bomb is refused before it is
|
||||
* held in memory (round9 review 2026-11-12, RR9-01-30).
|
||||
*/
|
||||
public final class RestClientResponseReader {
|
||||
|
||||
@@ -112,16 +120,42 @@ public final class RestClientResponseReader {
|
||||
java.util.function.LongConsumer responseBytes)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(responseBytes, "response byte observer");
|
||||
String coding;
|
||||
try {
|
||||
coding = ContentDecoding.codingOf(headers);
|
||||
} catch (ContentDecoding.RefusedCodingException refused) {
|
||||
// A response arrived and was refused, which the transport classifier would otherwise read
|
||||
// from a bare IOException as a call that got no answer.
|
||||
throw new HttpTargetRejectedException(
|
||||
"response content coding is not one this client asked for",
|
||||
limiter
|
||||
.failureMetadata()
|
||||
.withStatus(new HttpStatus(status))
|
||||
.withEvidence(ExecutionEvidence.RESPONSE_RECEIVED),
|
||||
refused);
|
||||
}
|
||||
try (InputStream counted =
|
||||
new CountingBoundedInputStream(
|
||||
body,
|
||||
read -> {
|
||||
limiter.recordWireBytes(read);
|
||||
responseBytes.accept(read);
|
||||
})) {
|
||||
byte[] bytes = counted.readAllBytes();
|
||||
limiter.recordDecodedBytes(bytes.length);
|
||||
return new RawResponse(status, toMap(headers), bytes);
|
||||
new CountingBoundedInputStream(
|
||||
body,
|
||||
read -> {
|
||||
limiter.recordWireBytes(read);
|
||||
responseBytes.accept(read);
|
||||
});
|
||||
InputStream decoded = ContentDecoding.decoding(coding, counted)) {
|
||||
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
|
||||
byte[] chunk = new byte[ContentDecoding.CHUNK];
|
||||
int read;
|
||||
while ((read = decoded.read(chunk)) != -1) {
|
||||
limiter.recordDecodedBytes(read);
|
||||
bytes.write(chunk, 0, read);
|
||||
}
|
||||
Map<String, List<String>> kept = toMap(headers);
|
||||
if (!coding.isEmpty()) {
|
||||
// The body handed on is the decoded one; keeping these would describe bytes it no longer
|
||||
// is.
|
||||
ContentDecoding.dropCodingHeaders(kept);
|
||||
}
|
||||
return new RawResponse(status, kept, bytes.toByteArray());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+10
-5
@@ -22,6 +22,7 @@ import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.random.RandomGenerator;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.client.ClientHttpRequestFactory;
|
||||
import org.springframework.web.client.RestClient;
|
||||
|
||||
@@ -77,11 +78,15 @@ public final class RestClientRuntimeFactory implements ClientRuntimeFactory {
|
||||
|
||||
ClientHttpRequestFactory requestFactory =
|
||||
provider.create(profile, generation, lifecycleListener);
|
||||
RestClient restClient =
|
||||
RestClient.builder()
|
||||
.requestFactory(requestFactory)
|
||||
.baseUrl(profile.baseUrl().toString())
|
||||
.build();
|
||||
RestClient.Builder builder =
|
||||
RestClient.builder().requestFactory(requestFactory).baseUrl(profile.baseUrl().toString());
|
||||
if (profile.request().compression()) {
|
||||
// Asked for here, once, for every transport: none of them decodes any more, so the reader
|
||||
// counts the coded bytes on the wire and decodes them itself. The JDK transport used to read
|
||||
// the setting nowhere, and Apache and Jetty each asked on their own.
|
||||
builder.defaultHeader(HttpHeaders.ACCEPT_ENCODING, ContentDecoding.ACCEPTED);
|
||||
}
|
||||
RestClient restClient = builder.build();
|
||||
|
||||
AttemptResiliencePipeline pipeline = newResiliencePipeline(profile);
|
||||
|
||||
|
||||
+2
-1
@@ -8,7 +8,8 @@ import java.util.Set;
|
||||
* What a blocking transport can actually guarantee (design §13.3).
|
||||
*
|
||||
* <p>A capability that is weaker than the profile requires fails at startup rather than degrading
|
||||
* silently at runtime.
|
||||
* silently at runtime. {@code proxySupport} is an HTTP forward proxy, the only kind {@code
|
||||
* ProxyType} has.
|
||||
*/
|
||||
public record BlockingTransportCapabilities(
|
||||
Set<HttpProtocol> protocols,
|
||||
|
||||
+25
@@ -2,13 +2,18 @@ package dev.caskeleton.adapter.outbound.httpclient.autoconfigure;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.httpclient.profile.ClientMode;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.profile.HttpProtocol;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.profile.TransportType;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.context.properties.bind.BindException;
|
||||
import org.springframework.boot.context.properties.bind.Binder;
|
||||
import org.springframework.boot.context.properties.source.MapConfigurationPropertySource;
|
||||
|
||||
/**
|
||||
* The aggregate rules a single client's own fields cannot express.
|
||||
@@ -96,6 +101,26 @@ class HttpClientPlatformSettingsTest {
|
||||
.hasMessageContaining("dynamic-targets");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aSocksProxyTypeFailsToBind() {
|
||||
// given
|
||||
Binder binder =
|
||||
new Binder(
|
||||
new MapConfigurationPropertySource(
|
||||
Map.of("proxy.enabled", "true", "proxy.type", "SOCKS")));
|
||||
|
||||
// when
|
||||
Throwable refused =
|
||||
catchThrowable(() -> binder.bind("proxy", HttpClientPlatformSettings.Proxy.class));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.as("no shipped transport speaks SOCKS; the value used to bind and then misroute")
|
||||
.isInstanceOf(BindException.class)
|
||||
.rootCause()
|
||||
.hasMessageContaining("SOCKS");
|
||||
}
|
||||
|
||||
/**
|
||||
* A client whose only meaningful field here is its name.
|
||||
*
|
||||
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
package dev.caskeleton.adapter.outbound.httpclient.http3;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.httpclient.testkit.ClientProfiles;
|
||||
import org.eclipse.jetty.client.HttpClient;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* The experimental Jetty client hands the response reader the coded bytes (round9 review
|
||||
* 2026-11-12, RR9-01-30).
|
||||
*
|
||||
* <p>Jetty loads its gzip decoder from the classpath on start and then decodes every response
|
||||
* itself, which made the reader's wire count the decoded size. Over a plain TCP transport, as the
|
||||
* deadline test does: what is checked is the client's configuration, not the wire.
|
||||
*/
|
||||
class JettyContentDecodingTest {
|
||||
|
||||
@Test
|
||||
void aStartedClientDecodesNoContentCodingItself() throws Exception {
|
||||
// given
|
||||
HttpClient client =
|
||||
JettyHttp3TransportProvider.configure(
|
||||
new HttpClient(), ClientProfiles.http3Experimental("edge"));
|
||||
|
||||
// when
|
||||
client.start();
|
||||
|
||||
// then
|
||||
try {
|
||||
assertThat(client.getContentDecoderFactories().isEmpty())
|
||||
.as("a decoder left registered decodes the body before the reader can count it")
|
||||
.isTrue();
|
||||
} finally {
|
||||
client.stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
package dev.caskeleton.adapter.outbound.httpclient.restclient;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpResponseTooLargeException;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpTargetRejectedException;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.operation.ExecutionEvidence;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
|
||||
/**
|
||||
* The response reader counts what arrives against the wire budget and what it decodes to against
|
||||
* the decoded budget, a chunk at a time.
|
||||
*
|
||||
* <p>The transport used to decode before the reader saw the body, so the "wire" count was the
|
||||
* decoded size and a well-compressed response was refused for a size it never had on the wire
|
||||
* (round9 review 2026-11-12, RR9-01-30).
|
||||
*/
|
||||
class RestClientResponseReaderEncodingTest {
|
||||
|
||||
private final RestClientResponseReader reader = new RestClientResponseReader();
|
||||
|
||||
@Test
|
||||
void aGzipBodyIsCountedCompressedOnTheWireAndDecodedForTheCaller() throws IOException {
|
||||
// given
|
||||
// Six thousand bytes that compress to a few dozen: inside a 1,000-byte wire budget only when
|
||||
// the wire count is the compressed size.
|
||||
byte[] decoded = "a".repeat(6_000).getBytes(StandardCharsets.US_ASCII);
|
||||
byte[] wire = gzip(decoded);
|
||||
ResponseSizeLimiter limiter = new ResponseSizeLimiter(1_000, 10_000);
|
||||
|
||||
// when
|
||||
RestClientResponseReader.RawResponse response =
|
||||
reader.readBounded(200, gzipHeaders(), new ByteArrayInputStream(wire), limiter);
|
||||
|
||||
// then
|
||||
assertThat(
|
||||
List.of(
|
||||
response.body().length,
|
||||
limiter.wireBytes(),
|
||||
limiter.decodedBytes(),
|
||||
response.headers().containsKey(HttpHeaders.CONTENT_ENCODING)))
|
||||
.containsExactly(6_000, (long) wire.length, 6_000L, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aBodyThatDecodesPastTheBudgetIsRefusedWhileItIsDecoded() throws IOException {
|
||||
// given
|
||||
// A small wire payload that would expand to a megabyte: refused at the decoded budget, before
|
||||
// the whole expansion is held in memory.
|
||||
byte[] wire = gzip(new byte[1_048_576]);
|
||||
ResponseSizeLimiter limiter = new ResponseSizeLimiter(1_048_576, 64 * 1024);
|
||||
|
||||
// when
|
||||
Throwable refused =
|
||||
catchThrowable(
|
||||
() -> reader.readBounded(200, gzipHeaders(), new ByteArrayInputStream(wire), limiter));
|
||||
|
||||
// then
|
||||
assertThat(
|
||||
List.of(
|
||||
refused instanceof HttpResponseTooLargeException,
|
||||
limiter.decodedBytes() <= 64L * 1024 + 8 * 1024))
|
||||
.as("decoding stopped within one chunk past the budget: %s", refused)
|
||||
.containsExactly(true, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
void anUnaskedForContentCodingIsRefused() {
|
||||
// given
|
||||
HttpHeaders brotli = new HttpHeaders();
|
||||
brotli.add(HttpHeaders.CONTENT_ENCODING, "br");
|
||||
ResponseSizeLimiter limiter = new ResponseSizeLimiter(1_000, 1_000);
|
||||
|
||||
// when
|
||||
Throwable refused =
|
||||
catchThrowable(
|
||||
() ->
|
||||
reader.readBounded(
|
||||
200, brotli, new ByteArrayInputStream(new byte[] {1, 2, 3}), limiter));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.as("a response arrived and was refused; it is not a call that got no answer")
|
||||
.isInstanceOfSatisfying(
|
||||
HttpTargetRejectedException.class,
|
||||
rejected ->
|
||||
assertThat(
|
||||
List.of(
|
||||
rejected.getMessage(),
|
||||
rejected.getCause().getMessage(),
|
||||
rejected.metadata().evidence()))
|
||||
.containsExactly(
|
||||
"response content coding is not one this client asked for",
|
||||
"the response used a content coding this client did not ask for: br",
|
||||
ExecutionEvidence.RESPONSE_RECEIVED));
|
||||
}
|
||||
|
||||
@Test
|
||||
void anEmptyBodyUnderGzipIsAnEmptyBody() throws IOException {
|
||||
// given
|
||||
// What a HEAD, a 204 or a 304 sends with the coding of the representation it describes.
|
||||
ResponseSizeLimiter limiter = new ResponseSizeLimiter(1_000, 1_000);
|
||||
|
||||
// when
|
||||
RestClientResponseReader.RawResponse response =
|
||||
reader.readBounded(204, gzipHeaders(), new ByteArrayInputStream(new byte[0]), limiter);
|
||||
|
||||
// then
|
||||
assertThat(response.body()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void stackedContentCodingsAreRefused() {
|
||||
// given
|
||||
HttpHeaders stacked = new HttpHeaders();
|
||||
stacked.add(HttpHeaders.CONTENT_ENCODING, "gzip");
|
||||
stacked.add(HttpHeaders.CONTENT_ENCODING, "br");
|
||||
ResponseSizeLimiter limiter = new ResponseSizeLimiter(1_000, 1_000);
|
||||
|
||||
// when
|
||||
Throwable refused =
|
||||
catchThrowable(
|
||||
() ->
|
||||
reader.readBounded(
|
||||
200, stacked, new ByteArrayInputStream(new byte[] {1, 2, 3}), limiter));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.isInstanceOf(HttpTargetRejectedException.class)
|
||||
.hasMessage("response content coding is not one this client asked for")
|
||||
.cause()
|
||||
.hasMessageContaining("stacked content codings");
|
||||
}
|
||||
|
||||
private static HttpHeaders gzipHeaders() {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.add(HttpHeaders.CONTENT_ENCODING, "gzip");
|
||||
return headers;
|
||||
}
|
||||
|
||||
private static byte[] gzip(byte[] plain) throws IOException {
|
||||
ByteArrayOutputStream compressed = new ByteArrayOutputStream();
|
||||
try (GZIPOutputStream out = new GZIPOutputStream(compressed)) {
|
||||
out.write(plain);
|
||||
}
|
||||
return compressed.toByteArray();
|
||||
}
|
||||
}
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
package dev.caskeleton.adapter.outbound.httpclient.restclient;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.OperationName;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpResponseTooLargeException;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpTargetRejectedException;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.operation.ExecutionEvidence;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.operation.HttpOperation;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.result.BlockingStreamingResponse;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.profile.ClientProfile;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.profile.RequestLimits;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.profile.ResponseLimits;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.profile.TransportType;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.testkit.ClientProfiles;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.testkit.MockHttpServer;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.testkit.TestGateways;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.EnumSource;
|
||||
|
||||
/**
|
||||
* A streamed download is decoded the way a buffered body is, on every blocking transport.
|
||||
*
|
||||
* <p>Once the transports stopped decoding — so the wire budget counts what arrives — the streaming
|
||||
* path still handed the body straight on, and a compressed response reached the caller as gzip
|
||||
* bytes (round9 review 2026-11-12, RR9-01-30). The wire budget here is smaller than the decoded
|
||||
* body and larger than the compressed one: the download passes only when the wire is counted
|
||||
* compressed, which is also what fails if a transport goes back to decoding on its own.
|
||||
*/
|
||||
class StreamingContentCodingTest {
|
||||
|
||||
private static final byte[] DECODED = "a".repeat(6_000).getBytes(StandardCharsets.US_ASCII);
|
||||
|
||||
@ParameterizedTest(name = "{0}")
|
||||
@EnumSource(
|
||||
value = TransportType.class,
|
||||
names = {"APACHE", "JDK"})
|
||||
void aGzipDownloadReachesTheCallerDecodedAndCountedCompressedOnTheWire(TransportType transport)
|
||||
throws Exception {
|
||||
try (MockHttpServer server = MockHttpServer.start();
|
||||
TestGateways.Harness harness = TestGateways.forProfile(profile(server, transport))) {
|
||||
// given
|
||||
server.enqueueEncodedBody(200, "text/plain", "gzip", gzip(DECODED));
|
||||
BlockingStreamingGateway gateway = new BlockingStreamingGateway(harness.registry());
|
||||
|
||||
// when
|
||||
List<Object> received;
|
||||
try (BlockingStreamingResponse response = gateway.download(harness.profile().name(), get())) {
|
||||
received =
|
||||
List.of(
|
||||
response.body().readAllBytes().length,
|
||||
response.headers().keySet().stream()
|
||||
.anyMatch(name -> name.equalsIgnoreCase("Content-Encoding")));
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(received).containsExactly(DECODED.length, false);
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest(name = "{0}")
|
||||
@EnumSource(
|
||||
value = TransportType.class,
|
||||
names = {"APACHE", "JDK"})
|
||||
void anIdentityDownloadPastTheDecodedBudgetFailsAsTheBufferedReadDoes(TransportType transport)
|
||||
throws Exception {
|
||||
try (MockHttpServer server = MockHttpServer.start();
|
||||
TestGateways.Harness harness =
|
||||
TestGateways.forProfile(
|
||||
profile(
|
||||
server, transport, new ResponseLimits(10_000, 1_000, Set.of("text/plain"))))) {
|
||||
// given
|
||||
// Within the wire budget and past the decoded one: an identity body is its own decoding.
|
||||
server.enqueueBody(200, "text/plain", DECODED);
|
||||
BlockingStreamingGateway gateway = new BlockingStreamingGateway(harness.registry());
|
||||
|
||||
// when
|
||||
Throwable failure;
|
||||
try (BlockingStreamingResponse response = gateway.download(harness.profile().name(), get())) {
|
||||
failure = catchThrowable(() -> response.body().readAllBytes());
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(failure)
|
||||
.isInstanceOf(HttpResponseTooLargeException.class)
|
||||
.hasMessageContaining("decoded bytes exceeded the profile limit of 1000");
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest(name = "{0}")
|
||||
@EnumSource(
|
||||
value = TransportType.class,
|
||||
names = {"APACHE", "JDK"})
|
||||
void aStreamedResponseInACodingThisClientDidNotAskForIsRefusedAsReceived(TransportType transport)
|
||||
throws Exception {
|
||||
try (MockHttpServer server = MockHttpServer.start();
|
||||
TestGateways.Harness harness = TestGateways.forProfile(profile(server, transport))) {
|
||||
// given
|
||||
server.enqueueEncodedBody(200, "text/plain", "br", new byte[] {1, 2, 3});
|
||||
BlockingStreamingGateway gateway = new BlockingStreamingGateway(harness.registry());
|
||||
|
||||
// when
|
||||
Throwable refused = catchThrowable(() -> gateway.download(harness.profile().name(), get()));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.isInstanceOfSatisfying(
|
||||
HttpTargetRejectedException.class,
|
||||
rejected ->
|
||||
assertThat(rejected.metadata().evidence())
|
||||
.as("the server answered; the client refused what it sent")
|
||||
.isEqualTo(ExecutionEvidence.RESPONSE_RECEIVED));
|
||||
}
|
||||
}
|
||||
|
||||
private static HttpOperation get() {
|
||||
return HttpOperation.get(new OperationName("download"), "/download", Map.of());
|
||||
}
|
||||
|
||||
private static ClientProfile profile(MockHttpServer server, TransportType transport) {
|
||||
return profile(server, transport, new ResponseLimits(1_000, 10_000, Set.of("text/plain")));
|
||||
}
|
||||
|
||||
private static ClientProfile profile(
|
||||
MockHttpServer server, TransportType transport, ResponseLimits limits) {
|
||||
return ClientProfiles.builder("streaming-coding")
|
||||
.baseUrl(server.uri("/"))
|
||||
.transport(transport)
|
||||
.request(new RequestLimits(1024, true))
|
||||
.response(limits)
|
||||
.build();
|
||||
}
|
||||
|
||||
private static byte[] gzip(byte[] plain) throws IOException {
|
||||
ByteArrayOutputStream compressed = new ByteArrayOutputStream();
|
||||
try (GZIPOutputStream out = new GZIPOutputStream(compressed)) {
|
||||
out.write(plain);
|
||||
}
|
||||
return compressed.toByteArray();
|
||||
}
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
package dev.caskeleton.adapter.outbound.httpclient.restclient;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpDeadlineExceededException;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpFailureMetadata;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpRemoteErrorException;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpTargetRejectedException;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.testkit.CoreFixtures;
|
||||
import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
|
||||
/**
|
||||
* What a streaming setup failure reaches the caller as, decided apart from the race that produces
|
||||
* it: a test against a server cannot make the budget run out between the headers arriving and the
|
||||
* coding being refused, so the decision is held here directly.
|
||||
*/
|
||||
class StreamingSetupFailureTest {
|
||||
|
||||
private static final HttpFailureMetadata METADATA = CoreFixtures.ambiguousMetadata();
|
||||
|
||||
static Stream<Arguments> failuresThatKeepWhatTheyAre() {
|
||||
return Stream.of(
|
||||
Arguments.of(
|
||||
"예산을 다 쓴 뒤 거절한 응답", new HttpTargetRejectedException("refused coding", METADATA), true),
|
||||
Arguments.of(
|
||||
"예산을 다 쓴 뒤 원격 오류", new HttpRemoteErrorException("remote error", METADATA), true),
|
||||
Arguments.of("예산이 남은 transport 실패", new IllegalStateException("transport"), false));
|
||||
}
|
||||
|
||||
@ParameterizedTest(name = "{0}")
|
||||
@MethodSource("failuresThatKeepWhatTheyAre")
|
||||
void aFailureTheDeadlineDidNotCauseKeepsWhatItIs(
|
||||
String name, RuntimeException failure, boolean budgetSpent) {
|
||||
// when
|
||||
RuntimeException thrown = BlockingStreamingGateway.setupFailure(failure, budgetSpent, METADATA);
|
||||
|
||||
// then
|
||||
assertThat(thrown).isSameAs(failure);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aTransportFailureAfterTheBudgetIsSpentIsTheDeadline() {
|
||||
// given
|
||||
IllegalStateException failure = new IllegalStateException("transport");
|
||||
|
||||
// when
|
||||
RuntimeException thrown = BlockingStreamingGateway.setupFailure(failure, true, METADATA);
|
||||
|
||||
// then
|
||||
assertThat(thrown)
|
||||
.isInstanceOf(HttpDeadlineExceededException.class)
|
||||
.hasMessage("effective deadline reached before the stream was set up")
|
||||
.hasCause(failure);
|
||||
}
|
||||
}
|
||||
+16
@@ -6,7 +6,10 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpConfigurationException;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.profile.ClientMode;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.profile.ClientProfile;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.profile.ProxySettings;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.profile.ProxyType;
|
||||
import dev.caskeleton.adapter.outbound.httpclient.testkit.ClientProfiles;
|
||||
import java.util.Optional;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class TransportCapabilityValidatorTest {
|
||||
@@ -46,6 +49,19 @@ class TransportCapabilityValidatorTest {
|
||||
.hasMessageContaining("validated DNS pinning");
|
||||
}
|
||||
|
||||
@Test
|
||||
void acceptsAnHttpProxyOnATransportThatHasOne() {
|
||||
ClientProfile http =
|
||||
ClientProfiles.builder("egress")
|
||||
.proxy(
|
||||
new ProxySettings(
|
||||
true, "proxy.internal", 3128, ProxyType.HTTP, Optional.empty(), false))
|
||||
.build();
|
||||
|
||||
assertThatCode(() -> validator.validate(http, BlockingTransportCapabilities.apacheClassic()))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void acceptsAMatchingCapabilitySet() {
|
||||
assertThatCode(
|
||||
|
||||
+16
@@ -121,6 +121,22 @@ public final class MockHttpServer implements AutoCloseable {
|
||||
.setBody(new okio.Buffer().write(body)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Answers with a body already in {@code contentEncoding}, as a server that compresses would.
|
||||
*
|
||||
* @param body the coded bytes, sent as they are
|
||||
*/
|
||||
@SuppressWarnings("resource") // the okio Buffer, as in enqueueBody
|
||||
public void enqueueEncodedBody(
|
||||
int status, String contentType, String contentEncoding, byte[] body) {
|
||||
server.enqueue(
|
||||
new MockResponse()
|
||||
.setResponseCode(status)
|
||||
.setHeader("Content-Type", contentType)
|
||||
.setHeader("Content-Encoding", contentEncoding)
|
||||
.setBody(new okio.Buffer().write(body)));
|
||||
}
|
||||
|
||||
/** Accepts the request and never answers; drives response-header timeout tests. */
|
||||
public void enqueueNoResponse() {
|
||||
server.enqueue(new MockResponse().setSocketPolicy(SocketPolicy.NO_RESPONSE));
|
||||
|
||||
@@ -37,8 +37,8 @@
|
||||
- 주 classpath에 JSON Schema 엔진(NetworkNT)이 없다. 증거 manifest schema 검사와 capability card 검사
|
||||
(`release/`)는 이 leaf의 시험(`evidence/`)이다. snakeyaml은 시험에서만 쓴다(소유자 결정 2026-09-25).
|
||||
- 유스케이스 순서나 업무 라우팅 정책을 broker 어댑터 안에 숨기지 않는다.
|
||||
- `OutboxMessagePublishAdapter`는 위임만 하고 의존성 로그를 내지 않는다. 확정된 FAILED/DEAD 전이가 정해진
|
||||
ERROR 하나를 가진다.
|
||||
- `OutboxMessagePublishAdapter`는 발행을 위임하고, publisher가 알린 `longestAttempt`까지만 기다린다(넘으면 결과 불명).
|
||||
의존성 로그를 내지 않는다. 확정된 FAILED/DEAD 전이가 정해진 ERROR 하나를 가진다.
|
||||
|
||||
## 테스트
|
||||
|
||||
|
||||
@@ -35,6 +35,17 @@ publish pipeline을 탄다. producer id는 host/pod/process 이름에서 추측
|
||||
결과를 application 결과(CONFIRMED·AMBIGUOUS·REJECTED_BEFORE_SEND·REJECTED_AFTER_BROKER·
|
||||
REJECTED_RETRYABLE)로 번역해 relay에 돌려준다. 다시 직렬화하지 않는다.
|
||||
|
||||
그 결과를 기다리는 시간은 `longestAttempt()`(relay가 batch와 lease를 맞춰 본 값)까지다. 넘으면 기다리기를 멈추고
|
||||
`AMBIGUOUS`(`PUBLISH_WAIT_EXCEEDED`)로 돌려준다. broker에 닿았을 수 있어 확정도 거부도 아니다. 예전에는 상한 없이
|
||||
`join()`해서, 답하지 않는 publisher가 relay thread를 claim의 lease와 종료 대기 시간 너머까지 붙잡았고 그 사이 다른
|
||||
인스턴스가 같은 행을 다시 보냈다(round9 리뷰 2026-11-12, RR9-01-05). `longestAttempt()`를 모르는 publisher는 자기
|
||||
deadline이 유일한 상한이다. 0이나 음수를 알리는 publisher는 결함이다. 그 행은 보내지 않고 `REJECTED_RETRYABLE`
|
||||
(`PUBLISH_BOUND_NOT_POSITIVE`)로 돌려준다. relay는 시도로 세지 않고 backoff 뒤에 다시 보며, 실패 보고가 원인을 남긴다.
|
||||
|
||||
relay thread가 interrupt되면 둘로 갈린다. 기다리는 중이면 `AMBIGUOUS`(`PUBLISH_WAIT_INTERRUPTED`)이고 interrupt 표시를
|
||||
남긴다. 발행하기 전에 이미 interrupt됐으면 아무것도 보내지 않고 `REJECTED_RETRYABLE`(`PUBLISH_INTERRUPTED_BEFORE_SEND`)로
|
||||
돌려준다. relay는 그 행을 backoff 뒤로 미뤘다가 발행한다(실패 보고 한 줄).
|
||||
|
||||
platform `REJECTED` 중 `FailureDescriptor.retryable()`이 true인 것(종료 중 `SHUTTING_DOWN`, permit
|
||||
대기 초과, `PUBLISH_RUNTIME_UNAVAILABLE`, `KAFKA_TRANSPORT_CLOSED`, Kafka 인증·인가 실패)은
|
||||
`REJECTED_RETRYABLE`로 번역한다. relay는 이 값을 DEAD로 보내지 않고 재시도한다. 확정 거부 두 값으로
|
||||
|
||||
+67
-2
@@ -10,6 +10,11 @@ import dev.caskeleton.application.outbox.OutboxPublishRefusedException;
|
||||
import java.time.Duration;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
/**
|
||||
* The relay's publish port: a claimed row's validated event, delegated unchanged to the canonical
|
||||
@@ -19,9 +24,28 @@ import java.util.Optional;
|
||||
* here re-renders it. The canonical publisher's attempt — its outcome and its failure code — is
|
||||
* passed through unchanged, so the relay can tell a possibly-stored message (ambiguous) from a
|
||||
* refused one, and a refused credential from a closed transport.
|
||||
*
|
||||
* <p>The wait for that attempt is bounded by {@link #longestAttempt()}, the same value the relay
|
||||
* sized its batch against. It used to be an unbounded {@code join()}: a publisher whose stage never
|
||||
* completed held the relay thread past its claim's lease — another instance then sent the row again
|
||||
* — and past the shutdown wait the relay promises. A wait that runs out is {@link
|
||||
* OutboxPublishOutcome#AMBIGUOUS}: the message may have reached the broker, so it is neither
|
||||
* confirmed nor refused (round9 review 2026-11-12, RR9-01-05).
|
||||
*/
|
||||
public final class OutboxMessagePublishAdapter implements OutboxMessagePublishPort {
|
||||
|
||||
/** The code an attempt carries when this adapter stopped waiting for it. */
|
||||
static final String WAIT_EXCEEDED = "PUBLISH_WAIT_EXCEEDED";
|
||||
|
||||
/** The code an attempt carries when the relay thread was interrupted while waiting. */
|
||||
static final String WAIT_INTERRUPTED = "PUBLISH_WAIT_INTERRUPTED";
|
||||
|
||||
/** The code an attempt carries when the relay thread was interrupted before it published. */
|
||||
static final String INTERRUPTED_BEFORE_SEND = "PUBLISH_INTERRUPTED_BEFORE_SEND";
|
||||
|
||||
/** The code an attempt carries when the publisher reported a bound of nothing. */
|
||||
static final String BOUND_NOT_POSITIVE = "PUBLISH_BOUND_NOT_POSITIVE";
|
||||
|
||||
private final IntegrationEventPublishPort canonicalPublisher;
|
||||
|
||||
public OutboxMessagePublishAdapter(IntegrationEventPublishPort canonicalPublisher) {
|
||||
@@ -41,7 +65,10 @@ public final class OutboxMessagePublishAdapter implements OutboxMessagePublishPo
|
||||
throw new OutboxPublishRefusedException(outcome);
|
||||
}
|
||||
|
||||
/** The canonical publisher's, unchanged: this adapter adds no wait of its own. */
|
||||
/**
|
||||
* The canonical publisher's, unchanged: this adapter adds no wait of its own, and waits no longer
|
||||
* than this.
|
||||
*/
|
||||
@Override
|
||||
public Optional<Duration> longestAttempt() {
|
||||
return canonicalPublisher.longestAttempt();
|
||||
@@ -50,6 +77,44 @@ public final class OutboxMessagePublishAdapter implements OutboxMessagePublishPo
|
||||
@Override
|
||||
public OutboxPublishAttempt publishForOutcome(ClaimedOutboxEvent event) {
|
||||
Objects.requireNonNull(event, "event must not be null");
|
||||
return canonicalPublisher.publish(event.event()).toCompletableFuture().join();
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
// Asked to stop before this row was sent: send nothing. The relay defers the row and
|
||||
// publishes it after its backoff; publishing and then reporting it ambiguous sent every
|
||||
// remaining row of the batch and left each to be sent again.
|
||||
return OutboxPublishAttempt.failed(
|
||||
OutboxPublishOutcome.REJECTED_RETRYABLE, INTERRUPTED_BEFORE_SEND);
|
||||
}
|
||||
Optional<Duration> bound = canonicalPublisher.longestAttempt();
|
||||
if (bound.isPresent() && (bound.get().isNegative() || bound.get().isZero())) {
|
||||
// A bound of nothing would make every publish ambiguous before the broker could answer. It
|
||||
// is the publisher's defect, not this row's: send nothing and let the relay retry the row
|
||||
// after its backoff, with the failure report naming the cause. Thrown, it counted against
|
||||
// the row's attempts and dead-lettered the whole outbox within minutes.
|
||||
return OutboxPublishAttempt.failed(
|
||||
OutboxPublishOutcome.REJECTED_RETRYABLE, BOUND_NOT_POSITIVE);
|
||||
}
|
||||
// The bound starts before publish() is called, not after it returns: a send can block the
|
||||
// caller for up to its own deadline before handing back a stage (a Kafka producer waiting on
|
||||
// metadata), and a wait measured from the return let one row take the bound twice over while
|
||||
// the relay sized its lease for once.
|
||||
long deadline = System.nanoTime() + bound.map(Duration::toNanos).orElse(0L);
|
||||
CompletableFuture<OutboxPublishAttempt> attempt =
|
||||
canonicalPublisher.publish(event.event()).toCompletableFuture();
|
||||
if (bound.isEmpty()) {
|
||||
// A publisher that cannot say how long it takes gave the relay nothing to size its batch
|
||||
// against either; its own deadline is the only bound there is.
|
||||
return attempt.join();
|
||||
}
|
||||
try {
|
||||
return attempt.get(Math.max(0L, deadline - System.nanoTime()), TimeUnit.NANOSECONDS);
|
||||
} catch (TimeoutException exceeded) {
|
||||
return OutboxPublishAttempt.failed(OutboxPublishOutcome.AMBIGUOUS, WAIT_EXCEEDED);
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
return OutboxPublishAttempt.failed(OutboxPublishOutcome.AMBIGUOUS, WAIT_INTERRUPTED);
|
||||
} catch (ExecutionException failed) {
|
||||
// As join() reported it, so the relay's handling of a failed stage does not change.
|
||||
throw new CompletionException(failed.getCause());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+150
@@ -2,16 +2,24 @@ package dev.caskeleton.adapter.outbound.messaging.outbox;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
|
||||
|
||||
import dev.caskeleton.application.messaging.event.testkit.BrokerAnswer;
|
||||
import dev.caskeleton.application.messaging.event.testkit.FakeIntegrationEventPublisher;
|
||||
import dev.caskeleton.application.outbox.ClaimedOutboxEvent;
|
||||
import dev.caskeleton.application.outbox.OutboxMessagePublishPort;
|
||||
import dev.caskeleton.application.outbox.OutboxPublishAmbiguousException;
|
||||
import dev.caskeleton.application.outbox.OutboxPublishAttempt;
|
||||
import dev.caskeleton.application.outbox.OutboxPublishOutcome;
|
||||
import dev.caskeleton.application.outbox.OutboxPublishRefusedException;
|
||||
import dev.caskeleton.application.outbox.testkit.OutboxMessagePublishContract;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -49,6 +57,148 @@ class OutboxMessagePublishAdapterTest extends OutboxMessagePublishContract {
|
||||
assertThat(canonical.published()).containsExactly(claim.event());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("broker가 longestAttempt 안에 답하지 않으면 더 기다리지 않고 결과 불명으로 끝낸다")
|
||||
void aPublishTheBrokerNeverAnswersEndsAmbiguousAtTheLongestAttempt() {
|
||||
// given
|
||||
OutboxMessagePublishPort adapter =
|
||||
new OutboxMessagePublishAdapter(canonical.silentPast(Duration.ofMillis(50)));
|
||||
|
||||
// when
|
||||
// A wait with no bound never returns here; ten seconds is far past the 50 ms the publisher
|
||||
// promised and far short of a hung relay.
|
||||
OutboxPublishAttempt attempt =
|
||||
assertTimeoutPreemptively(
|
||||
Duration.ofSeconds(10), () -> adapter.publishForOutcome(claimedRow()));
|
||||
|
||||
// then
|
||||
assertThat(attempt)
|
||||
.isEqualTo(
|
||||
OutboxPublishAttempt.failed(
|
||||
OutboxPublishOutcome.AMBIGUOUS, OutboxMessagePublishAdapter.WAIT_EXCEEDED));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("publish()가 호출자를 붙잡은 시간도 longestAttempt 안에 센다")
|
||||
void theWaitIncludesTheTimePublishHeldTheCaller() {
|
||||
// given
|
||||
// Blocks 1 s before handing back a stage that never completes, with a 1.2 s promise: a wait
|
||||
// measured from the return would end at about 2.2 s, one measured from the call at 1.2 s. The
|
||||
// 2 s threshold sits between them, 0.8 s above the right answer, so a loaded runner has room.
|
||||
OutboxMessagePublishPort adapter =
|
||||
new OutboxMessagePublishAdapter(
|
||||
canonical.silentPast(Duration.ofMillis(1_200)).blockingFor(Duration.ofSeconds(1)));
|
||||
long started = System.nanoTime();
|
||||
|
||||
// when
|
||||
var unused =
|
||||
assertTimeoutPreemptively(
|
||||
Duration.ofSeconds(10), () -> adapter.publishForOutcome(claimedRow()));
|
||||
|
||||
// then
|
||||
assertThat(Duration.ofNanos(System.nanoTime() - started))
|
||||
.as("one row used the bound twice over while the relay sized its lease for once")
|
||||
.isLessThan(Duration.ofSeconds(2));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("기다리던 relay 스레드가 interrupt되면 결과 불명으로 끝내고 interrupt 표시를 남긴다")
|
||||
void anInterruptedWaitEndsAmbiguousAndKeepsTheInterrupt() throws Exception {
|
||||
// given
|
||||
OutboxMessagePublishPort adapter =
|
||||
new OutboxMessagePublishAdapter(canonical.silentPast(Duration.ofSeconds(30)));
|
||||
AtomicReference<OutboxPublishAttempt> attempt = new AtomicReference<>();
|
||||
AtomicBoolean stillInterrupted = new AtomicBoolean();
|
||||
CountDownLatch waiting = new CountDownLatch(1);
|
||||
Thread relay =
|
||||
new Thread(
|
||||
() -> {
|
||||
waiting.countDown();
|
||||
attempt.set(adapter.publishForOutcome(claimedRow()));
|
||||
stillInterrupted.set(Thread.currentThread().isInterrupted());
|
||||
});
|
||||
relay.start();
|
||||
assertThat(waiting.await(5, TimeUnit.SECONDS)).as("the relay thread started").isTrue();
|
||||
// Until the publish has been handed out, an interrupt is the before-the-send case below.
|
||||
while (canonical.published().isEmpty()) {
|
||||
Thread.onSpinWait();
|
||||
}
|
||||
|
||||
// when
|
||||
relay.interrupt();
|
||||
boolean ended = relay.join(Duration.ofSeconds(10));
|
||||
|
||||
// then
|
||||
assertThat(List.of(ended, attempt.get(), stillInterrupted.get()))
|
||||
.containsExactly(
|
||||
true,
|
||||
OutboxPublishAttempt.failed(
|
||||
OutboxPublishOutcome.AMBIGUOUS, OutboxMessagePublishAdapter.WAIT_INTERRUPTED),
|
||||
true);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("이미 interrupt된 relay 스레드는 발행하지 않고 다시 claim할 수 있는 거부로 답한다")
|
||||
void anAlreadyInterruptedRelayPublishesNothing() {
|
||||
// given
|
||||
OutboxMessagePublishPort adapter =
|
||||
new OutboxMessagePublishAdapter(canonical.silentPast(Duration.ofSeconds(30)));
|
||||
Thread.currentThread().interrupt();
|
||||
|
||||
// when
|
||||
OutboxPublishAttempt attempt;
|
||||
try {
|
||||
attempt = adapter.publishForOutcome(claimedRow());
|
||||
} finally {
|
||||
Thread.interrupted();
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(List.of(attempt, canonical.published()))
|
||||
.as("a relay asked to stop sent the rest of its batch and left each row to be sent again")
|
||||
.containsExactly(
|
||||
OutboxPublishAttempt.failed(
|
||||
OutboxPublishOutcome.REJECTED_RETRYABLE,
|
||||
OutboxMessagePublishAdapter.INTERRUPTED_BEFORE_SEND),
|
||||
List.of());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("longestAttempt가 0인 publisher에는 보내지 않고, 시도로 세지 않는 거부로 답한다")
|
||||
void aPublisherReportingAZeroBoundIsSentNothing() {
|
||||
// given
|
||||
OutboxMessagePublishPort adapter =
|
||||
new OutboxMessagePublishAdapter(canonical.silentPast(Duration.ZERO));
|
||||
|
||||
// when
|
||||
OutboxPublishAttempt attempt = adapter.publishForOutcome(claimedRow());
|
||||
|
||||
// then
|
||||
assertThat(List.of(attempt, canonical.published()))
|
||||
.as("thrown, the defect counted against every row's attempts and dead-lettered the outbox")
|
||||
.containsExactly(
|
||||
OutboxPublishAttempt.failed(
|
||||
OutboxPublishOutcome.REJECTED_RETRYABLE,
|
||||
OutboxMessagePublishAdapter.BOUND_NOT_POSITIVE),
|
||||
List.of());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("실패로 끝난 발행 stage는 join()과 같은 CompletionException으로 알린다")
|
||||
void aFailedStageIsReportedAsJoinReportedIt() {
|
||||
// given
|
||||
IllegalStateException broker = new IllegalStateException("broker refused the connection");
|
||||
OutboxMessagePublishPort adapter =
|
||||
new OutboxMessagePublishAdapter(
|
||||
canonical.silentPast(Duration.ofSeconds(5)).failingWith(broker));
|
||||
|
||||
// when
|
||||
Throwable failure = catchThrowable(() -> adapter.publishForOutcome(claimedRow()));
|
||||
|
||||
// then
|
||||
assertThat(failure).isInstanceOf(CompletionException.class).hasCause(broker);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("확인을 받지 못한 발행은 예외로 알리는 메서드에서 AMBIGUOUS 신호를 던진다")
|
||||
void anAmbiguousPublishThrowsTheAmbiguousSignalFromTheThrowingMethod() {
|
||||
|
||||
+36
@@ -37,6 +37,8 @@ public record NotificationPlatformSettings(
|
||||
providers = providers == null ? Map.of() : Map.copyOf(providers);
|
||||
boolean callbacksEnabled = callbacks.enabled();
|
||||
providers.forEach((id, provider) -> provider.validate(id, callbacksEnabled));
|
||||
Duration leaseDuration = dispatch.leaseDuration();
|
||||
providers.forEach((id, provider) -> provider.requireCallFitsInLease(id, leaseDuration));
|
||||
}
|
||||
|
||||
/** Source-compatible constructor for callers that rely on the original settings shape. */
|
||||
@@ -264,6 +266,40 @@ public record NotificationPlatformSettings(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fail the boot when one call of this profile alone would outlast the dispatch lease.
|
||||
*
|
||||
* <p>A worker holds a delivery's lease while it waits on the provider, and the profile's
|
||||
* timeout is how long that wait can be. A lease that ends first shows a live call as a stuck
|
||||
* lease; if the call is still running one lease later, lease recovery takes the delivery for
|
||||
* abandoned and hands it to reconciliation, and the shipped families (SES, SMTP) have no status
|
||||
* query, so it is parked for an operator. A call that finishes after that still records its
|
||||
* result, since it holds the same owner and fence. No second worker claims it: the claim never
|
||||
* takes a dispatching delivery. Each value is fine on its own, so only the pair can be checked
|
||||
* (round9 review 2026-11-12, RR9-01-24).
|
||||
*
|
||||
* <p>A necessary condition, not a sufficient one: the lease starts at the claim, so writing the
|
||||
* attempt, rendering and waiting for a provider permit spend it before the call begins. A
|
||||
* deployment keeps the lease well above the timeout rather than just above it.
|
||||
*
|
||||
* @param profileId the profile id
|
||||
* @param leaseDuration {@code app.notification.platform.dispatch.lease-duration}
|
||||
*/
|
||||
void requireCallFitsInLease(String profileId, Duration leaseDuration) {
|
||||
if (!enabled) {
|
||||
return;
|
||||
}
|
||||
require(
|
||||
timeout.compareTo(leaseDuration) < 0,
|
||||
profileId,
|
||||
"timeout ("
|
||||
+ timeout
|
||||
+ ") must be shorter than app.notification.platform.dispatch.lease-duration ("
|
||||
+ leaseDuration
|
||||
+ "): a provider call that outlasts its lease shows as a stuck lease, and one lease"
|
||||
+ " later lease recovery hands its delivery to reconciliation while it may still run");
|
||||
}
|
||||
|
||||
private static void require(boolean condition, String profileId, String message) {
|
||||
if (!condition) {
|
||||
throw new IllegalArgumentException(
|
||||
|
||||
+28
@@ -5,6 +5,7 @@ import dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderRu
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import dev.caskeleton.application.notification.platform.provider.ReconciliationCapability;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumMap;
|
||||
import java.util.HashMap;
|
||||
@@ -112,6 +113,10 @@ public final class NotificationProviderAssembly {
|
||||
Map<ProviderProfileId, ReconciliationCapability> reconciliations = new LinkedHashMap<>();
|
||||
Map<Channel, List<String>> claimants = new EnumMap<>(Channel.class);
|
||||
|
||||
if (mode != NotificationPlatformMode.INGEST_ONLY) {
|
||||
requireFamiliesTogether(configured, settings.dispatch().leaseDuration());
|
||||
}
|
||||
|
||||
for (Map.Entry<String, NotificationPlatformSettings.Provider> entry : configured.entrySet()) {
|
||||
String profileId = entry.getKey();
|
||||
NotificationPlatformSettings.Provider profile = entry.getValue();
|
||||
@@ -198,6 +203,29 @@ public final class NotificationProviderAssembly {
|
||||
}
|
||||
}
|
||||
|
||||
/** Hands each family its enabled profiles at once, for the limits only the set can break. */
|
||||
private void requireFamiliesTogether(
|
||||
Map<String, NotificationPlatformSettings.Provider> configured, Duration deliveryLease) {
|
||||
Map<ProviderType, Map<String, NotificationPlatformSettings.Provider>> byFamily =
|
||||
new EnumMap<>(ProviderType.class);
|
||||
configured.forEach(
|
||||
(profileId, profile) -> {
|
||||
if (profile.enabled()) {
|
||||
byFamily
|
||||
.computeIfAbsent(
|
||||
ProviderType.parse(profileId, profile.type()), family -> new TreeMap<>())
|
||||
.put(profileId, profile);
|
||||
}
|
||||
});
|
||||
byFamily.forEach(
|
||||
(family, profiles) -> {
|
||||
ProviderRuntimeAssembler assembler = assemblers.get(family);
|
||||
if (assembler != null) {
|
||||
assembler.requireTogether(Map.copyOf(profiles), deliveryLease);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private static String missingAssemblerMessage(String profileId, ProviderType type) {
|
||||
if (type == ProviderType.SMTP) {
|
||||
return "notification provider profile '"
|
||||
|
||||
+17
-3
@@ -103,24 +103,38 @@ public class NotificationSmtpProviderConfig {
|
||||
return new SmtpFailureClassifier();
|
||||
}
|
||||
|
||||
/** How many sends may wait per send thread before the pool refuses one. */
|
||||
static final int QUEUED_SENDS_PER_THREAD = 4;
|
||||
|
||||
/**
|
||||
* Where an SMTP send runs.
|
||||
*
|
||||
* <p>Bounded, and it aborts rather than queues without limit: an SMTP relay that has stopped
|
||||
* answering turns an unbounded queue into retained heap and a dispatch worker that never notices.
|
||||
* {@code CallerRunsPolicy} would instead block the dispatch loop on the slow relay, which stalls
|
||||
* every other channel too.
|
||||
* every other channel too. A rejected send never reached the relay, and the adapter reports it so
|
||||
* ({@code ProviderCallNotStartedException}), not as an ambiguous submission.
|
||||
*
|
||||
* <p>{@code dispatch-concurrency} threads from the first send, idle ones timing out. The core
|
||||
* size used to be one: a {@code ThreadPoolExecutor} adds a thread past its core only when the
|
||||
* queue is full, and the profile's limiter admits fewer sends than the queue holds, so every SMTP
|
||||
* send ran on one thread whatever {@code dispatch-concurrency} said (round9 review 2026-11-12,
|
||||
* RR9-01-15).
|
||||
*
|
||||
* <p>A deployment's own executor must not run a send on the submitting thread, as {@code
|
||||
* CallerRunsPolicy} or a direct executor does: the send's two timeout deadlines complete its
|
||||
* future, and a dispatch thread blocked inside {@code execute} never gets to read it.
|
||||
*/
|
||||
@Bean(name = "notificationSmtpExecutor", destroyMethod = "shutdown")
|
||||
@ConditionalOnMissingBean(name = "notificationSmtpExecutor")
|
||||
public Executor notificationSmtpExecutor(NotificationSmtpSettings settings) {
|
||||
ThreadPoolExecutor executor =
|
||||
new ThreadPoolExecutor(
|
||||
1,
|
||||
settings.dispatchConcurrency(),
|
||||
settings.dispatchConcurrency(),
|
||||
60L,
|
||||
TimeUnit.SECONDS,
|
||||
new LinkedBlockingQueue<>(settings.dispatchConcurrency() * 4),
|
||||
new LinkedBlockingQueue<>(settings.dispatchConcurrency() * QUEUED_SENDS_PER_THREAD),
|
||||
runnable -> {
|
||||
Thread thread = new Thread(runnable, "notification-smtp");
|
||||
thread.setDaemon(true);
|
||||
|
||||
+18
@@ -1,5 +1,8 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Turns one configured profile into one working provider.
|
||||
*
|
||||
@@ -32,4 +35,19 @@ public interface ProviderRuntimeAssembler {
|
||||
* is missing
|
||||
*/
|
||||
AssembledProvider assemble(String profileId, NotificationPlatformSettings.Provider profile);
|
||||
|
||||
/**
|
||||
* Refuses this family's enabled profiles together, before any of them is assembled.
|
||||
*
|
||||
* <p>Some limits are shared by every profile of a family — one send pool — and can only be
|
||||
* checked against all of them at once; some depend on the platform's delivery lease, which no
|
||||
* single profile carries. Called once per boot with the whole set, so it holds no state between
|
||||
* calls. The default accepts.
|
||||
*
|
||||
* @param profiles the family's enabled profiles, by id
|
||||
* @param deliveryLease {@code app.notification.platform.dispatch.lease-duration}
|
||||
* @throws IllegalStateException naming the profile and the limit it breaks
|
||||
*/
|
||||
default void requireTogether(
|
||||
Map<String, NotificationPlatformSettings.Provider> profiles, Duration deliveryLease) {}
|
||||
}
|
||||
|
||||
+100
-1
@@ -15,9 +15,12 @@ import dev.caskeleton.application.notification.platform.provider.ProviderCapabil
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderProfileSnapshot;
|
||||
import dev.caskeleton.application.notification.platform.security.ContactPointProtector;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
|
||||
/**
|
||||
* The first production assembler: one configured SMTP profile becomes one working provider
|
||||
@@ -98,10 +101,18 @@ public final class SmtpProviderRuntimeAssembler implements ProviderRuntimeAssemb
|
||||
String profileId, NotificationPlatformSettings.Provider profile) {
|
||||
Objects.requireNonNull(profileId, "profileId");
|
||||
Objects.requireNonNull(profile, "profile");
|
||||
requireTimeoutOutlastsOneStalledStep(profileId, profile.timeout());
|
||||
|
||||
SmtpNotificationProviderAdapter adapter =
|
||||
new SmtpNotificationProviderAdapter(
|
||||
dispatch, mimeFactory, classifier, protector, transport, executor, attachmentGuard);
|
||||
dispatch,
|
||||
mimeFactory,
|
||||
classifier,
|
||||
protector,
|
||||
transport,
|
||||
executor,
|
||||
attachmentGuard,
|
||||
profile.timeout());
|
||||
|
||||
ProviderRuntime runtime =
|
||||
new ProviderRuntime(
|
||||
@@ -112,6 +123,94 @@ public final class SmtpProviderRuntimeAssembler implements ProviderRuntimeAssemb
|
||||
return AssembledProvider.dispatchOnly(runtime, Channel.EMAIL);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses a profile timeout that a single stalled connect and command can outlast.
|
||||
*
|
||||
* <p>The profile timeout bounds the whole send and reports what outlasts it as ambiguous, since
|
||||
* the DATA may have gone through. A relay that accepts the connection and never answers the
|
||||
* greeting has been sent nothing; its socket deadline reports that, as a failure the classifier
|
||||
* knows is safe to retry. With the profile timeout shorter than the connect deadline plus one
|
||||
* command's, the profile timeout fired first and turned that safe retry into an ambiguous attempt
|
||||
* an operator reconciles by hand (round9 review 2026-11-12, RR9-01-15).
|
||||
*/
|
||||
private void requireTimeoutOutlastsOneStalledStep(String profileId, Duration timeout) {
|
||||
Duration oneCommand =
|
||||
transport.readTimeout().compareTo(transport.writeTimeout()) >= 0
|
||||
? transport.readTimeout()
|
||||
: transport.writeTimeout();
|
||||
Duration stalledStep = transport.connectTimeout().plus(oneCommand);
|
||||
if (timeout.compareTo(stalledStep) <= 0) {
|
||||
throw new IllegalStateException(
|
||||
"notification provider profile '"
|
||||
+ profileId
|
||||
+ "': timeout ("
|
||||
+ timeout
|
||||
+ ") must be longer than mail.smtp.connectiontimeout plus the longer of"
|
||||
+ " mail.smtp.timeout and mail.smtp.writetimeout ("
|
||||
+ stalledStep
|
||||
+ "), or a relay that never answers is reported as an ambiguous send");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses SMTP profiles whose in-flight limits together are more than the send pool can hold, and
|
||||
* a delivery lease one SMTP send can outlast.
|
||||
*
|
||||
* <p>The limiter lets {@code max-concurrency} sends through per profile; every SMTP profile
|
||||
* shares the one pool, which runs its threads, queues the rest up to its queue's capacity, and
|
||||
* rejects what is left. Profiles allowed more in flight together than threads plus queue had
|
||||
* their excess rejected on every burst. Only a {@link ThreadPoolExecutor} says how much it holds,
|
||||
* so a deployment's own executor is not checked (round9 review 2026-11-12, RR9-01-16).
|
||||
*
|
||||
* <p>One send may wait in the pool for up to the profile's timeout and then run for up to the
|
||||
* timeout again, so it holds its delivery's lease for up to twice the timeout. A lease shorter
|
||||
* than that shows a running send as a stuck lease; if the send is still running one lease later,
|
||||
* lease recovery takes the delivery for abandoned and, since an SMTP relay cannot be asked, parks
|
||||
* it for an operator. A send that finishes after that still records its result.
|
||||
*/
|
||||
@Override
|
||||
public void requireTogether(
|
||||
Map<String, NotificationPlatformSettings.Provider> profiles, Duration deliveryLease) {
|
||||
new TreeMap<>(profiles)
|
||||
.forEach(
|
||||
(profileId, profile) -> {
|
||||
Duration oneSend = profile.timeout().multipliedBy(2);
|
||||
if (deliveryLease.compareTo(oneSend) <= 0) {
|
||||
throw new IllegalStateException(
|
||||
"notification provider profile '"
|
||||
+ profileId
|
||||
+ "': app.notification.platform.dispatch.lease-duration ("
|
||||
+ deliveryLease
|
||||
+ ") must be longer than twice the SMTP timeout ("
|
||||
+ oneSend
|
||||
+ "): a send may wait in the pool for the timeout and then run for it");
|
||||
}
|
||||
});
|
||||
if (!(executor instanceof ThreadPoolExecutor pool)) {
|
||||
return;
|
||||
}
|
||||
int threads = pool.getMaximumPoolSize();
|
||||
int queued = pool.getQueue().remainingCapacity();
|
||||
long holds = (long) threads + queued;
|
||||
long together =
|
||||
profiles.values().stream()
|
||||
.mapToLong(NotificationPlatformSettings.Provider::maxConcurrency)
|
||||
.sum();
|
||||
if (together > holds) {
|
||||
throw new IllegalStateException(
|
||||
"notification provider profiles "
|
||||
+ new TreeMap<>(profiles).keySet()
|
||||
+ ": max-concurrency ("
|
||||
+ together
|
||||
+ " together) is more than the SMTP send pool holds ("
|
||||
+ threads
|
||||
+ " threads and "
|
||||
+ queued
|
||||
+ " queued) — raise app.notification.platform.smtp.dispatch-concurrency or lower"
|
||||
+ " max-concurrency");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The snapshot the dispatch loop plans against. Its capabilities are the adapter's own, as the
|
||||
* SES assembler does: a second copy here had drifted to a different payload limit.
|
||||
|
||||
+3
-2
@@ -42,8 +42,9 @@ public record NotificationDispatchProperties(
|
||||
requirePositive(estimatedDispatchDuration, "estimatedDispatchDuration");
|
||||
requirePositive(maxQueueAge, "maxQueueAge");
|
||||
if (leaseDuration.compareTo(pollInterval) <= 0) {
|
||||
// A lease shorter than the poll interval expires before the worker can renew it, so two
|
||||
// workers would routinely claim the same job.
|
||||
// A sanity bound, not a renewal rule (see RecipientLeaseStorePort.renew). The lease is how
|
||||
// long one dispatch may hold a delivery and the poll interval only how long an idle worker
|
||||
// sleeps, so a lease no longer than that nap is too short for any real send.
|
||||
throw new IllegalArgumentException("leaseDuration must exceed pollInterval");
|
||||
}
|
||||
}
|
||||
|
||||
+3
-3
@@ -81,10 +81,10 @@ public final class NotificationSchedulerWorker implements AutoCloseable {
|
||||
// by the claim batch size — so a queue of ten and a queue of ten million published the same
|
||||
// number, and the one metric named "queue depth" was the one that could not show a queue
|
||||
// growing.
|
||||
// The depth alone: the full serving snapshot is several statements and is the health
|
||||
// endpoint's, not something to run on every poll.
|
||||
metrics.gauge(
|
||||
NotificationMetricName.QUEUE_DEPTH,
|
||||
Map.of(),
|
||||
(double) servingState.currentState().backlogDepth());
|
||||
NotificationMetricName.QUEUE_DEPTH, Map.of(), (double) servingState.backlogDepth());
|
||||
|
||||
for (RecipientLease lease : claimed) {
|
||||
globalConcurrency.acquireUninterruptibly();
|
||||
|
||||
+138
-2
@@ -1,13 +1,20 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.provider.smtp;
|
||||
|
||||
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.provider.EmailAttachments;
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.provider.EmailRecipients;
|
||||
import dev.caskeleton.application.notification.platform.api.ProviderId;
|
||||
import dev.caskeleton.application.notification.platform.api.error.FailureCategory;
|
||||
import dev.caskeleton.application.notification.platform.api.error.NotificationFailureCode;
|
||||
import dev.caskeleton.application.notification.platform.api.routing.Channel;
|
||||
import dev.caskeleton.application.notification.platform.contact.EmailAddress;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.AttachmentIntegrityGuard;
|
||||
import dev.caskeleton.application.notification.platform.provider.NotificationProviderAdapter;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderCallNotStartedException;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderCapabilities;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderFailure;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmission;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmissionResult;
|
||||
import dev.caskeleton.application.notification.platform.provider.ResolvedAttachment;
|
||||
@@ -15,10 +22,13 @@ import dev.caskeleton.application.notification.platform.security.ContactPointPro
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionStage;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
/**
|
||||
* SMTP email adapter.
|
||||
@@ -39,7 +49,14 @@ public final class SmtpNotificationProviderAdapter implements NotificationProvid
|
||||
private final SmtpProviderProperties properties;
|
||||
private final Executor executor;
|
||||
private final AttachmentIntegrityGuard attachmentGuard;
|
||||
private final Duration callTimeout;
|
||||
|
||||
/**
|
||||
* @param callTimeout the profile's {@code timeout}: the longest one send may wait in the pool,
|
||||
* and then run, before it is reported as not started or as ambiguous. Startup holds it above
|
||||
* one stalled connect and command
|
||||
* @throws IllegalStateException if the timeout is not positive, which is the assembly's fault
|
||||
*/
|
||||
public SmtpNotificationProviderAdapter(
|
||||
SmtpDispatch dispatch,
|
||||
SmtpMimeMessageFactory mimeFactory,
|
||||
@@ -47,7 +64,8 @@ public final class SmtpNotificationProviderAdapter implements NotificationProvid
|
||||
ContactPointProtector protector,
|
||||
SmtpProviderProperties properties,
|
||||
Executor executor,
|
||||
AttachmentIntegrityGuard attachmentGuard) {
|
||||
AttachmentIntegrityGuard attachmentGuard,
|
||||
Duration callTimeout) {
|
||||
this.dispatch = Objects.requireNonNull(dispatch, "dispatch");
|
||||
this.mimeFactory = Objects.requireNonNull(mimeFactory, "mimeFactory");
|
||||
this.classifier = Objects.requireNonNull(classifier, "classifier");
|
||||
@@ -55,6 +73,10 @@ public final class SmtpNotificationProviderAdapter implements NotificationProvid
|
||||
this.properties = Objects.requireNonNull(properties, "properties");
|
||||
this.executor = Objects.requireNonNull(executor, "executor");
|
||||
this.attachmentGuard = Objects.requireNonNull(attachmentGuard, "attachmentGuard");
|
||||
this.callTimeout = Objects.requireNonNull(callTimeout, "callTimeout");
|
||||
if (callTimeout.isNegative() || callTimeout.isZero()) {
|
||||
throw new IllegalStateException("callTimeout must be positive");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -77,7 +99,109 @@ public final class SmtpNotificationProviderAdapter implements NotificationProvid
|
||||
@Override
|
||||
public CompletionStage<ProviderSubmissionResult> submit(ProviderSubmission submission) {
|
||||
Objects.requireNonNull(submission, "submission");
|
||||
return CompletableFuture.supplyAsync(() -> send(submission), executor);
|
||||
PooledSend send = new PooledSend(submission);
|
||||
try {
|
||||
executor.execute(send);
|
||||
} catch (RejectedExecutionException full) {
|
||||
// The send pool refused the task: nothing was handed to the relay. Reported as not started,
|
||||
// so the delivery is retried, rather than falling through to the ambiguous residue an
|
||||
// operator has to reconcile by hand (round9 review 2026-11-12, RR9-01-16).
|
||||
throw new ProviderCallNotStartedException(
|
||||
NotificationFailureCode.PROVIDER_UNAVAILABLE,
|
||||
FailureCategory.CAPACITY_REJECTED,
|
||||
true,
|
||||
"the SMTP send pool is full; the message was not handed to the relay");
|
||||
}
|
||||
send.armQueueDeadline();
|
||||
return send.result();
|
||||
}
|
||||
|
||||
/**
|
||||
* One send through the pool, bounded by the profile's timeout twice over.
|
||||
*
|
||||
* <p>Waiting in the pool: a send that has not started when the timeout passes is abandoned and
|
||||
* never runs. Nothing reached the relay, so it is reported as not started and retried; the old
|
||||
* single timer, started at the submit, closed it as ambiguous and the send then went out anyway,
|
||||
* after its permit was released.
|
||||
*
|
||||
* <p>Running: from the moment a pool thread takes it, the send has the whole timeout. The socket
|
||||
* timeouts bound each SMTP command, not the conversation, so a relay answering slowly but within
|
||||
* each one held the send past the delivery lease: it showed as a stuck lease, and one lease later
|
||||
* lease recovery parked the delivery for an operator while the send still ran. Past the bound the
|
||||
* outcome is ambiguous — the DATA may have gone through — and is never retried automatically; the
|
||||
* send keeps its thread until the socket gives up. Startup holds the timeout above one stalled
|
||||
* connect and command, so a relay that never answers fails on its socket deadline first, as a
|
||||
* classified failure, rather than on this one.
|
||||
*/
|
||||
private final class PooledSend implements Runnable {
|
||||
|
||||
private final ProviderSubmission submission;
|
||||
private final AtomicReference<SendState> state = new AtomicReference<>(SendState.QUEUED);
|
||||
private final CompletableFuture<ProviderSubmissionResult> result = new CompletableFuture<>();
|
||||
|
||||
/**
|
||||
* Completes when a pool thread takes the send, or when the timeout passes first.
|
||||
*
|
||||
* <p>Both deadlines are {@code completeOnTimeout}: the JDK's delay thread completes the future
|
||||
* itself, so nothing waits behind the common pool, and completing the future early cancels the
|
||||
* timer, so a send that finishes in time leaves no timer holding it.
|
||||
*/
|
||||
private final CompletableFuture<Void> started = new CompletableFuture<>();
|
||||
|
||||
private PooledSend(ProviderSubmission submission) {
|
||||
this.submission = submission;
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the queue deadline, once the pool has taken the send: a send the pool refused needs
|
||||
* none, and one the pool already ran has completed {@link #started}, so no timer is made.
|
||||
*/
|
||||
private void armQueueDeadline() {
|
||||
// Nothing to observe on the stage it returns: the abandon reports through the result.
|
||||
var unused =
|
||||
started
|
||||
.completeOnTimeout(null, callTimeout.toMillis(), MILLISECONDS)
|
||||
.thenRun(this::abandonIfStillQueued);
|
||||
}
|
||||
|
||||
private CompletableFuture<ProviderSubmissionResult> result() {
|
||||
return result;
|
||||
}
|
||||
|
||||
private void abandonIfStillQueued() {
|
||||
if (state.compareAndSet(SendState.QUEUED, SendState.ABANDONED)) {
|
||||
result.completeExceptionally(
|
||||
new ProviderCallNotStartedException(
|
||||
NotificationFailureCode.PROVIDER_UNAVAILABLE,
|
||||
FailureCategory.CAPACITY_REJECTED,
|
||||
true,
|
||||
"the SMTP send waited in the pool past the profile timeout of "
|
||||
+ callTimeout
|
||||
+ "; the message was not handed to the relay"));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
if (!state.compareAndSet(SendState.QUEUED, SendState.RUNNING)) {
|
||||
return;
|
||||
}
|
||||
// Cancels the queue deadline; its abandon then finds the send running and does nothing.
|
||||
started.complete(null);
|
||||
result.completeOnTimeout(callOutlastedItsTimeout(), callTimeout.toMillis(), MILLISECONDS);
|
||||
try {
|
||||
result.complete(send(submission));
|
||||
} catch (RuntimeException failure) {
|
||||
result.completeExceptionally(failure);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Where a pooled send is: only a queued one may still be abandoned. */
|
||||
private enum SendState {
|
||||
QUEUED,
|
||||
RUNNING,
|
||||
ABANDONED
|
||||
}
|
||||
|
||||
private ProviderSubmissionResult send(ProviderSubmission submission) {
|
||||
@@ -100,6 +224,18 @@ public final class SmtpNotificationProviderAdapter implements NotificationProvid
|
||||
}
|
||||
}
|
||||
|
||||
private ProviderSubmissionResult callOutlastedItsTimeout() {
|
||||
return ProviderSubmissionResult.ambiguous(
|
||||
new ProviderFailure(
|
||||
NotificationFailureCode.PROVIDER_RESPONSE_LOST,
|
||||
FailureCategory.AMBIGUOUS_SUBMISSION,
|
||||
false,
|
||||
Optional.empty(),
|
||||
Optional.of("the SMTP send outlasted the profile timeout of " + callTimeout)),
|
||||
ProviderExecutionEvidence.undetermined(),
|
||||
callTimeout);
|
||||
}
|
||||
|
||||
private static Duration elapsedSince(long startedNanos) {
|
||||
return Duration.ofNanos(System.nanoTime() - startedNanos);
|
||||
}
|
||||
|
||||
+33
@@ -31,6 +31,39 @@ class NotificationPlatformSettingsTest {
|
||||
.hasMessageContaining("unknown provider type '" + removedType + "'");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aProfileWhoseTimeoutIsNotShorterThanTheLeaseFailsStartup() {
|
||||
// given
|
||||
Duration lease = NotificationPlatformSettings.Dispatch.defaults().leaseDuration();
|
||||
NotificationPlatformSettings.Provider slow =
|
||||
new NotificationPlatformSettings.Provider(
|
||||
"SMTP", true, false, "PRODUCTION", "smtp-key", lease, 8, 20, null);
|
||||
|
||||
// when
|
||||
Throwable failure = catchThrowable(() -> properties(slow));
|
||||
|
||||
// then
|
||||
assertThat(failure)
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("notification provider profile 'profile'")
|
||||
.hasMessageContaining("must be shorter than app.notification.platform.dispatch.lease");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aDisabledProfileIsNotHeldToTheLease() {
|
||||
// given
|
||||
Duration lease = NotificationPlatformSettings.Dispatch.defaults().leaseDuration();
|
||||
NotificationPlatformSettings.Provider off =
|
||||
new NotificationPlatformSettings.Provider(
|
||||
"SMTP", false, false, "PRODUCTION", "smtp-key", lease.multipliedBy(2), 8, 20, null);
|
||||
|
||||
// when
|
||||
NotificationPlatformSettings settings = properties(off);
|
||||
|
||||
// then
|
||||
assertThat(settings.providers()).containsOnlyKeys("profile");
|
||||
}
|
||||
|
||||
@Test
|
||||
void enabledSesRequiresItsTypedTransportIdentity() {
|
||||
assertThatThrownBy(
|
||||
|
||||
+118
@@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.callback.CallbackVerificationResult;
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.callback.ProviderCallbackAdapter;
|
||||
@@ -45,6 +46,9 @@ import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletionStage;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -199,6 +203,120 @@ class NotificationProviderAssemblyTest {
|
||||
.hasMessageContaining("INGEST_ONLY");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("SERVING 조립은 SMTP timeout 20초의 두 배(40초)에 못 미치는 lease 30초를 거절한다")
|
||||
void aServingAssemblyRefusesALeaseOneSmtpSendCanOutlast() {
|
||||
// given
|
||||
// The settings fallback, Dispatch.defaults(), leases for 30 s (the module defaults ship 2 m):
|
||||
// longer than the timeout, so the settings accept it, and shorter than twice the timeout, which
|
||||
// only the SMTP family check refuses.
|
||||
Map<String, NotificationPlatformSettings.Provider> providers =
|
||||
Map.of("primary-email", smtpProfile(true, Duration.ofSeconds(20), 4));
|
||||
|
||||
// when
|
||||
Throwable failure =
|
||||
catchThrowable(
|
||||
() -> assemble(providers, SmtpProviderRuntimeAssemblerTest.assembler(Runnable::run)));
|
||||
|
||||
// then
|
||||
assertThat(failure)
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining(
|
||||
"lease-duration (PT30S) must be longer than twice the SMTP timeout (PT40S)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void anIngestOnlyAssemblyDoesNotHoldSmtpToTheDispatchLease() {
|
||||
// given
|
||||
Map<String, NotificationPlatformSettings.Provider> providers =
|
||||
Map.of("primary-email", smtpProfile(true, Duration.ofSeconds(20), 4));
|
||||
|
||||
// when
|
||||
NotificationProviderAssembly.AssembledPlatform platform =
|
||||
assemble(
|
||||
providers,
|
||||
NotificationPlatformMode.INGEST_ONLY,
|
||||
SmtpProviderRuntimeAssemblerTest.assembler(Runnable::run));
|
||||
|
||||
// then
|
||||
assertThat(platform.routeFor(Channel.EMAIL))
|
||||
.as("an ingest-only deployment sends nothing, so no send can outlast its lease")
|
||||
.contains(new ProviderProfileId("primary-email"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"SERVING 조립은 켜진 SMTP profile의 max-concurrency 합(3+3)이 pool(thread 1, 대기열 4)을 넘으면 거절한다")
|
||||
void aServingAssemblyRefusesSmtpProfilesThatTogetherOverfillThePool() {
|
||||
// given
|
||||
Map<String, NotificationPlatformSettings.Provider> providers =
|
||||
Map.of(
|
||||
"primary-email", smtpProfile(true, Duration.ofSeconds(10), 3),
|
||||
"secondary-email", smtpSecondary(true, 3));
|
||||
|
||||
// when
|
||||
Throwable failure = catchThrowable(() -> assembleOnASmallPool(providers));
|
||||
|
||||
// then
|
||||
assertThat(failure)
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("max-concurrency (6 together) is more than the SMTP send pool holds");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("꺼진 SMTP profile의 max-concurrency는 pool(thread 1, 대기열 4) 합에 들지 않는다")
|
||||
void aDisabledSmtpProfileIsLeftOutOfThePoolSum() {
|
||||
// given
|
||||
Map<String, NotificationPlatformSettings.Provider> providers =
|
||||
Map.of(
|
||||
"primary-email", smtpProfile(true, Duration.ofSeconds(10), 3),
|
||||
"secondary-email", smtpSecondary(false, 3));
|
||||
|
||||
// when
|
||||
NotificationProviderAssembly.AssembledPlatform platform = assembleOnASmallPool(providers);
|
||||
|
||||
// then
|
||||
assertThat(
|
||||
List.of(
|
||||
platform.runtimes().find(new ProviderProfileId("primary-email")).isPresent(),
|
||||
platform.runtimes().find(new ProviderProfileId("secondary-email")).isPresent()))
|
||||
.as("the enabled profile is assembled; the disabled one is neither assembled nor counted")
|
||||
.containsExactly(true, false);
|
||||
}
|
||||
|
||||
/** Assembles {@code providers} over one SMTP send thread with four queued. */
|
||||
private static NotificationProviderAssembly.AssembledPlatform assembleOnASmallPool(
|
||||
Map<String, NotificationPlatformSettings.Provider> providers) {
|
||||
ThreadPoolExecutor pool =
|
||||
new ThreadPoolExecutor(1, 1, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>(4));
|
||||
try {
|
||||
return assemble(providers, SmtpProviderRuntimeAssemblerTest.assembler(pool));
|
||||
} finally {
|
||||
pool.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
private static NotificationPlatformSettings.Provider smtpProfile(
|
||||
boolean enabled, Duration timeout, int maxConcurrency) {
|
||||
return new NotificationPlatformSettings.Provider(
|
||||
"SMTP", enabled, true, "local", "smtp-local", timeout, maxConcurrency, 8, null);
|
||||
}
|
||||
|
||||
/** A second SMTP profile on the same channel, not its primary. */
|
||||
private static NotificationPlatformSettings.Provider smtpSecondary(
|
||||
boolean enabled, int maxConcurrency) {
|
||||
return new NotificationPlatformSettings.Provider(
|
||||
"SMTP",
|
||||
enabled,
|
||||
false,
|
||||
"local",
|
||||
"smtp-local",
|
||||
Duration.ofSeconds(10),
|
||||
maxConcurrency,
|
||||
8,
|
||||
null);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("INGEST_ONLY is a declaration, and it reports non-serving")
|
||||
void ingestOnlyIsADeclaration() {
|
||||
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* The SMTP send pool runs {@code dispatch-concurrency} sends at once.
|
||||
*
|
||||
* <p>Its core size was one, and a {@code ThreadPoolExecutor} adds a thread past its core only when
|
||||
* its queue is full. The profile's limiter admits fewer sends than the queue holds, so the queue
|
||||
* never filled and every send waited for the one thread (round9 review 2026-11-12, RR9-01-15).
|
||||
*/
|
||||
class NotificationSmtpExecutorTest {
|
||||
|
||||
@Test
|
||||
void asManySendsAsTheDispatchConcurrencyRunAtOnce() throws InterruptedException {
|
||||
// given
|
||||
Executor executor =
|
||||
new NotificationSmtpProviderConfig()
|
||||
.notificationSmtpExecutor(new NotificationSmtpSettings("no-reply@example.test", 3));
|
||||
CountDownLatch allRunning = new CountDownLatch(3);
|
||||
CountDownLatch release = new CountDownLatch(1);
|
||||
|
||||
// when
|
||||
boolean concurrent;
|
||||
try {
|
||||
for (int send = 0; send < 3; send++) {
|
||||
executor.execute(
|
||||
() -> {
|
||||
allRunning.countDown();
|
||||
awaitQuietly(release);
|
||||
});
|
||||
}
|
||||
concurrent = allRunning.await(10, TimeUnit.SECONDS);
|
||||
} finally {
|
||||
release.countDown();
|
||||
((ExecutorService) executor).shutdownNow();
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(concurrent).as("three sends in flight on three threads").isTrue();
|
||||
}
|
||||
|
||||
private static void awaitQuietly(CountDownLatch latch) {
|
||||
try {
|
||||
if (!latch.await(30, TimeUnit.SECONDS)) {
|
||||
throw new IllegalStateException("the test never released the send");
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
+112
-2
@@ -1,6 +1,7 @@
|
||||
package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.provider.smtp.SmtpFailureClassifier;
|
||||
import dev.caskeleton.adapter.outbound.notification.platform.provider.smtp.SmtpMimeMessageFactory;
|
||||
@@ -10,7 +11,12 @@ import dev.caskeleton.application.notification.platform.dispatch.AttachmentInteg
|
||||
import jakarta.mail.Session;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -25,6 +31,8 @@ import org.junit.jupiter.api.Test;
|
||||
*/
|
||||
class SmtpProviderRuntimeAssemblerTest {
|
||||
|
||||
private static final Duration LEASE = Duration.ofMinutes(2);
|
||||
|
||||
@Test
|
||||
@DisplayName("the assembler serves the SMTP family on the email channel")
|
||||
void theAssemblerServesSmtpOnEmail() {
|
||||
@@ -81,7 +89,105 @@ class SmtpProviderRuntimeAssemblerTest {
|
||||
.isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aProfileAllowedMoreInFlightThanTheSendPoolHoldsFailsAssembly() {
|
||||
// given
|
||||
// One thread and four queued: five sends fit, the profile would let eight through.
|
||||
ThreadPoolExecutor pool =
|
||||
new ThreadPoolExecutor(1, 1, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>(4));
|
||||
|
||||
// when
|
||||
Throwable failure;
|
||||
try {
|
||||
failure =
|
||||
catchThrowable(
|
||||
() -> assembler(pool).requireTogether(Map.of("primary-email", profile(8)), LEASE));
|
||||
} finally {
|
||||
pool.shutdownNow();
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(failure)
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining(
|
||||
"max-concurrency (8 together) is more than the SMTP send pool holds (1 threads and 4");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("SMTP profile 둘이 합쳐 send pool이 담는 것보다 많이 내보내면 조립이 실패한다")
|
||||
void twoProfilesThatTogetherOverfillTheSendPoolFailAssembly() {
|
||||
// given
|
||||
// One thread and four queued: five sends fit; the two profiles would let six through.
|
||||
ThreadPoolExecutor pool =
|
||||
new ThreadPoolExecutor(1, 1, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>(4));
|
||||
SmtpProviderRuntimeAssembler assembler = assembler(pool);
|
||||
|
||||
// when
|
||||
Throwable failure;
|
||||
try {
|
||||
failure =
|
||||
catchThrowable(
|
||||
() ->
|
||||
assembler.requireTogether(
|
||||
Map.of("primary-email", profile(3), "secondary-email", profile(3)), LEASE));
|
||||
} finally {
|
||||
pool.shutdownNow();
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(failure)
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining(
|
||||
"[primary-email, secondary-email]: max-concurrency (6 together) is more than the SMTP"
|
||||
+ " send pool holds");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("SMTP timeout 10초의 두 배인 20초를 넘지 않는 delivery lease는 조립이 실패한다")
|
||||
void aDeliveryLeaseOneSendCanOutlastFailsAssembly() {
|
||||
// when
|
||||
Throwable failure =
|
||||
catchThrowable(
|
||||
() ->
|
||||
assembler()
|
||||
.requireTogether(Map.of("primary-email", profile(4)), Duration.ofSeconds(20)));
|
||||
|
||||
// then
|
||||
assertThat(failure)
|
||||
.as("a send may wait in the pool for the timeout and then run for it")
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("lease-duration (PT20S) must be longer than twice the SMTP timeout");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("connect 2초와 명령 5초가 다 지나도록 기다릴 수 없는 7초 timeout은 조립이 실패한다")
|
||||
void aProfileTimeoutOneStalledConnectAndCommandOutlastFailsAssembly() {
|
||||
// given
|
||||
NotificationPlatformSettings.Provider sevenSeconds =
|
||||
new NotificationPlatformSettings.Provider(
|
||||
"SMTP", true, true, "local", "smtp-local", Duration.ofSeconds(7), 4, 8, null);
|
||||
|
||||
// when
|
||||
Throwable failure = catchThrowable(() -> assembler().assemble("primary-email", sevenSeconds));
|
||||
|
||||
// then
|
||||
assertThat(failure)
|
||||
.as(
|
||||
"a relay that never answers the greeting has been sent nothing; its socket deadline says so")
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("timeout (PT7S) must be longer than mail.smtp.connectiontimeout")
|
||||
.hasMessageContaining("(PT7S), or a relay that never answers");
|
||||
}
|
||||
|
||||
private static SmtpProviderRuntimeAssembler assembler() {
|
||||
return assembler(Runnable::run);
|
||||
}
|
||||
|
||||
/**
|
||||
* The real SMTP assembler over {@code executor}; also used by {@code
|
||||
* NotificationProviderAssemblyTest}.
|
||||
*/
|
||||
static SmtpProviderRuntimeAssembler assembler(Executor executor) {
|
||||
Session session = Session.getInstance(new Properties());
|
||||
return new SmtpProviderRuntimeAssembler(
|
||||
message -> {
|
||||
@@ -90,7 +196,7 @@ class SmtpProviderRuntimeAssemblerTest {
|
||||
new SmtpMimeMessageFactory(session),
|
||||
new SmtpFailureClassifier(),
|
||||
protector(),
|
||||
Runnable::run,
|
||||
executor,
|
||||
new AttachmentIntegrityGuard((reference, context) -> null),
|
||||
new SmtpProviderProperties(
|
||||
"localhost",
|
||||
@@ -108,8 +214,12 @@ class SmtpProviderRuntimeAssemblerTest {
|
||||
}
|
||||
|
||||
private static NotificationPlatformSettings.Provider profile() {
|
||||
return profile(4);
|
||||
}
|
||||
|
||||
private static NotificationPlatformSettings.Provider profile(int maxConcurrency) {
|
||||
return new NotificationPlatformSettings.Provider(
|
||||
"SMTP", true, true, "local", "smtp-local", Duration.ofSeconds(10), 4, 8, null);
|
||||
"SMTP", true, true, "local", "smtp-local", Duration.ofSeconds(10), maxConcurrency, 8, null);
|
||||
}
|
||||
|
||||
/** Not a lambda: the protector has three operations, and only one of them is exercised here. */
|
||||
|
||||
+128
-2
@@ -15,18 +15,28 @@ import dev.caskeleton.application.notification.platform.contact.EmailAddress;
|
||||
import dev.caskeleton.application.notification.platform.contact.PhoneNumber;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderCallNotStartedException;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmission;
|
||||
import dev.caskeleton.application.notification.platform.provider.ProviderSubmissionResult;
|
||||
import dev.caskeleton.application.notification.platform.provider.testkit.ProviderFixtures;
|
||||
import dev.caskeleton.application.notification.platform.security.ContactPointProtector;
|
||||
import jakarta.mail.Session;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class SmtpNotificationProviderAdapterTest {
|
||||
|
||||
private static final Duration CALL_TIMEOUT = Duration.ofSeconds(5);
|
||||
|
||||
private final ContactPointProtector protector =
|
||||
new AesGcmContactPointProtector(SecurityFixtures.keys());
|
||||
|
||||
@@ -138,7 +148,122 @@ class SmtpNotificationProviderAdapterTest {
|
||||
.hasMessageContaining("email contact point");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aSendThePoolRejectsIsReportedAsNotStarted() {
|
||||
// given
|
||||
Executor full =
|
||||
command -> {
|
||||
throw new RejectedExecutionException("the send pool is full");
|
||||
};
|
||||
|
||||
// when
|
||||
Throwable failure = catchThrowable(() -> adapter(message -> {}, full).submit(submission()));
|
||||
|
||||
// then
|
||||
assertThat(failure)
|
||||
.isInstanceOf(ProviderCallNotStartedException.class)
|
||||
.extracting("code", "category", "retryable")
|
||||
.containsExactly("PROVIDER_UNAVAILABLE", FailureCategory.CAPACITY_REJECTED, true);
|
||||
}
|
||||
|
||||
private SmtpNotificationProviderAdapter adapter(SmtpDispatch dispatch) {
|
||||
return adapter(dispatch, Executors.newSingleThreadExecutor());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aSendThatOutlastsTheProfileTimeoutIsAmbiguousNotRetried() throws Exception {
|
||||
// given
|
||||
CountDownLatch released = new CountDownLatch(1);
|
||||
SmtpDispatch slowRelay =
|
||||
message -> {
|
||||
try {
|
||||
released.await();
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
};
|
||||
ExecutorService pool = Executors.newSingleThreadExecutor();
|
||||
// A second, not milliseconds: the send has to leave the pool queue inside it on a loaded
|
||||
// runner, or it is abandoned before it starts and this is the other test.
|
||||
SmtpNotificationProviderAdapter adapter = adapter(slowRelay, pool, Duration.ofSeconds(1));
|
||||
|
||||
// when
|
||||
ProviderSubmissionResult result;
|
||||
try {
|
||||
result = adapter.submit(submission()).toCompletableFuture().get(5, TimeUnit.SECONDS);
|
||||
} finally {
|
||||
released.countDown();
|
||||
pool.shutdownNow();
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(result)
|
||||
.as("past the profile timeout the DATA may have gone through: never resend on our own")
|
||||
.extracting(
|
||||
ProviderSubmissionResult::confirmation,
|
||||
outcome -> outcome.failure().orElseThrow().code(),
|
||||
outcome -> outcome.failure().orElseThrow().retryable())
|
||||
.containsExactly(AttemptConfirmation.AMBIGUOUS, "PROVIDER_RESPONSE_LOST", false);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("pool에서 profile timeout을 넘겨 기다린 발송은 relay에 보내지 않고 시작 전 실패로 답한다")
|
||||
void aSendThatWaitedInThePoolPastTheTimeoutIsNotStartedAndNeverSent() throws Exception {
|
||||
// given
|
||||
CountDownLatch released = new CountDownLatch(1);
|
||||
CountDownLatch entered = new CountDownLatch(1);
|
||||
AtomicInteger handedToTheRelay = new AtomicInteger();
|
||||
SmtpDispatch slowRelay =
|
||||
message -> {
|
||||
handedToTheRelay.incrementAndGet();
|
||||
entered.countDown();
|
||||
try {
|
||||
released.await();
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
};
|
||||
ExecutorService pool = Executors.newSingleThreadExecutor();
|
||||
SmtpNotificationProviderAdapter adapter = adapter(slowRelay, pool, Duration.ofSeconds(1));
|
||||
var occupyingThePool = adapter.submit(submission());
|
||||
// The pool's one thread is in the relay before the second send is queued behind it.
|
||||
assertThat(entered.await(5, TimeUnit.SECONDS)).as("the first send reached the relay").isTrue();
|
||||
|
||||
// when
|
||||
Throwable waited;
|
||||
boolean drained;
|
||||
try {
|
||||
waited =
|
||||
catchThrowable(
|
||||
() -> adapter.submit(submission()).toCompletableFuture().get(5, TimeUnit.SECONDS));
|
||||
} finally {
|
||||
released.countDown();
|
||||
pool.shutdown();
|
||||
// Every queued task has run once this returns true, the abandoned send's included, so the
|
||||
// relay count below is final rather than read before the late send could have gone out.
|
||||
drained = pool.awaitTermination(5, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(
|
||||
List.of(
|
||||
waited.getCause().getClass(),
|
||||
waited.getCause().getMessage().contains("waited in the pool past the profile"),
|
||||
drained,
|
||||
handedToTheRelay.get()))
|
||||
.as(
|
||||
"nothing reached the relay, so the delivery is retried rather than reconciled, and"
|
||||
+ " the abandoned send does not go out late beside %s",
|
||||
occupyingThePool)
|
||||
.containsExactly(ProviderCallNotStartedException.class, true, true, 1);
|
||||
}
|
||||
|
||||
private SmtpNotificationProviderAdapter adapter(SmtpDispatch dispatch, Executor executor) {
|
||||
return adapter(dispatch, executor, CALL_TIMEOUT);
|
||||
}
|
||||
|
||||
private SmtpNotificationProviderAdapter adapter(
|
||||
SmtpDispatch dispatch, Executor executor, Duration callTimeout) {
|
||||
var properties =
|
||||
new SmtpProviderProperties(
|
||||
"smtp.example.com",
|
||||
@@ -155,12 +280,13 @@ class SmtpNotificationProviderAdapterTest {
|
||||
new SmtpFailureClassifier(),
|
||||
protector,
|
||||
properties,
|
||||
Executors.newSingleThreadExecutor(),
|
||||
executor,
|
||||
// The unconfigured resolver refuses on use, which is the honest default: an attachment the
|
||||
// deployment has no way to fetch must fail before the send, not silently vanish from it.
|
||||
new dev.caskeleton.application.notification.platform.dispatch.AttachmentIntegrityGuard(
|
||||
new dev.caskeleton.adapter.outbound.notification.platform.provider
|
||||
.UnconfiguredAttachmentResolver()));
|
||||
.UnconfiguredAttachmentResolver()),
|
||||
callTimeout);
|
||||
}
|
||||
|
||||
private ProviderSubmission submission() {
|
||||
|
||||
+44
-8
@@ -29,6 +29,8 @@ import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
|
||||
/** Exact-key S3 control store using native conditional PUT rather than check-then-act emulation. */
|
||||
public final class S3ConditionalObjectControlStore implements ObjectControlStore {
|
||||
|
||||
private static final String CONFLICT = "S3 control conditional mutation conflicted";
|
||||
|
||||
private static final int MAXIMUM_CONTROL_BYTES = 64 * 1024;
|
||||
|
||||
private final S3AsyncClient client;
|
||||
@@ -107,9 +109,11 @@ public final class S3ConditionalObjectControlStore implements ObjectControlStore
|
||||
true);
|
||||
return stored(response.eTag(), record);
|
||||
} catch (ObjectStorageProviderException exception) {
|
||||
if (exception.failure() == ObjectStorageProviderException.Failure.CONFLICT
|
||||
|| exception.failure() == ObjectStorageProviderException.Failure.PRECONDITION_FAILED) {
|
||||
throw new ObjectControlConflictException("S3 control conditional mutation conflicted");
|
||||
if (exception.failure() == ObjectStorageProviderException.Failure.PRECONDITION_FAILED) {
|
||||
return reconcilePreconditionFailure(key, record);
|
||||
}
|
||||
if (exception.failure() == ObjectStorageProviderException.Failure.CONFLICT) {
|
||||
throw new ObjectControlConflictException(CONFLICT);
|
||||
}
|
||||
if (exception.failure() == ObjectStorageProviderException.Failure.INDETERMINATE) {
|
||||
return reconcileLostResponse(key, record);
|
||||
@@ -118,20 +122,52 @@ public final class S3ConditionalObjectControlStore implements ObjectControlStore
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides a failed precondition by what the key holds now.
|
||||
*
|
||||
* <p>A 412 is not always somebody else's write. The SDK resends a conditional {@code PUT} whose
|
||||
* response it lost — the body is replayable — and when the first send landed, the resend fails
|
||||
* the very precondition the first one satisfied: {@code If-None-Match: *} now finds the object,
|
||||
* {@code If-Match} now finds a new ETag. Reported as a conflict, this store's own write was
|
||||
* called stale and the CAS above it gave up on a mutation that had succeeded (round9 review
|
||||
* 2026-11-12, RR9-01-19). The key holding exactly the record this call wrote is that case; any
|
||||
* other content, or none, is a real conflict.
|
||||
*
|
||||
* <p>"Exactly the record" is equality, not authorship: two writers that CAS the same version to
|
||||
* equal records — same revision, state and timestamp — are indistinguishable here, and both are
|
||||
* told they succeeded. The record's revision is what normally tells them apart; a caller that can
|
||||
* produce equal records from two places carries its own token in the record.
|
||||
*/
|
||||
private StoredControlRecord reconcilePreconditionFailure(
|
||||
String key, ObjectControlRecord expected) {
|
||||
return holdingExactly(key, expected)
|
||||
.orElseThrow(() -> new ObjectControlConflictException(CONFLICT));
|
||||
}
|
||||
|
||||
private StoredControlRecord reconcileLostResponse(String key, ObjectControlRecord expected) {
|
||||
Optional<StoredControlRecord> current = read(key);
|
||||
if (current.isPresent() && current.orElseThrow().record().equals(expected)) {
|
||||
return current.orElseThrow();
|
||||
}
|
||||
if (current.isPresent()) {
|
||||
throw new ObjectControlConflictException(
|
||||
"S3 control response loss resolved to different exact record evidence");
|
||||
return holding(current.orElseThrow(), expected)
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new ObjectControlConflictException(
|
||||
"S3 control response loss resolved to different exact record evidence"));
|
||||
}
|
||||
throw new ObjectStorageProviderException(
|
||||
ObjectStorageProviderException.Failure.INDETERMINATE,
|
||||
"S3 control mutation response was lost and no authoritative applied evidence exists");
|
||||
}
|
||||
|
||||
/** What the key holds now, when it is exactly {@code expected}. */
|
||||
private Optional<StoredControlRecord> holdingExactly(String key, ObjectControlRecord expected) {
|
||||
return read(key).flatMap(current -> holding(current, expected));
|
||||
}
|
||||
|
||||
private static Optional<StoredControlRecord> holding(
|
||||
StoredControlRecord current, ObjectControlRecord expected) {
|
||||
return current.record().equals(expected) ? Optional.of(current) : Optional.empty();
|
||||
}
|
||||
|
||||
private static StoredControlRecord stored(String eTag, ObjectControlRecord record) {
|
||||
if (eTag == null || eTag.isBlank() || eTag.length() > 256) {
|
||||
throw new ObjectStorageProviderException(
|
||||
|
||||
+59
-2
@@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.objectstorage.s3;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
@@ -26,6 +27,7 @@ import java.util.List;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.ArgumentMatchers;
|
||||
import software.amazon.awssdk.core.async.AsyncRequestBody;
|
||||
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
|
||||
import software.amazon.awssdk.core.async.SdkPublisher;
|
||||
@@ -99,6 +101,9 @@ class S3ConditionalObjectControlStoreTest {
|
||||
S3AsyncClient client = mock(S3AsyncClient.class);
|
||||
when(client.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class)))
|
||||
.thenReturn(CompletableFuture.failedFuture(S3Exception.builder().statusCode(412).build()));
|
||||
// Another writer's record holds the key: the precondition failed on somebody else's write.
|
||||
respondToGet(
|
||||
client, CODEC.encode(record(3, PublishedReferenceState.PUBLISHED)), "\"other-writer\"");
|
||||
S3ConditionalObjectControlStore store =
|
||||
new S3ConditionalObjectControlStore(client, awsBinding(), CODEC);
|
||||
|
||||
@@ -118,12 +123,64 @@ class S3ConditionalObjectControlStoreTest {
|
||||
.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aPreconditionFailedByTheResendOfOurOwnLandedWriteIsThatWrite() {
|
||||
// given
|
||||
// The SDK lost the first response and resent the conditional PUT; the first one had landed, so
|
||||
// the resend fails If-Match against the ETag it produced.
|
||||
S3AsyncClient client = mock(S3AsyncClient.class);
|
||||
ObjectReferenceRecord written = record(2, PublishedReferenceState.RETIRED);
|
||||
when(client.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class)))
|
||||
.thenReturn(CompletableFuture.failedFuture(S3Exception.builder().statusCode(412).build()));
|
||||
respondToGet(client, CODEC.encode(written), "\"after-our-write\"");
|
||||
S3ConditionalObjectControlStore store =
|
||||
new S3ConditionalObjectControlStore(client, awsBinding(), CODEC);
|
||||
|
||||
// when
|
||||
ObjectControlStore.StoredControlRecord stored =
|
||||
store.compareAndSet(
|
||||
KEY, new ObjectControlMutation(ObjectControlVersion.of("\"before\""), written));
|
||||
|
||||
// then
|
||||
assertThat(List.of(stored.record(), stored.version()))
|
||||
.containsExactly(written, ObjectControlVersion.of("\"after-our-write\""));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aPreconditionFailureOnAnAbsentKeyIsAConflict() {
|
||||
// given
|
||||
S3AsyncClient client = mock(S3AsyncClient.class);
|
||||
when(client.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class)))
|
||||
.thenReturn(CompletableFuture.failedFuture(S3Exception.builder().statusCode(412).build()));
|
||||
when(client.getObject(
|
||||
any(GetObjectRequest.class),
|
||||
ArgumentMatchers.<AsyncResponseTransformer<GetObjectResponse, ?>>any()))
|
||||
.thenReturn(CompletableFuture.failedFuture(S3Exception.builder().statusCode(404).build()));
|
||||
S3ConditionalObjectControlStore store =
|
||||
new S3ConditionalObjectControlStore(client, awsBinding(), CODEC);
|
||||
|
||||
// when
|
||||
Throwable refused =
|
||||
catchThrowable(
|
||||
() ->
|
||||
store.compareAndSet(
|
||||
KEY,
|
||||
new ObjectControlMutation(
|
||||
ObjectControlVersion.of("\"stale\""),
|
||||
record(2, PublishedReferenceState.RETIRED))));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.isInstanceOf(ObjectControlConflictException.class)
|
||||
.hasMessageContaining("conflicted");
|
||||
}
|
||||
|
||||
@Test
|
||||
void exact404IsTheOnlyAbsentRead() {
|
||||
S3AsyncClient client = mock(S3AsyncClient.class);
|
||||
when(client.getObject(
|
||||
any(GetObjectRequest.class),
|
||||
org.mockito.ArgumentMatchers.<AsyncResponseTransformer<GetObjectResponse, ?>>any()))
|
||||
ArgumentMatchers.<AsyncResponseTransformer<GetObjectResponse, ?>>any()))
|
||||
.thenReturn(CompletableFuture.failedFuture(S3Exception.builder().statusCode(404).build()));
|
||||
S3ConditionalObjectControlStore store =
|
||||
new S3ConditionalObjectControlStore(client, awsBinding(), CODEC);
|
||||
@@ -154,7 +211,7 @@ class S3ConditionalObjectControlStoreTest {
|
||||
private static void respondToGet(S3AsyncClient client, byte[] encoded, String eTag) {
|
||||
when(client.getObject(
|
||||
any(GetObjectRequest.class),
|
||||
org.mockito.ArgumentMatchers.<AsyncResponseTransformer<GetObjectResponse, ?>>any()))
|
||||
ArgumentMatchers.<AsyncResponseTransformer<GetObjectResponse, ?>>any()))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
AsyncResponseTransformer<GetObjectResponse, Object> transformer =
|
||||
|
||||
+6
@@ -118,6 +118,12 @@ public class JpaCleanupQueue implements CleanupQueuePort {
|
||||
settle(item, STATUS_FAILED, nextAttemptAt, reasonCode, clock.instant());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void release(CleanupItem item, Instant nextAttemptAt) {
|
||||
CallerTransaction.require("cleanup queue release");
|
||||
items.releaseClaim(item.cleanupId(), item.claimToken(), nextAttemptAt, clock.instant());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markAbandoned(CleanupItem item, String reasonCode) {
|
||||
CallerTransaction.require("cleanup queue markAbandoned");
|
||||
|
||||
+6
-13
@@ -7,7 +7,6 @@ import dev.caskeleton.application.fileserver.api.FileId;
|
||||
import dev.caskeleton.application.fileserver.recovery.ReconciliationStatus;
|
||||
import dev.caskeleton.application.fileserver.recovery.RecoveryQueuePort;
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import org.springframework.data.domain.Limit;
|
||||
@@ -15,10 +14,11 @@ import org.springframework.data.domain.Limit;
|
||||
/**
|
||||
* Durable recovery queue over {@code fs_recovery_item}.
|
||||
*
|
||||
* <p>An enqueue is an upsert: the same file reported twice updates the open item rather than adding
|
||||
* a second one. Reconciliation runs on a schedule and re-raises whatever it still cannot settle, so
|
||||
* an append-only queue would grow one row per sweep per unresolved file and bury the distinct
|
||||
* problems under repetitions of the same one.
|
||||
* <p>An enqueue is an upsert, one {@code INSERT … ON CONFLICT} statement: the same file reported
|
||||
* twice — in sequence or in two concurrent transactions — updates the open item rather than adding
|
||||
* a second one or failing on the open-item index. Reconciliation runs on a schedule and re-raises
|
||||
* whatever it still cannot settle, so an append-only queue would grow one row per sweep per
|
||||
* unresolved file and bury the distinct problems under repetitions of the same one.
|
||||
*
|
||||
* <p>Resolution keeps the outcome rather than deleting the row. {@code UNRESOLVED} and {@code
|
||||
* QUARANTINE_REQUIRED} are the two answers a human has to act on, and both are worthless if the
|
||||
@@ -26,8 +26,6 @@ import org.springframework.data.domain.Limit;
|
||||
*/
|
||||
public class JpaRecoveryQueue implements RecoveryQueuePort {
|
||||
|
||||
private static final String STATUS_PENDING = "PENDING";
|
||||
|
||||
private final FileserverRecoveryRepository items;
|
||||
private final Clock clock;
|
||||
|
||||
@@ -39,12 +37,7 @@ public class JpaRecoveryQueue implements RecoveryQueuePort {
|
||||
@Override
|
||||
public void enqueue(FileId fileId, String reasonCode) {
|
||||
CallerTransaction.require("recovery queue enqueue");
|
||||
Instant now = clock.instant();
|
||||
if (items.refreshPending(fileId.value(), reasonCode, now) > 0) {
|
||||
return;
|
||||
}
|
||||
items.save(
|
||||
new RecoveryItemEntity(UUID.randomUUID(), fileId.value(), reasonCode, STATUS_PENDING, now));
|
||||
items.upsertPending(UUID.randomUUID(), fileId.value(), reasonCode, clock.instant());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
-11
@@ -52,17 +52,6 @@ public class RecoveryItemEntity extends AssignedIdEntity {
|
||||
return recoveryId;
|
||||
}
|
||||
|
||||
public RecoveryItemEntity(
|
||||
UUID recoveryId, UUID fileId, String reasonCode, String status, Instant createdAt) {
|
||||
this.recoveryId = recoveryId;
|
||||
this.fileId = fileId;
|
||||
this.reasonCode = reasonCode;
|
||||
this.status = status;
|
||||
this.attempt = 0;
|
||||
this.createdAt = createdAt;
|
||||
this.updatedAt = createdAt;
|
||||
}
|
||||
|
||||
public UUID getRecoveryId() {
|
||||
return recoveryId;
|
||||
}
|
||||
|
||||
+26
@@ -63,6 +63,32 @@ public interface FileserverCleanupRepository extends JpaRepository<CleanupItemEn
|
||||
@Param("lastErrorCode") String lastErrorCode,
|
||||
@Param("now") Instant now);
|
||||
|
||||
/**
|
||||
* Gives a claim back without counting an attempt, for the worker that still holds it.
|
||||
*
|
||||
* <p>The item goes back to {@code PENDING} when it was never tried and to {@code FAILED} when an
|
||||
* earlier attempt was — the two statuses a claim takes from — and keeps its counter and its last
|
||||
* error.
|
||||
*/
|
||||
@Modifying(clearAutomatically = true, flushAutomatically = true)
|
||||
@Query(
|
||||
"""
|
||||
update CleanupItemEntity c
|
||||
set c.status = case when c.attempt = 0 then 'PENDING' else 'FAILED' end,
|
||||
c.nextAttemptAt = :nextAttemptAt,
|
||||
c.claimOwner = null,
|
||||
c.claimToken = null,
|
||||
c.leaseUntil = null,
|
||||
c.updatedAt = :now
|
||||
where c.cleanupId = :cleanupId
|
||||
and c.claimToken = :token
|
||||
""")
|
||||
int releaseClaim(
|
||||
@Param("cleanupId") UUID cleanupId,
|
||||
@Param("token") UUID token,
|
||||
@Param("nextAttemptAt") Instant nextAttemptAt,
|
||||
@Param("now") Instant now);
|
||||
|
||||
/**
|
||||
* Returns items whose claim has expired, for a reaper to take over.
|
||||
*
|
||||
|
||||
+21
-10
@@ -22,23 +22,34 @@ public interface FileserverRecoveryRepository extends JpaRepository<RecoveryItem
|
||||
List<RecoveryItemEntity> findPending(Limit limit);
|
||||
|
||||
/**
|
||||
* Re-raises an open item instead of adding a second one.
|
||||
* Opens an item for the file, or re-raises the open one instead of adding a second, in one
|
||||
* statement.
|
||||
*
|
||||
* <p>Reconciliation is retried on a schedule, so the same file reaches the queue repeatedly. One
|
||||
* open item per file keeps the queue a worklist rather than a failure log; the newest reason wins
|
||||
* because it describes the most recent evidence.
|
||||
*
|
||||
* <p>One {@code INSERT … ON CONFLICT} on the partial unique index {@code uq_fs_recovery_open}
|
||||
* rather than an update followed by an insert when it matched nothing. Two enqueues of the same
|
||||
* file in concurrent transactions both saw no open row, both inserted, and the second hit the
|
||||
* index — a unique violation that aborts its transaction ({@code 25P02}) and fails the sweep that
|
||||
* reported it. Here the second waits for the first to commit and then updates the row it wrote.
|
||||
*/
|
||||
@Modifying(clearAutomatically = true, flushAutomatically = true)
|
||||
@Query(
|
||||
"""
|
||||
update RecoveryItemEntity r
|
||||
set r.reasonCode = :reasonCode,
|
||||
r.attempt = r.attempt + 1,
|
||||
r.updatedAt = :now
|
||||
where r.fileId = :fileId
|
||||
and r.status = 'PENDING'
|
||||
""")
|
||||
int refreshPending(
|
||||
value =
|
||||
"""
|
||||
INSERT INTO fs_recovery_item
|
||||
(recovery_id, file_id, reason_code, status, attempt, created_at, updated_at)
|
||||
VALUES (:recoveryId, :fileId, :reasonCode, 'PENDING', 0, :now, :now)
|
||||
ON CONFLICT (file_id) WHERE status = 'PENDING'
|
||||
DO UPDATE SET reason_code = EXCLUDED.reason_code,
|
||||
attempt = fs_recovery_item.attempt + 1,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
""",
|
||||
nativeQuery = true)
|
||||
int upsertPending(
|
||||
@Param("recoveryId") UUID recoveryId,
|
||||
@Param("fileId") UUID fileId,
|
||||
@Param("reasonCode") String reasonCode,
|
||||
@Param("now") Instant now);
|
||||
|
||||
@@ -88,6 +88,9 @@ lock TTL)과 계약 표(트랜잭션 모드, SQLState → 오류 코드, vendor,
|
||||
`lease_token`을 올린다(`V17`). `markPublished`/`markFailed`/`markDead`는 `event_id` + `IN_FLIGHT` + owner + token에
|
||||
대한 조건부 UPDATE 하나이고 `APPLIED`/`STALE`을 돌려준다. 완료는 행을 읽어서 덮어쓰지 않는다. V2 outbox 세대는
|
||||
2026-09-25에 지웠다(`docs/capability-status.md`).
|
||||
- durable operation fencing: `claimNext`가 claim의 행 lock 아래에서 `lease_fence`를 올리고(`V23`) `DurableOperationClaim`
|
||||
으로 돌려준다. heartbeat·progress·succeed·fail은 operation id + `RUNNING` + owner + fence에 대한 조건부 UPDATE 하나다.
|
||||
같은 worker id가 다시 claim하면 앞 실행의 쓰기는 0행이다.
|
||||
- `TransactionPort` 구현(`SpringTransaction`) — 애플리케이션의 트랜잭션 의도와 Spring `PlatformTransactionManager` 사이의
|
||||
다리.
|
||||
- 감사 metadata 기반 + actor 이음매(`audit/AuditableEntity`, `audit/AuditContextPort`, `audit/DomainContextAuditContext`) —
|
||||
|
||||
@@ -389,6 +389,39 @@ PUBLISHED backlog 를 한 transaction 으로 지우면 그 사이 모든 row loc
|
||||
cadence 측정 필요). retention 한 값은 reaper-local 이라 `@Value` 로 받지만, canonical 6-property
|
||||
문서는 app-bootstrap `OutboxSettings` / `application.yml` 에 있다.
|
||||
|
||||
## liveevent — `JpaLiveEventReplay`와 `LiveEventReaper`
|
||||
|
||||
`app.realtime.replay.enabled=true`이고 vendor가 PostgreSQL이면 replay store와 reaper가 함께 조립된다. 설정은
|
||||
`LiveEventReplaySettings`가 `app.realtime.replay.*`를 엄격하게 묶는다. 모르는 키는 기동을 실패시킨다.
|
||||
|
||||
- `retention`: 한 event를 다시 받을 수 있는 시간. 기본값이 없다. 모듈 기본값이 `APP_REALTIME_REPLAY_RETENTION`(1시간)으로
|
||||
채운다. 0이나 음수는 기동 실패다.
|
||||
- `reaper-interval`: `LiveEventReaper`가 retention이 지난 행을 지우는 간격(기본 10분, 모듈 기본값이 같은 값을 적는다). 0이나
|
||||
음수는 기동 실패다. reaper 자체에는 일정이 없다. 조립 설정이 이 값으로 `reap()`을 등록한다(첫 실행도 한 간격 뒤).
|
||||
reaper를 직접 만드는 서비스는 일정도 직접 건다.
|
||||
|
||||
reaper는 `BatchedDeletes`로 1,000행씩, 배치마다 따로 commit하고, 배치가 덜 차면 멈춘다. 배치 문장은 `SKIP LOCKED`라 여러
|
||||
replica가 같이 돌아도 서로 기다리지 않는다. 트랜잭션 시한은 `SpringTransaction.DEFAULT_TIMEOUT_SECONDS`(30초)다. replay 읽기는
|
||||
`retained_until`로 지난 행을 이미 거르므로, reaper가 늦어도 지난 event를 내주지는 않는다. reaper가 없으면 테이블이 끝없이
|
||||
자란다(2026-11-13 전에는 지우는 곳이 없었다).
|
||||
|
||||
## operation — `JpaDurableOperationStore`
|
||||
|
||||
### claim과 fence (`V23`)
|
||||
|
||||
lease가 지난 실행은 아직 살아서 일을 끝내는 중일 수 있다. 예전 heartbeat·progress·succeed·fail은
|
||||
`lease_owner = :workerId`만 확인해서, 같은 worker id가 다시 claim하면(재시작한 worker, 한 프로세스의 두 실행)
|
||||
앞 실행의 결과가 넘겨받은 실행 위에 기록됐다(round9 리뷰 2026-11-12, RR9-01-02). outbox(`lease_token`),
|
||||
알림 수신자(`lease_fence`), reconciliation(`claim_fence`), fileserver cleanup(`claim_token`)과 같은 모양으로 맞췄다.
|
||||
|
||||
- **claim**: `claimNext(workerId, …)`가 row lock(`FOR UPDATE SKIP LOCKED`)을 쥔 상태에서 `lease_owner`,
|
||||
`lease_fence = lease_fence + 1`을 기록하고 `DurableOperationClaim(submission, workerId, fence)`을 돌려준다.
|
||||
- **쓰기**: heartbeat·progress·succeed·fail은 한 문장의 조건부 UPDATE다 —
|
||||
`WHERE operation_id = ? AND state = 'RUNNING' AND lease_owner = ? AND lease_fence = ?`(heartbeat는
|
||||
`lease_expires_at > now`도). 1행이면 적용, 0행이면 그 claim은 더 이상 이 operation을 쥐지 않는다.
|
||||
- fence는 줄지 않는다. reclaim·cancel·expire는 owner를 비우고 fence는 그대로 둔다. `V23` 전에 쓴 행은 0이고,
|
||||
claim은 1 이상을 나눠 주므로 fence 0을 쥔 worker는 없다.
|
||||
|
||||
## JPA 운영 capability 후보
|
||||
|
||||
`src/adapter/outbound/persistence-jpa/release/readiness-cards.yaml`이 capability와 독립 schema stream의
|
||||
@@ -533,7 +566,7 @@ provider 선택(flag → bean → registry)의 정본은 설정 코드 `lock/Dis
|
||||
반환 핸들은 `lock::unlock` 이다. lease TTL 이 `close()` 전에 만료된 `JdbcLockRegistry`
|
||||
의 경우, 내부 `JdbcLock.unlock()` 이 `ConcurrentModificationException` 을 던진다(row 가 이미
|
||||
회수됨). 이 어댑터는 여기서 일부러 잡지 않는다 — metered `distributedLockProvider` 데코레이터
|
||||
(컴포지션 루트)가 SI-LOCK-C5 계약을 소유한다: 로그 + `lock.lease.expired` metric 후 `close()` 에서
|
||||
(이 모듈의 `DistributedLockPersistenceConfig`가 감싼 `MeteredDistributedLock`)가 SI-LOCK-C5 계약을 소유한다: 로그 + `lock.lease.expired` metric 후 `close()` 에서
|
||||
정상 return 해, 만료가 호출자의 보호작업 예외를 가리지 않게 한다. in-process `DefaultLockRegistry`
|
||||
경로는 만료가 없어 그 `close()` 가 이 예외를 던질 수 없다.
|
||||
|
||||
@@ -555,11 +588,11 @@ provider 선택(flag → bean → registry)의 정본은 설정 코드 `lock/Dis
|
||||
`DistributedLock.tryLock(waitTime, leaseTtl)` 로 전달한다.
|
||||
|
||||
### `LockSettings`
|
||||
`app.lock.*` 에서 바인딩되는 yaml-only 기본값이다(`config/module-defaults/persistence-jpa.yml`).
|
||||
새 `APP_*` env 키가 아니므로 `env-keys.yaml` 엔트리가 필요 없다. `DistributedLockPersistenceConfig`의
|
||||
`@EnableConfigurationProperties`가 바인딩한다. cross-field invariant: `leaseTtl >= waitTime`
|
||||
이어야 한다 — TTL 이 waitTime 보다 먼저 만료되면 첫 holder 의 보호작업이 끝나기 전에 두 번째
|
||||
holder 가 락을 잡을 수 있다.
|
||||
`app.lock.lease-ttl` 하나를 바인딩하는 yaml-only 기본값이다(`config/module-defaults/persistence-jpa.yml`).
|
||||
새 `APP_*` env 키가 아니므로 `env-keys.yaml` 엔트리가 필요 없다. `DistributedLockPersistenceConfig#lockSettings`가
|
||||
`StrictSettingsBinder`로 바인딩해, `app.lock` 아래 모르는 키는 기동을 실패시킨다. 기다릴 시간은 호출자가
|
||||
`tryAcquire(key, waitTime, leaseTtl)`마다 넘긴다. 예전 `app.lock.wait-time`은 lease와 비교만 하고 읽는 곳이
|
||||
없었다(round9 리뷰 2026-11-12, RR9-01-07). 지금은 모르는 키라 설정해 두면 기동이 실패한다.
|
||||
|
||||
### `MeteredDistributedLock` (app-bootstrap에서 이동)
|
||||
- **`DistributedLockPort`를 감싸는 얇은 Micrometer 데코레이터다.** `tryAcquire` 호출마다
|
||||
|
||||
+40
-7
@@ -8,6 +8,8 @@ import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.IdempotencyRe
|
||||
import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.IdempotencyRecordJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.liveevent.JpaLiveEventReplay;
|
||||
import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReaper;
|
||||
import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReplaySettings;
|
||||
import dev.caskeleton.adapter.outbound.persistence.lock.DistributedLockPersistenceConfig;
|
||||
import dev.caskeleton.adapter.outbound.persistence.operation.DurableOperationJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.operation.JpaDurableOperationStore;
|
||||
@@ -26,7 +28,11 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.jdbc.core.JdbcOperations;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.SchedulingConfigurer;
|
||||
import org.springframework.scheduling.config.FixedDelayTask;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
|
||||
/**
|
||||
@@ -120,11 +126,15 @@ class JpaAdapterBeansConfig {
|
||||
}
|
||||
|
||||
/**
|
||||
* Retained live-event history: on when replay is switched on, and only on PostgreSQL (the
|
||||
* position allocator is an {@code INSERT … ON CONFLICT … RETURNING}). The retention has no
|
||||
* default on purpose: a replay window nobody chose is one nobody sized.
|
||||
* Retained live-event history and the reaper that removes it past retention: on when replay is
|
||||
* switched on, and only on PostgreSQL (the position allocator is an {@code INSERT … ON CONFLICT …
|
||||
* RETURNING}). The settings are {@link LiveEventReplaySettings}, bound strictly: the retention is
|
||||
* one hour unless the deployment sets {@code APP_REALTIME_REPLAY_RETENTION} (the module
|
||||
* defaults), the reaper runs every {@code reaper-interval}. Scheduling is enabled here because
|
||||
* the reaper is part of the store, as for the idempotency reaper.
|
||||
*/
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@EnableScheduling
|
||||
@ConditionalOnProperty(prefix = "app.realtime.replay", name = "enabled", havingValue = "true")
|
||||
@ConditionalOnProperty(
|
||||
prefix = "app.persistence",
|
||||
@@ -133,13 +143,36 @@ class JpaAdapterBeansConfig {
|
||||
matchIfMissing = true)
|
||||
static class LiveEventReplay {
|
||||
|
||||
@Bean
|
||||
LiveEventReplaySettings liveEventReplaySettings(Environment environment) {
|
||||
return LiveEventReplaySettings.bind(environment);
|
||||
}
|
||||
|
||||
@Bean
|
||||
JpaLiveEventReplay jpaLiveEventReplay(
|
||||
LiveEventJpaRepository repository,
|
||||
JdbcOperations jdbc,
|
||||
Clock clock,
|
||||
@Value("${app.realtime.replay.retention}") Duration retention) {
|
||||
return new JpaLiveEventReplay(repository, jdbc, clock, retention);
|
||||
LiveEventReplaySettings settings) {
|
||||
return new JpaLiveEventReplay(repository, jdbc, clock, settings.retention());
|
||||
}
|
||||
|
||||
@Bean
|
||||
LiveEventReaper liveEventReaper(LiveEventJpaRepository repository, Clock clock) {
|
||||
return new LiveEventReaper(repository, clock);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the reaper every bound {@code reaper-interval}, first one interval after startup. The
|
||||
* interval is handed over as the settings' value, so it has one default and a zero has already
|
||||
* failed startup in the settings rather than scheduling a loop.
|
||||
*/
|
||||
@Bean
|
||||
SchedulingConfigurer liveEventReaperSchedule(
|
||||
LiveEventReaper reaper, LiveEventReplaySettings settings) {
|
||||
Duration interval = settings.reaperInterval();
|
||||
return registrar ->
|
||||
registrar.addFixedDelayTask(new FixedDelayTask(reaper::reap, interval, interval));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,8 +187,8 @@ class JpaAdapterBeansConfig {
|
||||
static class IdempotencyExpiry {
|
||||
|
||||
@Bean
|
||||
IdempotencyReaper idempotencyReaper(IdempotencyRecordJpaRepository repository, Clock clock) {
|
||||
return new IdempotencyReaper(repository, clock);
|
||||
IdempotencyReaper idempotencyReaper(IdempotencyRecordJpaRepository repository) {
|
||||
return new IdempotencyReaper(repository);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+17
-6
@@ -23,26 +23,32 @@ import javax.sql.DataSource;
|
||||
public final class TenantDataSourceRegistry implements AutoCloseable {
|
||||
|
||||
private final Map<TenantId, DataSource> pools = new LinkedHashMap<>();
|
||||
private final Map<TenantId, Integer> poolSizes = new LinkedHashMap<>();
|
||||
private final Function<TenantId, DataSource> poolFactory;
|
||||
private final ToIntFunction<DataSource> poolSize;
|
||||
private final ToIntFunction<TenantId> plannedPoolSize;
|
||||
private final TenantPoolBudget budget;
|
||||
|
||||
/**
|
||||
* @param poolFactory opens a pool for a tenant from its secret-backed connection profile
|
||||
* @param poolSize reports a pool's maximum connection count, for the global budget
|
||||
* @param plannedPoolSize the maximum connection count a tenant's pool will have, read from its
|
||||
* connection profile before the pool is opened, for the global budget
|
||||
*/
|
||||
public TenantDataSourceRegistry(
|
||||
Function<TenantId, DataSource> poolFactory,
|
||||
ToIntFunction<DataSource> poolSize,
|
||||
ToIntFunction<TenantId> plannedPoolSize,
|
||||
TenantPoolBudget budget) {
|
||||
this.poolFactory = Objects.requireNonNull(poolFactory, "poolFactory");
|
||||
this.poolSize = Objects.requireNonNull(poolSize, "poolSize");
|
||||
this.plannedPoolSize = Objects.requireNonNull(plannedPoolSize, "plannedPoolSize");
|
||||
this.budget = Objects.requireNonNull(budget, "budget");
|
||||
}
|
||||
|
||||
/**
|
||||
* The tenant's pool, opening it if the budget allows.
|
||||
*
|
||||
* <p>The budget is checked against the pool's planned size before the pool is opened: a pool
|
||||
* opens its first connections as it is built, so one checked afterwards had already taken the
|
||||
* server past the ceiling, if only until it was closed again.
|
||||
*
|
||||
* @throws IllegalStateException when the global pool budget is exhausted
|
||||
*/
|
||||
public synchronized DataSource require(TenantId tenant) {
|
||||
@@ -51,9 +57,12 @@ public final class TenantDataSourceRegistry implements AutoCloseable {
|
||||
if (existing != null) {
|
||||
return existing;
|
||||
}
|
||||
budget.requireCapacity(pools.size(), allocatedConnections());
|
||||
budget.requirePoolSlot(pools.size());
|
||||
int planned = plannedPoolSize.applyAsInt(tenant);
|
||||
budget.requireCapacity(pools.size(), allocatedConnections(), planned);
|
||||
DataSource opened = poolFactory.apply(tenant);
|
||||
pools.put(tenant, opened);
|
||||
poolSizes.put(tenant, planned);
|
||||
return opened;
|
||||
}
|
||||
|
||||
@@ -72,6 +81,7 @@ public final class TenantDataSourceRegistry implements AutoCloseable {
|
||||
*/
|
||||
public synchronized void evict(TenantId tenant) {
|
||||
DataSource removed = pools.remove(tenant);
|
||||
poolSizes.remove(tenant);
|
||||
if (removed instanceof AutoCloseable closeable) {
|
||||
TenantDataSourceLifecycle.drainAndClose(closeable);
|
||||
}
|
||||
@@ -84,7 +94,7 @@ public final class TenantDataSourceRegistry implements AutoCloseable {
|
||||
|
||||
/** How many connections all open pools may hold between them. */
|
||||
public synchronized int allocatedConnections() {
|
||||
return pools.values().stream().mapToInt(poolSize).sum();
|
||||
return poolSizes.values().stream().mapToInt(Integer::intValue).sum();
|
||||
}
|
||||
|
||||
/** The tenants with an open pool. */
|
||||
@@ -99,5 +109,6 @@ public final class TenantDataSourceRegistry implements AutoCloseable {
|
||||
.map(AutoCloseable.class::cast)
|
||||
.forEach(TenantDataSourceLifecycle::drainAndClose);
|
||||
pools.clear();
|
||||
poolSizes.clear();
|
||||
}
|
||||
}
|
||||
|
||||
+32
-7
@@ -23,19 +23,44 @@ public record TenantPoolBudget(int maxOpenPools, int maxConnectionsAcrossPools)
|
||||
}
|
||||
|
||||
/**
|
||||
* Fails when another tenant pool would exceed either ceiling.
|
||||
* Fails when another tenant pool would exceed the pool count, before that pool is opened.
|
||||
*
|
||||
* @throws IllegalStateException when the budget is exhausted
|
||||
* @param openPools how many pools are open now
|
||||
* @throws IllegalStateException when no other pool fits
|
||||
*/
|
||||
public void requireCapacity(int openPools, int allocatedConnections) {
|
||||
if (openPools >= maxOpenPools || allocatedConnections >= maxConnectionsAcrossPools) {
|
||||
public void requirePoolSlot(int openPools) {
|
||||
if (openPools >= maxOpenPools) {
|
||||
throw new IllegalStateException("tenant pool budget exhausted");
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether another tenant pool fits inside both ceilings. */
|
||||
public boolean hasCapacity(int openPools, int allocatedConnections) {
|
||||
return openPools < maxOpenPools && allocatedConnections < maxConnectionsAcrossPools;
|
||||
/**
|
||||
* Fails when a new tenant pool of {@code newPoolConnections} would exceed either ceiling.
|
||||
*
|
||||
* <p>The new pool's own size is part of the question. Checking only that the pools already open
|
||||
* stay under the ceiling let a ten-connection pool open with one connection to spare, and the
|
||||
* total then stood nine above the ceiling the budget exists to keep (round9 review 2026-11-12,
|
||||
* RR9-01-26).
|
||||
*
|
||||
* @param openPools how many pools are open now, the new one not included
|
||||
* @param allocatedConnections how many connections the open pools may hold between them
|
||||
* @param newPoolConnections how many the new pool may hold
|
||||
* @throws IllegalStateException when the budget cannot take the new pool
|
||||
*/
|
||||
public void requireCapacity(int openPools, int allocatedConnections, int newPoolConnections) {
|
||||
if (!hasCapacity(openPools, allocatedConnections, newPoolConnections)) {
|
||||
throw new IllegalStateException("tenant pool budget exhausted");
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether a new tenant pool of {@code newPoolConnections} fits inside both ceilings. */
|
||||
public boolean hasCapacity(int openPools, int allocatedConnections, int newPoolConnections) {
|
||||
if (newPoolConnections < 1) {
|
||||
// The size is the server's own configuration, not the caller's input: a state error.
|
||||
throw new IllegalStateException("a tenant pool holds at least one connection");
|
||||
}
|
||||
return openPools < maxOpenPools
|
||||
&& (long) allocatedConnections + newPoolConnections <= maxConnectionsAcrossPools;
|
||||
}
|
||||
|
||||
/** The maximum number of tenants that can be open at once. */
|
||||
|
||||
+45
-2
@@ -1,5 +1,6 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.hibernate.stateless;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
@@ -19,23 +20,63 @@ import org.hibernate.Transaction;
|
||||
*
|
||||
* <p>Only registered work names may run, and each carries a declared row cap. An admin bulk path
|
||||
* without a cap is how a maintenance job becomes an outage.
|
||||
*
|
||||
* <p>The transaction has the same time bound as the application's write transactions (30 seconds
|
||||
* unless the constructor is given another, {@code docs/work/reference/decisions.md} 2.5). The row
|
||||
* cap can only be checked once the work reports what it touched, so without a bound a runaway
|
||||
* statement held its locks for as long as it ran before the cap could roll it back. Hibernate hands
|
||||
* every statement of the work the time left, and the database cancels the one that runs past it
|
||||
* (round9 review 2026-11-12, RR9-01-25).
|
||||
*/
|
||||
public final class HibernateStatelessSessionRunner {
|
||||
|
||||
/**
|
||||
* The default write-transaction bound, which this runner's transactions share: {@code
|
||||
* SpringTransaction.DEFAULT_TIMEOUT_SECONDS}. Restated rather than read from it, because the
|
||||
* {@code hibernate} package does not depend on {@code transaction} (JpaModuleBoundaryTest);
|
||||
* {@code StatelessRunnerTimeoutTest} holds the two equal. A deployment that moves its write bound
|
||||
* passes the same value to the three-argument constructor.
|
||||
*/
|
||||
public static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30);
|
||||
|
||||
private final SessionFactory sessionFactory;
|
||||
private final Set<StatelessWorkName> registeredWork;
|
||||
private final int timeoutSeconds;
|
||||
|
||||
/** A runner whose transactions are bounded by {@link #DEFAULT_TIMEOUT}. */
|
||||
public HibernateStatelessSessionRunner(
|
||||
SessionFactory sessionFactory, Set<StatelessWorkName> registeredWork) {
|
||||
this(sessionFactory, registeredWork, DEFAULT_TIMEOUT);
|
||||
}
|
||||
|
||||
/**
|
||||
* A runner whose transactions are bounded by {@code timeout}.
|
||||
*
|
||||
* @param timeout a whole, positive number of seconds; Hibernate's transaction timeout has no
|
||||
* finer unit, so a fraction would be dropped without a word
|
||||
*/
|
||||
public HibernateStatelessSessionRunner(
|
||||
SessionFactory sessionFactory, Set<StatelessWorkName> registeredWork, Duration timeout) {
|
||||
this.sessionFactory = Objects.requireNonNull(sessionFactory, "sessionFactory");
|
||||
this.registeredWork = Set.copyOf(Objects.requireNonNull(registeredWork, "registeredWork"));
|
||||
Objects.requireNonNull(timeout, "timeout");
|
||||
if (timeout.isNegative()
|
||||
|| timeout.isZero()
|
||||
|| timeout.toNanosPart() != 0
|
||||
|| timeout.toSeconds() > Integer.MAX_VALUE) {
|
||||
throw new IllegalStateException(
|
||||
"a stateless transaction timeout must be a positive whole number of seconds, was "
|
||||
+ timeout);
|
||||
}
|
||||
this.timeoutSeconds = Math.toIntExact(timeout.toSeconds());
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs {@code work} in its own stateless session and transaction.
|
||||
*
|
||||
* <p>The work reports how many rows it touched, and exceeding {@code maxRows} rolls the
|
||||
* transaction back.
|
||||
* transaction back. A statement still running when the transaction's time is up is cancelled, and
|
||||
* the failure rolls the transaction back.
|
||||
*
|
||||
* @param maxRows the row cap for this unit of work
|
||||
* @throws StatelessRowCapExceededException when the work touched more, after rolling back
|
||||
@@ -54,7 +95,9 @@ public final class HibernateStatelessSessionRunner {
|
||||
}
|
||||
|
||||
try (StatelessSession session = sessionFactory.openStatelessSession()) {
|
||||
Transaction transaction = session.beginTransaction();
|
||||
Transaction transaction = session.getTransaction();
|
||||
transaction.setTimeout(timeoutSeconds);
|
||||
transaction.begin();
|
||||
try {
|
||||
StatelessWorkResult<T> result = work.apply(session);
|
||||
if (result.affectedRows() > maxRows) {
|
||||
|
||||
+16
-7
@@ -1,32 +1,41 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.idempotency.jpa;
|
||||
|
||||
import java.time.Clock;
|
||||
import dev.caskeleton.adapter.outbound.persistence.transaction.BatchedDeletes;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* Periodically deletes idempotency rows whose {@code expires_at} has passed. The PostgreSQL store
|
||||
* moves {@code expires_at} forward on every completion and failure, and a claim is reset in place
|
||||
* rather than re-inserted, so this is the only thing that bounds table growth.
|
||||
*
|
||||
* <p>Deletes in batches of {@link #BATCH_SIZE}, each committed on its own, until a batch comes back
|
||||
* short — the outbox reaper's shape. Expiry is the database's clock, not this process's: the store
|
||||
* wrote {@code expires_at} from {@code clock_timestamp()} (see {@link
|
||||
* IdempotencyRecordJpaRepository#deleteExpiredBatch}).
|
||||
*/
|
||||
public class IdempotencyReaper {
|
||||
|
||||
/** Rows per delete statement and transaction. */
|
||||
static final int BATCH_SIZE = BatchedDeletes.BATCH_SIZE;
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(IdempotencyReaper.class);
|
||||
|
||||
private final IdempotencyRecordJpaRepository repository;
|
||||
private final Clock clock;
|
||||
|
||||
public IdempotencyReaper(IdempotencyRecordJpaRepository repository, Clock clock) {
|
||||
public IdempotencyReaper(IdempotencyRecordJpaRepository repository) {
|
||||
this.repository = repository;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes every record whose expiry has passed by the database clock; returns the count.
|
||||
*
|
||||
* <p>Not transactional itself: each batch is its own transaction.
|
||||
*/
|
||||
@Scheduled(fixedDelayString = "${app.capabilities.idempotency.reaper-interval:PT10M}")
|
||||
@Transactional
|
||||
public int reap() {
|
||||
int purged = repository.deleteExpired(clock.instant());
|
||||
int purged = BatchedDeletes.untilShort(repository::deleteExpiredBatch);
|
||||
if (purged > 0) {
|
||||
log.debug("idempotency reaper purged {} expired record(s)", purged);
|
||||
}
|
||||
|
||||
+31
-4
@@ -1,12 +1,13 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.idempotency.jpa;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.entity.IdempotencyRecordEntity;
|
||||
import java.time.Instant;
|
||||
import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction;
|
||||
import java.util.UUID;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.Repository;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* Spring Data repository for {@link IdempotencyRecordEntity}: the reaper's expiry purge, and
|
||||
@@ -18,8 +19,34 @@ import org.springframework.data.repository.query.Param;
|
||||
*/
|
||||
public interface IdempotencyRecordJpaRepository extends Repository<IdempotencyRecordEntity, UUID> {
|
||||
|
||||
/** Reaper / TTL boundary (§E): purge every record whose expiry is at/before {@code now}. */
|
||||
/**
|
||||
* Reaper / TTL boundary (§E): deletes at most {@code limit} records whose expiry has passed, in a
|
||||
* transaction of its own unless the caller already has one; returns the count.
|
||||
*
|
||||
* <p>Expiry is judged by the database clock ({@code now()}) because that is the clock that wrote
|
||||
* it: the PostgreSQL store sets {@code expires_at} from {@code clock_timestamp()}. Compared with
|
||||
* the application's clock instead, a host whose clock runs ahead purged a record whose lease or
|
||||
* replay window was still open.
|
||||
*
|
||||
* <p>Bounded for the reason the outbox purge is: a backlog of expired rows is what a reaper
|
||||
* outage leaves behind, and one unbounded {@code DELETE} over it is a long transaction holding
|
||||
* every deleted row's lock.
|
||||
*
|
||||
* <p>A same-key request reclaims an expired record in place (the store's reset). The delete must
|
||||
* not take that claim: the next retry would run the work again. Two things stop it. The subquery
|
||||
* locks the rows it picks and skips rows another transaction holds, so a record being reclaimed
|
||||
* is not picked. The expiry is also a condition of the {@code DELETE} itself, so if the lock
|
||||
* clause is ever lost — and a delete waits on the row and, under READ COMMITTED, re-checks only
|
||||
* its own predicate — the record it waited for is still kept once it is no longer expired.
|
||||
*/
|
||||
@Transactional(timeout = SpringTransaction.DEFAULT_TIMEOUT_SECONDS)
|
||||
@Modifying
|
||||
@Query("delete from IdempotencyRecordEntity e where e.expiresAt <= :now")
|
||||
int deleteExpired(@Param("now") Instant now);
|
||||
@Query(
|
||||
value =
|
||||
"DELETE FROM idempotency_record WHERE id IN ("
|
||||
+ "SELECT id FROM idempotency_record WHERE expires_at <= now()"
|
||||
+ " LIMIT :limit FOR UPDATE SKIP LOCKED)"
|
||||
+ " AND expires_at <= now()",
|
||||
nativeQuery = true)
|
||||
int deleteExpiredBatch(@Param("limit") int limit);
|
||||
}
|
||||
|
||||
-10
@@ -134,16 +134,6 @@ public final class JpaLiveEventReplay implements LiveEventReplayPort {
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes events past their retention.
|
||||
*
|
||||
* @return how many went
|
||||
*/
|
||||
public int sweepExpired() {
|
||||
requireCallerTransaction("sweepExpired");
|
||||
return repository.sweepExpired(clock.instant());
|
||||
}
|
||||
|
||||
private static void requireCallerTransaction(String operation) {
|
||||
CallerTransaction.require("live-event " + operation);
|
||||
}
|
||||
|
||||
+20
-3
@@ -1,6 +1,7 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.liveevent;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.liveevent.entity.LiveEventEntity;
|
||||
import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import org.springframework.data.domain.Limit;
|
||||
@@ -8,6 +9,7 @@ import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* Spring Data access for {@code live_event_log}.
|
||||
@@ -66,12 +68,27 @@ public interface LiveEventJpaRepository
|
||||
Long latestRetained(@Param("streamId") String streamId, @Param("now") Instant now);
|
||||
|
||||
/**
|
||||
* Removes expired rows.
|
||||
* Deletes at most {@code limit} rows retained no later than {@code now}, in a transaction of its
|
||||
* own unless the caller already has one; returns the count.
|
||||
*
|
||||
* <p>Bounded, and skipping rows another replica's reaper is deleting, for the reason the outbox
|
||||
* purge is bounded: a backlog of expired history is one long transaction holding every row lock
|
||||
* otherwise. {@code retained_until} is written from the application clock, so it is compared with
|
||||
* the application clock.
|
||||
*
|
||||
* @param now rows retained no later than this are removed
|
||||
* @param limit the most rows one statement removes
|
||||
* @return how many went
|
||||
*/
|
||||
// Bounded by the default transaction bound: a statement stuck behind a lock gives the connection
|
||||
// back instead of holding it until the pool is empty.
|
||||
@Transactional(timeout = SpringTransaction.DEFAULT_TIMEOUT_SECONDS)
|
||||
@Modifying
|
||||
@Query("delete from LiveEventEntity e where e.retainedUntil <= :now")
|
||||
int sweepExpired(@Param("now") Instant now);
|
||||
@Query(
|
||||
value =
|
||||
"DELETE FROM live_event_log WHERE (stream_id, position) IN ("
|
||||
+ "SELECT stream_id, position FROM live_event_log WHERE retained_until <= :now"
|
||||
+ " FOR UPDATE SKIP LOCKED LIMIT :limit)",
|
||||
nativeQuery = true)
|
||||
int deleteExpiredBatch(@Param("now") Instant now, @Param("limit") int limit);
|
||||
}
|
||||
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.liveevent;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.transaction.BatchedDeletes;
|
||||
import java.time.Clock;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Periodically deletes {@code live_event_log} rows past their retention.
|
||||
*
|
||||
* <p>A replay read already filters on {@code retained_until}, so an expired row is never served;
|
||||
* without this nothing ever removed one, and the table grew by every event ever published for as
|
||||
* long as replay was on (round9 review 2026-11-12, RR9-01-03). It is assembled with the replay
|
||||
* store, as the idempotency and outbox reapers are with theirs, rather than left to each service to
|
||||
* schedule: a duty that every deployment must remember is one some deployment forgets.
|
||||
*
|
||||
* <p>Deletes in batches of {@link #BATCH_SIZE}, each committed on its own, until a batch comes back
|
||||
* short. Every replica runs it; the batch skips rows another replica's batch holds.
|
||||
*
|
||||
* <p>It carries no schedule of its own. The replay configuration registers {@link #reap()} at the
|
||||
* bound {@link LiveEventReplaySettings#reaperInterval()}, so the interval reaches the schedule as a
|
||||
* value rather than through a bean name an expression looks up. A deployment that builds a reaper
|
||||
* itself schedules it too.
|
||||
*/
|
||||
public class LiveEventReaper {
|
||||
|
||||
/** Rows per delete statement and transaction. */
|
||||
static final int BATCH_SIZE = BatchedDeletes.BATCH_SIZE;
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(LiveEventReaper.class);
|
||||
|
||||
private final LiveEventJpaRepository repository;
|
||||
private final Clock clock;
|
||||
|
||||
public LiveEventReaper(LiveEventJpaRepository repository, Clock clock) {
|
||||
this.repository = repository;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes every row retained no later than now; returns the count.
|
||||
*
|
||||
* <p>Not transactional itself: each batch is its own transaction.
|
||||
*/
|
||||
public int reap() {
|
||||
int purged =
|
||||
BatchedDeletes.untilShort(limit -> repository.deleteExpiredBatch(clock.instant(), limit));
|
||||
if (purged > 0) {
|
||||
log.debug("live event reaper purged {} expired row(s)", purged);
|
||||
}
|
||||
return purged;
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.liveevent;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.support.StrictSettingsBinder;
|
||||
import java.time.Duration;
|
||||
import org.springframework.core.env.Environment;
|
||||
|
||||
/**
|
||||
* The replay store's settings, bound strictly from {@code app.realtime.replay.*}: a misspelt key
|
||||
* fails startup rather than leaving a default in force.
|
||||
*
|
||||
* <p>The reaper's interval was read only by the {@code @Scheduled} placeholder, with its default
|
||||
* written twice and nothing to refuse a zero (round9 review 2026-11-12). It is bound here with the
|
||||
* rest of the store's settings and read from this record by the reaper's schedule.
|
||||
*
|
||||
* @param enabled whether the replay store is assembled; the configuration's condition reads it
|
||||
* @param retention how long an event stays replayable. No default: how long a reconnecting client
|
||||
* may resume is the deployment's decision, made against what it is willing to store
|
||||
* @param reaperInterval how often {@link LiveEventReaper} deletes rows past retention; 10 minutes
|
||||
* unless set, which the module defaults mirror
|
||||
*/
|
||||
public record LiveEventReplaySettings(
|
||||
boolean enabled, Duration retention, Duration reaperInterval) {
|
||||
|
||||
static final String PREFIX = "app.realtime.replay";
|
||||
|
||||
private static final Duration DEFAULT_REAPER_INTERVAL = Duration.ofMinutes(10);
|
||||
|
||||
public LiveEventReplaySettings {
|
||||
if (retention == null) {
|
||||
throw new IllegalArgumentException(
|
||||
"app.realtime.replay.retention must be set when replay is enabled: how long a client may"
|
||||
+ " resume is the deployment's decision");
|
||||
}
|
||||
if (retention.isNegative() || retention.isZero()) {
|
||||
throw new IllegalArgumentException(
|
||||
"app.realtime.replay.retention must be positive, was " + retention);
|
||||
}
|
||||
if (reaperInterval == null) {
|
||||
reaperInterval = DEFAULT_REAPER_INTERVAL;
|
||||
} else if (reaperInterval.isNegative() || reaperInterval.isZero()) {
|
||||
throw new IllegalArgumentException(
|
||||
"app.realtime.replay.reaper-interval must be positive, was " + reaperInterval);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Binds {@link #PREFIX} strictly.
|
||||
*
|
||||
* @param environment the environment to bind from
|
||||
* @return the bound settings
|
||||
*/
|
||||
public static LiveEventReplaySettings bind(Environment environment) {
|
||||
return StrictSettingsBinder.bind(environment, PREFIX, LiveEventReplaySettings.class);
|
||||
}
|
||||
}
|
||||
+9
-22
@@ -5,41 +5,28 @@ import java.time.Duration;
|
||||
import org.springframework.core.env.Environment;
|
||||
|
||||
/**
|
||||
* Distributed-lock tuning knobs bound from {@code app.lock.*} (yaml-only defaults, not {@code
|
||||
* APP_*} env keys). {@code waitTime} = max wait before timeout (default 3s); {@code leaseTtl} = max
|
||||
* lock hold before auto-expiry (default 30s, must be ≥ {@code waitTime}). See README "lock".
|
||||
* Distributed-lock tuning bound from {@code app.lock.*} (yaml-only defaults, not {@code APP_*} env
|
||||
* keys). {@code leaseTtl} = the longest a lock may be held before the provider expires it (default
|
||||
* 30s); a caller's {@code tryAcquire} may ask for that long or less. See README "lock".
|
||||
*
|
||||
* <p>There is no wait setting. How long to wait is every caller's own argument to {@code
|
||||
* tryAcquire}; an {@code app.lock.wait-time} used to be bound here, checked against the lease, and
|
||||
* read by nothing (round9 review 2026-11-12, RR9-01-07). It now fails startup as an unknown key, so
|
||||
* a deployment that set it learns that it never did anything.
|
||||
*/
|
||||
public record LockSettings(Duration waitTime, Duration leaseTtl) {
|
||||
public record LockSettings(Duration leaseTtl) {
|
||||
|
||||
static final String PREFIX = "app.lock";
|
||||
|
||||
private static final Duration DEFAULT_WAIT_TIME = Duration.ofSeconds(3);
|
||||
private static final Duration DEFAULT_LEASE_TTL = Duration.ofSeconds(30);
|
||||
|
||||
public LockSettings {
|
||||
if (waitTime == null) {
|
||||
waitTime = DEFAULT_WAIT_TIME;
|
||||
} else if (waitTime.isNegative() || waitTime.isZero()) {
|
||||
throw new IllegalArgumentException(
|
||||
"app.lock.wait-time must be positive (> 0), was " + waitTime);
|
||||
}
|
||||
if (leaseTtl == null) {
|
||||
leaseTtl = DEFAULT_LEASE_TTL;
|
||||
} else if (leaseTtl.isNegative() || leaseTtl.isZero()) {
|
||||
throw new IllegalArgumentException(
|
||||
"app.lock.lease-ttl must be positive (> 0), was " + leaseTtl);
|
||||
}
|
||||
// Cross-field invariant: a lease shorter than the max wait is nonsensical.
|
||||
// If the TTL fires before the waitTime elapses, a second holder could acquire
|
||||
// the lock before the first holder's protected work is complete.
|
||||
if (leaseTtl.compareTo(waitTime) < 0) {
|
||||
throw new IllegalArgumentException(
|
||||
"app.lock.lease-ttl ("
|
||||
+ leaseTtl
|
||||
+ ") must be >= app.lock.wait-time ("
|
||||
+ waitTime
|
||||
+ ") — a lease shorter than the max wait is nonsensical");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+18
-6
@@ -17,10 +17,11 @@ import org.springframework.data.repository.query.Param;
|
||||
* claim-then-check has every worker fetch the same head row and all but one retry, and a plain
|
||||
* {@code FOR UPDATE} makes them queue behind each other so the pool serialises.
|
||||
*
|
||||
* <p>Every state-changing statement carries the owning worker in its {@code WHERE} clause and
|
||||
* returns the affected count. A worker whose lease lapsed while it was still working must not be
|
||||
* able to record a result over the worker that took over; the update simply matches nothing, and
|
||||
* the caller learns it lost the lease from the count.
|
||||
* <p>Every state-changing statement carries the owning worker and the fence of its claim in its
|
||||
* {@code WHERE} clause and returns the affected count. A run whose lease lapsed while it was still
|
||||
* working must not be able to record a result over the run that took over — even when both runs use
|
||||
* the same worker id, which only the fence tells apart; the update simply matches nothing, and the
|
||||
* caller learns it lost the lease from the count.
|
||||
*/
|
||||
public interface DurableOperationJpaRepository extends Repository<DurableOperationEntity, String> {
|
||||
|
||||
@@ -79,6 +80,8 @@ public interface DurableOperationJpaRepository extends Repository<DurableOperati
|
||||
* <p>Runnable is PENDING, or RUNNING whose lease has lapsed. The second half is the crash
|
||||
* recovery path: without it an operation claimed by a worker that then died is never picked up
|
||||
* again, and the queue stops draining while every row still reads RUNNING.
|
||||
*
|
||||
* @return one row of {@code [operation_id, lease_fence]} when a row was claimed, none otherwise
|
||||
*/
|
||||
@Modifying(clearAutomatically = true, flushAutomatically = true)
|
||||
@Query(
|
||||
@@ -90,6 +93,7 @@ public interface DurableOperationJpaRepository extends Repository<DurableOperati
|
||||
lease_acquired_at = :now,
|
||||
lease_expires_at = :leaseExpiresAt,
|
||||
started_at = COALESCE(started_at, :now),
|
||||
lease_fence = lease_fence + 1,
|
||||
row_version = row_version + 1
|
||||
WHERE operation_id = (
|
||||
SELECT operation_id
|
||||
@@ -100,10 +104,10 @@ public interface DurableOperationJpaRepository extends Repository<DurableOperati
|
||||
ORDER BY submitted_at
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1)
|
||||
RETURNING operation_id
|
||||
RETURNING operation_id, lease_fence
|
||||
""",
|
||||
nativeQuery = true)
|
||||
List<String> claimNext(
|
||||
List<Object[]> claimNext(
|
||||
@Param("workerId") String workerId,
|
||||
@Param("leaseExpiresAt") Instant leaseExpiresAt,
|
||||
@Param("now") Instant now);
|
||||
@@ -119,12 +123,14 @@ public interface DurableOperationJpaRepository extends Repository<DurableOperati
|
||||
WHERE operation_id = :operationId
|
||||
AND state = 'RUNNING'
|
||||
AND lease_owner = :workerId
|
||||
AND lease_fence = :fence
|
||||
AND lease_expires_at > :now
|
||||
""",
|
||||
nativeQuery = true)
|
||||
int heartbeat(
|
||||
@Param("operationId") String operationId,
|
||||
@Param("workerId") String workerId,
|
||||
@Param("fence") long fence,
|
||||
@Param("leaseExpiresAt") Instant leaseExpiresAt,
|
||||
@Param("now") Instant now);
|
||||
|
||||
@@ -141,11 +147,13 @@ public interface DurableOperationJpaRepository extends Repository<DurableOperati
|
||||
WHERE operation_id = :operationId
|
||||
AND state = 'RUNNING'
|
||||
AND lease_owner = :workerId
|
||||
AND lease_fence = :fence
|
||||
""",
|
||||
nativeQuery = true)
|
||||
int reportProgress(
|
||||
@Param("operationId") String operationId,
|
||||
@Param("workerId") String workerId,
|
||||
@Param("fence") long fence,
|
||||
@Param("completed") long completed,
|
||||
@Param("total") Long total,
|
||||
@Param("phase") String phase);
|
||||
@@ -166,11 +174,13 @@ public interface DurableOperationJpaRepository extends Repository<DurableOperati
|
||||
WHERE operation_id = :operationId
|
||||
AND state = 'RUNNING'
|
||||
AND lease_owner = :workerId
|
||||
AND lease_fence = :fence
|
||||
""",
|
||||
nativeQuery = true)
|
||||
int succeed(
|
||||
@Param("operationId") String operationId,
|
||||
@Param("workerId") String workerId,
|
||||
@Param("fence") long fence,
|
||||
@Param("resultReference") String resultReference,
|
||||
@Param("completedAt") Instant completedAt);
|
||||
|
||||
@@ -192,11 +202,13 @@ public interface DurableOperationJpaRepository extends Repository<DurableOperati
|
||||
WHERE operation_id = :operationId
|
||||
AND state = 'RUNNING'
|
||||
AND lease_owner = :workerId
|
||||
AND lease_fence = :fence
|
||||
""",
|
||||
nativeQuery = true)
|
||||
int fail(
|
||||
@Param("operationId") String operationId,
|
||||
@Param("workerId") String workerId,
|
||||
@Param("fence") long fence,
|
||||
@Param("code") String code,
|
||||
@Param("detail") String detail,
|
||||
@Param("retryable") boolean retryable,
|
||||
|
||||
+42
-32
@@ -4,6 +4,7 @@ import dev.caskeleton.adapter.outbound.persistence.operation.entity.DurableOpera
|
||||
import dev.caskeleton.adapter.outbound.persistence.transaction.CallerTransaction;
|
||||
import dev.caskeleton.application.idempotency.RequestFingerprint;
|
||||
import dev.caskeleton.application.operation.DurableOperation;
|
||||
import dev.caskeleton.application.operation.DurableOperationClaim;
|
||||
import dev.caskeleton.application.operation.DurableOperationId;
|
||||
import dev.caskeleton.application.operation.DurableOperationState;
|
||||
import dev.caskeleton.application.operation.DurableOperationStorePort;
|
||||
@@ -81,42 +82,52 @@ public class JpaDurableOperationStore implements DurableOperationStorePort {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<DurableOperationSubmission> claimNext(
|
||||
public Optional<DurableOperationClaim> claimNext(
|
||||
String workerId, Duration leaseDuration, Instant now) {
|
||||
requireCallerTransaction("claimNext");
|
||||
List<String> claimed = repository.claimNext(workerId, now.plus(leaseDuration), now);
|
||||
List<Object[]> claimed = repository.claimNext(workerId, now.plus(leaseDuration), now);
|
||||
if (claimed.isEmpty()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
String operationId = (String) claimed.get(0)[0];
|
||||
long fence = ((Number) claimed.get(0)[1]).longValue();
|
||||
return repository
|
||||
.findById(claimed.get(0))
|
||||
.findById(operationId)
|
||||
.map(
|
||||
entity ->
|
||||
new DurableOperationSubmission(
|
||||
new DurableOperationId(entity.getOperationId()),
|
||||
entity.getOperationName(),
|
||||
entity.getPrincipal(),
|
||||
entity.getTenant().isEmpty() ? null : entity.getTenant(),
|
||||
new RequestFingerprint(entity.getRequestHash()),
|
||||
entity.getPayload(),
|
||||
entity.getSubmittedAt(),
|
||||
Duration.between(entity.getSubmittedAt(), entity.getExpiresAt())));
|
||||
new DurableOperationClaim(
|
||||
new DurableOperationSubmission(
|
||||
new DurableOperationId(entity.getOperationId()),
|
||||
entity.getOperationName(),
|
||||
entity.getPrincipal(),
|
||||
entity.getTenant().isEmpty() ? null : entity.getTenant(),
|
||||
new RequestFingerprint(entity.getRequestHash()),
|
||||
entity.getPayload(),
|
||||
entity.getSubmittedAt(),
|
||||
Duration.between(entity.getSubmittedAt(), entity.getExpiresAt())),
|
||||
workerId,
|
||||
fence));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean heartbeat(
|
||||
DurableOperationId operationId, String workerId, Duration leaseDuration, Instant now) {
|
||||
public boolean heartbeat(DurableOperationClaim claim, Duration leaseDuration, Instant now) {
|
||||
requireCallerTransaction("heartbeat");
|
||||
return repository.heartbeat(operationId.value(), workerId, now.plus(leaseDuration), now) == 1;
|
||||
return repository.heartbeat(
|
||||
claim.operationId().value(),
|
||||
claim.workerId(),
|
||||
claim.fence(),
|
||||
now.plus(leaseDuration),
|
||||
now)
|
||||
== 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean reportProgress(
|
||||
DurableOperationId operationId, String workerId, OperationProgressSnapshot progress) {
|
||||
public boolean reportProgress(DurableOperationClaim claim, OperationProgressSnapshot progress) {
|
||||
requireCallerTransaction("reportProgress");
|
||||
return repository.reportProgress(
|
||||
operationId.value(),
|
||||
workerId,
|
||||
claim.operationId().value(),
|
||||
claim.workerId(),
|
||||
claim.fence(),
|
||||
progress.completedUnits(),
|
||||
progress.totalUnits().orElse(null),
|
||||
progress.phase().orElse(null))
|
||||
@@ -124,25 +135,24 @@ public class JpaDurableOperationStore implements DurableOperationStorePort {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean succeed(
|
||||
DurableOperationId operationId,
|
||||
String workerId,
|
||||
String resultReference,
|
||||
Instant completedAt) {
|
||||
public boolean succeed(DurableOperationClaim claim, String resultReference, Instant completedAt) {
|
||||
requireCallerTransaction("succeed");
|
||||
return repository.succeed(operationId.value(), workerId, resultReference, completedAt) == 1;
|
||||
return repository.succeed(
|
||||
claim.operationId().value(),
|
||||
claim.workerId(),
|
||||
claim.fence(),
|
||||
resultReference,
|
||||
completedAt)
|
||||
== 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean fail(
|
||||
DurableOperationId operationId,
|
||||
String workerId,
|
||||
OperationFailure failure,
|
||||
Instant completedAt) {
|
||||
public boolean fail(DurableOperationClaim claim, OperationFailure failure, Instant completedAt) {
|
||||
requireCallerTransaction("fail");
|
||||
return repository.fail(
|
||||
operationId.value(),
|
||||
workerId,
|
||||
claim.operationId().value(),
|
||||
claim.workerId(),
|
||||
claim.fence(),
|
||||
failure.code(),
|
||||
failure.detail(),
|
||||
failure.retryable(),
|
||||
|
||||
+1
-1
@@ -378,7 +378,7 @@ public class JpaOutboxStore
|
||||
OutboxEventStatus.valueOf(entity.getStatus()),
|
||||
entity.getAttemptCount(),
|
||||
entity.getDeferredAttemptCount());
|
||||
} catch (IllegalArgumentException exception) {
|
||||
} catch (IllegalArgumentException | IllegalStateException exception) {
|
||||
throw new IllegalStateException(
|
||||
"invalid canonical outbox metadata for eventId=" + entity.getEventId(), exception);
|
||||
}
|
||||
|
||||
+4
-2
@@ -1,6 +1,7 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.outbox.jpa;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.outbox.jpa.entity.OutboxEventEntity;
|
||||
import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
@@ -34,9 +35,10 @@ public interface OutboxEventJpaRepository extends Repository<OutboxEventEntity,
|
||||
*
|
||||
* <p>Bounded because a backlog of published rows is exactly what a relay outage leaves behind,
|
||||
* and one unbounded {@code DELETE} over it is a long transaction holding every deleted row's
|
||||
* lock. The key subquery is plain SQL that PostgreSQL and H2 both run.
|
||||
* lock. The key subquery is plain SQL that PostgreSQL and H2 both run. The transaction has the
|
||||
* default bound, as the other reapers' do.
|
||||
*/
|
||||
@Transactional
|
||||
@Transactional(timeout = SpringTransaction.DEFAULT_TIMEOUT_SECONDS)
|
||||
@Modifying
|
||||
@Query(
|
||||
value =
|
||||
|
||||
+3
-7
@@ -1,5 +1,6 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.outbox.jpa;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.transaction.BatchedDeletes;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
@@ -18,7 +19,7 @@ public class OutboxReaper {
|
||||
|
||||
/** Rows per delete statement and transaction. */
|
||||
// ponytail: fixed batch size; make it a setting if a deployment's backlog needs another value.
|
||||
static final int BATCH_SIZE = 1_000;
|
||||
static final int BATCH_SIZE = BatchedDeletes.BATCH_SIZE;
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(OutboxReaper.class);
|
||||
|
||||
@@ -41,12 +42,7 @@ public class OutboxReaper {
|
||||
@Scheduled(fixedDelayString = "${app.outbox.reaper-interval:PT10M}")
|
||||
public int reap() {
|
||||
Instant cutoff = clock.instant().minus(retention);
|
||||
int purged = 0;
|
||||
int deleted;
|
||||
do {
|
||||
deleted = repository.deletePublishedBatch(cutoff, BATCH_SIZE);
|
||||
purged += deleted;
|
||||
} while (deleted == BATCH_SIZE);
|
||||
int purged = BatchedDeletes.untilShort(limit -> repository.deletePublishedBatch(cutoff, limit));
|
||||
if (purged > 0) {
|
||||
log.debug("outbox reaper purged {} published row(s)", purged);
|
||||
}
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.transaction;
|
||||
|
||||
import java.util.function.IntUnaryOperator;
|
||||
|
||||
/**
|
||||
* Deletes a backlog in bounded batches, one transaction each, until a batch comes back short.
|
||||
*
|
||||
* <p>Every reaper here has the same reason to: a backlog of expired rows is what a reaper outage
|
||||
* leaves behind, and one unbounded {@code DELETE} over it is a long transaction holding every
|
||||
* deleted row's lock. The outbox, idempotency and live-event reapers each held their own copy of
|
||||
* this loop; one copy is the one that cannot drift.
|
||||
*/
|
||||
public final class BatchedDeletes {
|
||||
|
||||
/** The most rows one batch deletes. */
|
||||
public static final int BATCH_SIZE = 1_000;
|
||||
|
||||
private BatchedDeletes() {}
|
||||
|
||||
/**
|
||||
* Runs {@code batch} until it deletes fewer than {@link #BATCH_SIZE} rows.
|
||||
*
|
||||
* @param batch deletes at most its argument's number of rows, in a transaction of its own, and
|
||||
* returns how many it deleted
|
||||
* @return how many rows all the batches deleted
|
||||
*/
|
||||
public static int untilShort(IntUnaryOperator batch) {
|
||||
int purged = 0;
|
||||
int deleted;
|
||||
do {
|
||||
deleted = batch.applyAsInt(BATCH_SIZE);
|
||||
purged += deleted;
|
||||
} while (deleted == BATCH_SIZE);
|
||||
return purged;
|
||||
}
|
||||
}
|
||||
+12
-1
@@ -115,6 +115,15 @@ public final class SpringTransaction implements TransactionPort {
|
||||
return execute(requiresNewTemplate, action);
|
||||
}
|
||||
|
||||
/**
|
||||
* The default bound, in whole seconds, of every transaction this leaf opens: {@link
|
||||
* Timeouts#DEFAULT}, and the reapers' own transactions, whose {@code @Transactional(timeout)}
|
||||
* needs a compile-time constant. A deployment's {@code app.persistence-jpa.transaction.*} moves
|
||||
* the first and not the second; a reaper deletes one bounded batch per transaction, so the
|
||||
* default is ample for it whatever the application's units of work need.
|
||||
*/
|
||||
public static final int DEFAULT_TIMEOUT_SECONDS = 30;
|
||||
|
||||
/**
|
||||
* How long one {@link SpringTransaction} write or read transaction may run (review 04-10).
|
||||
*
|
||||
@@ -140,7 +149,9 @@ public final class SpringTransaction implements TransactionPort {
|
||||
|
||||
/** 30 seconds each; see the class comment for why. */
|
||||
public static final Timeouts DEFAULT =
|
||||
new Timeouts(Duration.ofSeconds(30), Duration.ofSeconds(30));
|
||||
new Timeouts(
|
||||
Duration.ofSeconds(DEFAULT_TIMEOUT_SECONDS),
|
||||
Duration.ofSeconds(DEFAULT_TIMEOUT_SECONDS));
|
||||
|
||||
public Timeouts {
|
||||
requireWholePositiveSeconds(write, "write");
|
||||
|
||||
+7
-3
@@ -199,10 +199,10 @@ app:
|
||||
# true | false (whether to run Flyway migrations automatically at startup)
|
||||
migration-on-startup: ${APP_MIGRATION_ON_STARTUP:true}
|
||||
# feature-distributed-lock-contract D5 — distributed lock acquisition contract (try-lock +
|
||||
# finite wait + lease TTL). Bound to LockSettings. Plain values, not APP_* env keys; the code
|
||||
# defaults in LockSettings mirror these.
|
||||
# finite wait + lease TTL). Bound to LockSettings. A plain value, not an APP_* env key; the code
|
||||
# default in LockSettings mirrors it. The wait is each caller's tryAcquire argument, so there is
|
||||
# no wait setting (app.lock.wait-time was read by nothing and is now an unknown key).
|
||||
lock:
|
||||
wait-time: 3s
|
||||
lease-ttl: 30s
|
||||
realtime:
|
||||
replay:
|
||||
@@ -210,6 +210,10 @@ app:
|
||||
# events into the store and exposes replay/resume semantics.
|
||||
enabled: ${APP_REALTIME_REPLAY_ENABLED:false}
|
||||
retention: ${APP_REALTIME_REPLAY_RETENTION:1h}
|
||||
# ISO-8601 duration — how often LiveEventReaper deletes rows past retention, 1 000 a batch.
|
||||
# A plain value like the outbox reaper's interval. Bound strictly by LiveEventReplaySettings,
|
||||
# whose default mirrors it; zero or negative fails startup.
|
||||
reaper-interval: PT10M
|
||||
outbox:
|
||||
# feature-domain-event-outbox-contract I11 — the tuning values are literal defaults;
|
||||
# NO env placeholders (spec: 신규 env key 없음). Declared, validated and strictly bound by
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
-- A fence on the durable operation lease.
|
||||
--
|
||||
-- Until now a heartbeat, a progress report and a completion checked only lease_owner. A worker that
|
||||
-- restarts under the same id, or one process running two executions under one id, claims the
|
||||
-- operation again, and the earlier run — still alive and still finishing — then matches the same
|
||||
-- lease_owner and records its result over the run that took over. The outbox (lease_token), the
|
||||
-- notification recipient lease (lease_fence), the reconciliation claim (claim_fence) and the
|
||||
-- fileserver claim (claim_token) all fence; this was the one store that did not (round9 review
|
||||
-- 2026-11-12, RR9-01-02).
|
||||
--
|
||||
-- Every claim increments lease_fence and returns it; every later write names the fence it holds in
|
||||
-- its WHERE clause. NOT NULL DEFAULT 0 is a constant default, so PostgreSQL 11+ records it in the
|
||||
-- catalog and rewrites no row; rows written before this keep 0 until their next claim, and no
|
||||
-- worker holds fence 0 because a claim hands out 1 or more.
|
||||
|
||||
-- Bound the wait for the lock this migration takes (ACCESS EXCLUSIVE on durable_operation, held
|
||||
-- until it commits). Unbounded, it queues behind any open transaction that has touched the table,
|
||||
-- and every submit, poll and claim queues behind the migration. Five seconds, then the migration
|
||||
-- fails, rolls back whole and the database stays at V22: deploy again once that transaction has
|
||||
-- ended (docs/jpa/migration-guide.md "Bound the lock wait").
|
||||
SET LOCAL lock_timeout = '5s';
|
||||
|
||||
ALTER TABLE durable_operation
|
||||
ADD COLUMN lease_fence BIGINT NOT NULL DEFAULT 0;
|
||||
+205
@@ -0,0 +1,205 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.idempotency.postgresql;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.IdempotencyReaper;
|
||||
import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.IdempotencyRecordJpaRepository;
|
||||
import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport;
|
||||
import jakarta.persistence.EntityManagerFactory;
|
||||
import java.sql.Connection;
|
||||
import java.sql.Statement;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.data.jpa.repository.support.JpaRepositoryFactory;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.orm.jpa.JpaTransactionManager;
|
||||
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
|
||||
import org.springframework.orm.jpa.SharedEntityManagerCreator;
|
||||
import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter;
|
||||
import org.springframework.transaction.TransactionManager;
|
||||
import org.springframework.transaction.annotation.AnnotationTransactionAttributeSource;
|
||||
import org.springframework.transaction.interceptor.TransactionInterceptor;
|
||||
|
||||
/**
|
||||
* The idempotency reaper's delete against a real PostgreSQL: expiry is the database clock's, and a
|
||||
* backlog larger than one batch is drained.
|
||||
*
|
||||
* <p>The store writes {@code expires_at} from {@code clock_timestamp()}, so the rows here are
|
||||
* written the same way, relative to the database's {@code now()}. A reaper that compared them with
|
||||
* the application's clock purged a live record whenever the host clock ran ahead of the database's
|
||||
* (round9 review 2026-11-12, RR9-01-04); the reaper now has no clock of its own to compare with.
|
||||
*
|
||||
* <p>The repository is the production shape: its own {@code @Transactional} opens each batch's
|
||||
* transaction, through the same interceptor Spring Data installs, and no test transaction wraps the
|
||||
* reap.
|
||||
*/
|
||||
@Tag("jpa-contract")
|
||||
class IdempotencyReaperIntegrationTest {
|
||||
|
||||
private static JpaPlatformContractSupport support;
|
||||
private static JdbcTemplate jdbc;
|
||||
private static LocalContainerEntityManagerFactoryBean persistence;
|
||||
private static IdempotencyReaper reaper;
|
||||
|
||||
@BeforeAll
|
||||
static void start() {
|
||||
support = JpaPlatformContractSupport.start();
|
||||
Flyway.configure()
|
||||
.dataSource(support.dataSource())
|
||||
.locations("classpath:db/migration/postgresql")
|
||||
.table("flyway_schema_history")
|
||||
.baselineOnMigrate(false)
|
||||
.outOfOrder(false)
|
||||
.load()
|
||||
.migrate();
|
||||
jdbc = new JdbcTemplate(support.dataSource());
|
||||
persistence = new LocalContainerEntityManagerFactoryBean();
|
||||
persistence.setDataSource(support.dataSource());
|
||||
persistence.setJpaVendorAdapter(new HibernateJpaVendorAdapter());
|
||||
persistence.setPackagesToScan(
|
||||
"dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.entity");
|
||||
Properties properties = new Properties();
|
||||
properties.setProperty("hibernate.jdbc.time_zone", "UTC");
|
||||
persistence.setJpaProperties(properties);
|
||||
persistence.afterPropertiesSet();
|
||||
EntityManagerFactory entityManagers = persistence.getObject();
|
||||
JpaTransactionManager transactionManager = new JpaTransactionManager(entityManagers);
|
||||
JpaRepositoryFactory repositories =
|
||||
new JpaRepositoryFactory(
|
||||
SharedEntityManagerCreator.createSharedEntityManager(entityManagers));
|
||||
repositories.addRepositoryProxyPostProcessor(
|
||||
(proxy, information) ->
|
||||
proxy.addAdvice(
|
||||
new TransactionInterceptor(
|
||||
(TransactionManager) transactionManager,
|
||||
new AnnotationTransactionAttributeSource())));
|
||||
reaper =
|
||||
new IdempotencyReaper(repositories.getRepository(IdempotencyRecordJpaRepository.class));
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
static void stop() {
|
||||
if (persistence != null) {
|
||||
persistence.destroy();
|
||||
}
|
||||
if (support != null) {
|
||||
support.close();
|
||||
}
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void empty() {
|
||||
jdbc.update("delete from idempotency_record");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRecordTheDatabaseClockHasNotExpiredSurvivesTheReap() {
|
||||
// given
|
||||
insert("live", 1, "now() + interval '1 hour'");
|
||||
insert("expired", 1, "now() - interval '1 second'");
|
||||
|
||||
// when
|
||||
reap();
|
||||
|
||||
// then
|
||||
assertThat(jdbc.queryForList("select idempotency_key from idempotency_record", String.class))
|
||||
.containsExactly("live-0");
|
||||
}
|
||||
|
||||
@Test
|
||||
void backlogLargerThanOneBatchIsDrainedInOneReap() {
|
||||
// given
|
||||
insert("expired", 2_501, "now() - interval '1 minute'");
|
||||
|
||||
// when
|
||||
int purged = reap();
|
||||
|
||||
// then
|
||||
assertThat(List.of(purged, count())).containsExactly(2_501, 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRecordReclaimedWhileTheReapRunsSurvivesIt() throws Exception {
|
||||
// given
|
||||
insert("reset", 1, "now() - interval '1 second'");
|
||||
try (Connection reclaim = support.dataSource().getConnection()) {
|
||||
reclaim.setAutoCommit(false);
|
||||
try (Statement statement = reclaim.createStatement()) {
|
||||
// What the store's reset does to an expired record a same-key request finds: it holds the
|
||||
// row and claims it again in place.
|
||||
statement.executeUpdate(
|
||||
"update idempotency_record set status = 'CLAIMED',"
|
||||
+ " expires_at = now() + interval '1 hour' where idempotency_key = 'reset-0'");
|
||||
}
|
||||
try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) {
|
||||
Future<Integer> reaping = executor.submit(() -> reap());
|
||||
awaitDoneOrBlocked(reaping, Duration.ofSeconds(5));
|
||||
reclaim.commit();
|
||||
|
||||
// when
|
||||
reaping.get(30, TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(jdbc.queryForList("select status from idempotency_record", String.class))
|
||||
.as("the reap deleted the claim a same-key request had just taken")
|
||||
.containsExactly("CLAIMED");
|
||||
}
|
||||
|
||||
private static int reap() {
|
||||
return reaper.reap();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns once the reap has finished or is waiting on a row lock, whichever comes first, and
|
||||
* fails the test when neither happened in time: committing the reclaim before the reap had run
|
||||
* would pass without the race this test exists for.
|
||||
*/
|
||||
private static void awaitDoneOrBlocked(Future<?> reaping, Duration within)
|
||||
throws InterruptedException {
|
||||
long deadline = System.nanoTime() + within.toNanos();
|
||||
while (!reaping.isDone()) {
|
||||
Integer waiting =
|
||||
jdbc.queryForObject(
|
||||
"select count(*) from pg_locks where not granted and locktype = 'transactionid'",
|
||||
Integer.class);
|
||||
if (waiting != null && waiting > 0) {
|
||||
return;
|
||||
}
|
||||
if (System.nanoTime() >= deadline) {
|
||||
throw new AssertionError(
|
||||
"the reap neither finished nor waited on the reclaimed row within " + within);
|
||||
}
|
||||
Thread.sleep(20);
|
||||
}
|
||||
}
|
||||
|
||||
private static int count() {
|
||||
Integer rows = jdbc.queryForObject("select count(*) from idempotency_record", Integer.class);
|
||||
return rows == null ? -1 : rows;
|
||||
}
|
||||
|
||||
private static void insert(String keyPrefix, int rows, String expiresAtSql) {
|
||||
jdbc.update(
|
||||
"insert into idempotency_record (id, principal, idempotency_key, use_case_name,"
|
||||
+ " request_hash, status, created_at, expires_at)"
|
||||
+ " select gen_random_uuid(), 'alice', ? || '-' || n, 'orders.create',"
|
||||
+ " repeat('a', 64), 'COMPLETED', now(), "
|
||||
+ expiresAtSql
|
||||
+ " from generate_series(0, ? - 1) as n",
|
||||
keyPrefix,
|
||||
rows);
|
||||
}
|
||||
}
|
||||
+1
-2
@@ -7,7 +7,6 @@ import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction
|
||||
import dev.caskeleton.application.idempotency.IdempotencyStorePort;
|
||||
import dev.caskeleton.application.idempotency.testkit.IdempotencyStoreContract;
|
||||
import jakarta.persistence.EntityManagerFactory;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.Properties;
|
||||
import java.util.function.Supplier;
|
||||
@@ -85,7 +84,7 @@ class PostgreSqlIdempotencyStoreContractTest extends IdempotencyStoreContract {
|
||||
new JpaRepositoryFactory(
|
||||
SharedEntityManagerCreator.createSharedEntityManager(entityManagers))
|
||||
.getRepository(IdempotencyRecordJpaRepository.class);
|
||||
reaper = new IdempotencyReaper(repository, Clock.systemUTC());
|
||||
reaper = new IdempotencyReaper(repository);
|
||||
reaperTransactions = new TransactionTemplate(new JpaTransactionManager(entityManagers));
|
||||
}
|
||||
|
||||
|
||||
+41
@@ -1,5 +1,7 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.liveevent;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.liveevent.entity.LiveEventEntity;
|
||||
import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport;
|
||||
import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction;
|
||||
@@ -13,12 +15,17 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import javax.sql.DataSource;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
@@ -98,6 +105,40 @@ class JpaLiveEventReplayContractTest extends LiveEventReplayContract {
|
||||
return RETENTION;
|
||||
}
|
||||
|
||||
@Test
|
||||
void theReaperRemovesRowsPastTheirRetentionAndKeepsTheRest() {
|
||||
// given
|
||||
String stream = "reaper-" + UUID.randomUUID();
|
||||
Instant start = Instant.parse("2026-11-03T00:00:00Z");
|
||||
appendAt(stream, "expired", start);
|
||||
appendAt(stream, "retained", start.plus(RETENTION).minusSeconds(60));
|
||||
LiveEventReaper reaper =
|
||||
new LiveEventReaper(
|
||||
context.getBean(LiveEventJpaRepository.class),
|
||||
Clock.fixed(start.plus(RETENTION).plusSeconds(1), ZoneOffset.UTC));
|
||||
|
||||
// when
|
||||
var unused = reaper.reap();
|
||||
|
||||
// then
|
||||
assertThat(
|
||||
new JdbcTemplate(support.dataSource())
|
||||
.queryForList(
|
||||
"SELECT payload FROM live_event_log WHERE stream_id = ?", String.class, stream))
|
||||
.containsExactly("retained");
|
||||
}
|
||||
|
||||
private void appendAt(String streamId, String payload, Instant at) {
|
||||
JpaLiveEventReplay writer =
|
||||
new JpaLiveEventReplay(
|
||||
context.getBean(LiveEventJpaRepository.class),
|
||||
new JdbcTemplate(support.dataSource()),
|
||||
Clock.fixed(at, ZoneOffset.UTC),
|
||||
RETENTION);
|
||||
var unused =
|
||||
context.getBean(TransactionPort.class).inWrite(() -> writer.append(streamId, payload));
|
||||
}
|
||||
|
||||
private static void execute(String sql) throws SQLException {
|
||||
try (Connection connection = support.connection();
|
||||
Statement statement = connection.createStatement()) {
|
||||
|
||||
+2
-1
@@ -49,7 +49,8 @@ final class NotificationFixtures {
|
||||
migration("jpa/notification-platform", "V10__variables_payload_envelope_guard.sql"),
|
||||
migration("jpa/notification-platform", "V11__notification_reconciliation_claim_lease.sql"),
|
||||
migration("jpa/notification-platform", "V12__notification_suppression_target_unique.sql"),
|
||||
migration("jpa/notification-platform", "V13__notification_request_acceptance.sql"));
|
||||
migration("jpa/notification-platform", "V13__notification_request_acceptance.sql"),
|
||||
migration("jpa/notification-platform", "V14__notification_provider_event_open_index.sql"));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+30
@@ -3,7 +3,10 @@ package dev.caskeleton.adapter.outbound.persistence.notification;
|
||||
import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport;
|
||||
import jakarta.persistence.EntityManager;
|
||||
import jakarta.persistence.EntityManagerFactory;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.function.Supplier;
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.springframework.data.jpa.repository.support.JpaRepositoryFactory;
|
||||
@@ -69,6 +72,33 @@ public final class NotificationJpaStack implements AutoCloseable {
|
||||
return new JdbcTemplate(database.dataSource());
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits until {@code work} has finished or some session of this database is blocked on a lock.
|
||||
*
|
||||
* <p>A race case holds one transaction open and starts a second statement that may or may not
|
||||
* need the first's row lock; this is how it knows the second has either finished or queued behind
|
||||
* the first before it lets the first commit, without sleeping a guessed interval.
|
||||
*/
|
||||
public void awaitDoneOrBlocked(Future<?> work, Duration deadline) {
|
||||
Instant giveUp = Instant.now().plus(deadline);
|
||||
JdbcTemplate jdbc = jdbc();
|
||||
while (Instant.now().isBefore(giveUp)) {
|
||||
if (work.isDone()) {
|
||||
return;
|
||||
}
|
||||
Integer waiting =
|
||||
jdbc.queryForObject(
|
||||
"SELECT count(*) FROM pg_stat_activity"
|
||||
+ " WHERE datname = current_database() AND wait_event_type = 'Lock'",
|
||||
Integer.class);
|
||||
if (waiting != null && waiting > 0) {
|
||||
return;
|
||||
}
|
||||
Thread.onSpinWait();
|
||||
}
|
||||
throw new IllegalStateException("work neither finished nor waited on a lock in " + deadline);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes every row of the delivery tables, children first, so a contract case starts empty.
|
||||
*
|
||||
|
||||
+72
@@ -15,11 +15,17 @@ import dev.caskeleton.application.notification.platform.dispatch.NotificationReq
|
||||
import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryRecord;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.testkit.ContractClock;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.testkit.NotificationRows;
|
||||
import dev.caskeleton.application.notification.platform.dispatch.testkit.RecipientLeaseStoreContract;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Supplier;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
@@ -114,6 +120,72 @@ class PostgreSqlRecipientLeaseStoreContractTest extends RecipientLeaseStoreContr
|
||||
.isGreaterThan(before);
|
||||
}
|
||||
|
||||
@Test
|
||||
void anExpirySweepRacingAClaimLeavesTheClaimedJobDispatching() throws Exception {
|
||||
// given
|
||||
NotificationRequestRecord request =
|
||||
NotificationRows.request(new TenantId("tenant-a"), "idem-race", NOW);
|
||||
RecipientDeliveryRecord job =
|
||||
NotificationRows.job(
|
||||
request,
|
||||
RecipientDeliveryState.READY_TO_DISPATCH,
|
||||
Optional.of(NOW),
|
||||
Optional.of(NOW.plusSeconds(10)),
|
||||
NOW);
|
||||
inTransaction(() -> requests().insert(request, List.of(job)));
|
||||
CountDownLatch claimed = new CountDownLatch(1);
|
||||
CountDownLatch release = new CountDownLatch(1);
|
||||
ExecutorService pool = Executors.newFixedThreadPool(2);
|
||||
|
||||
// when
|
||||
List<Object> outcome;
|
||||
try {
|
||||
// The claim holds its row lock, uncommitted, while the sweep runs at an instant past the
|
||||
// job's expiry. The sweep used to pick the id from the committed READY row, wait for the
|
||||
// lock, and then retire the row the claim had just made DISPATCHING.
|
||||
CompletableFuture<Void> claim =
|
||||
CompletableFuture.runAsync(
|
||||
() ->
|
||||
inTransaction(
|
||||
() -> {
|
||||
var unused = leases().claim("worker-a", 10, Duration.ofMinutes(5));
|
||||
claimed.countDown();
|
||||
awaitOrFail(release);
|
||||
return null;
|
||||
}),
|
||||
pool);
|
||||
awaitOrFail(claimed);
|
||||
((ContractClock) clock()).moveTo(NOW.plusSeconds(11));
|
||||
CompletableFuture<Integer> sweep =
|
||||
CompletableFuture.supplyAsync(
|
||||
() -> inTransaction(() -> leases().expireOverdue(10)), pool);
|
||||
stack.awaitDoneOrBlocked(sweep, Duration.ofSeconds(30));
|
||||
release.countDown();
|
||||
claim.get(30, TimeUnit.SECONDS);
|
||||
outcome = List.of(sweep.get(30, TimeUnit.SECONDS), stateOf(job.id()));
|
||||
} finally {
|
||||
pool.shutdownNow();
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(outcome).containsExactly(0, RecipientDeliveryState.DISPATCHING);
|
||||
}
|
||||
|
||||
private RecipientDeliveryState stateOf(RecipientDeliveryId id) {
|
||||
return deliveries().find(id).orElseThrow().state();
|
||||
}
|
||||
|
||||
private static void awaitOrFail(CountDownLatch latch) {
|
||||
try {
|
||||
if (!latch.await(30, TimeUnit.SECONDS)) {
|
||||
throw new IllegalStateException("the other transaction did not arrive within 30 seconds");
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("interrupted while waiting", interrupted);
|
||||
}
|
||||
}
|
||||
|
||||
private static long versionOf(RecipientDeliveryId id) {
|
||||
return stack
|
||||
.jdbc()
|
||||
|
||||
+167
-10
@@ -8,6 +8,7 @@ import dev.caskeleton.application.notification.platform.observation.Notification
|
||||
import java.io.IOException;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.sql.Timestamp;
|
||||
@@ -15,6 +16,9 @@ import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import javax.sql.DataSource;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
@@ -23,7 +27,12 @@ import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.CsvSource;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.PreparedStatementCreator;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.jdbc.core.SqlProvider;
|
||||
import org.springframework.jdbc.datasource.SingleConnectionDataSource;
|
||||
|
||||
/**
|
||||
@@ -100,6 +109,39 @@ class ServingStateContractTest {
|
||||
assertThat(measure().backlogDepth()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aDeliveryWhoseWindowClosedIsNotBacklog() throws SQLException {
|
||||
// given
|
||||
// Due, but past its expiry: the claim refuses it and the expiry sweep retires it unsent.
|
||||
insertDelivery(
|
||||
"READY_TO_DISPATCH", NOW.minus(Duration.ofMinutes(5)), null, null, NOW.minusSeconds(1));
|
||||
|
||||
// when
|
||||
long depth = measure().backlogDepth();
|
||||
|
||||
// then
|
||||
assertThat(depth).isZero();
|
||||
}
|
||||
|
||||
@ParameterizedTest(name = "[{index}] lease_until = NOW에서 {0}초 → backlog {1}")
|
||||
@CsvSource({"-1, 1", "0, 0"})
|
||||
@DisplayName("lease는 NOW 전에 끝났을 때만 끝난 것이다: claim과 같은 lease_until < now")
|
||||
void aLeaseHasRunOutOnlyOnceItEndedBeforeNow(long offsetSeconds, long expectedDepth)
|
||||
throws SQLException {
|
||||
// given
|
||||
insertDelivery(
|
||||
"READY_TO_DISPATCH",
|
||||
NOW.minus(Duration.ofMinutes(1)),
|
||||
"worker-1",
|
||||
NOW.plusSeconds(offsetSeconds));
|
||||
|
||||
// when
|
||||
long depth = measure().backlogDepth();
|
||||
|
||||
// then
|
||||
assertThat(depth).isEqualTo(expectedDepth);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a delivery held under a live lease is not backlog")
|
||||
void aDeliveryUnderALiveLeaseIsNotBacklog() throws SQLException {
|
||||
@@ -113,22 +155,24 @@ class ServingStateContractTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a lease whose holder never came back is counted as stuck")
|
||||
void anExpiredLeaseIsStuck() throws SQLException {
|
||||
// given
|
||||
insertDelivery(
|
||||
"DISPATCHING",
|
||||
NOW.minus(Duration.ofMinutes(20)),
|
||||
"worker-1",
|
||||
NOW.minus(Duration.ofMinutes(10)));
|
||||
|
||||
// when
|
||||
NotificationServingState state = measure();
|
||||
|
||||
assertThat(state.stuckLeases())
|
||||
.as("a worker died mid-dispatch, so a provider call may have happened with no record")
|
||||
.isEqualTo(1);
|
||||
assertThat(state.backlogDepth())
|
||||
.as("an expired lease is claimable again, but this row is DISPATCHING, not due")
|
||||
.isZero();
|
||||
// then
|
||||
assertThat(state)
|
||||
.as(
|
||||
"a worker died mid-dispatch, so a provider call may have happened with no record; a"
|
||||
+ " dispatching row is never claimed, so it is not backlog")
|
||||
.extracting(NotificationServingState::stuckLeases, NotificationServingState::backlogDepth)
|
||||
.containsExactly(1L, 0L);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -162,6 +206,108 @@ class ServingStateContractTest {
|
||||
assertThat(state.oldestReconciliationAge()).isEqualTo(Duration.ofHours(2));
|
||||
}
|
||||
|
||||
@Test
|
||||
void theSchedulersBacklogReadIsOneStatement() throws SQLException {
|
||||
// given
|
||||
insertDelivery("READY_TO_DISPATCH", NOW.minus(Duration.ofMinutes(30)), null, null);
|
||||
|
||||
// when
|
||||
List<Object> outcome;
|
||||
try (Connection connection = support.connection()) {
|
||||
RecordingJdbc jdbc = new RecordingJdbc(new SingleConnectionDataSource(connection, true));
|
||||
long depth = new JdbcNotificationServingState(jdbc, CLOCK).backlogDepth();
|
||||
outcome = List.of(depth, jdbc.statements.size());
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(outcome)
|
||||
.as("the scheduler asks this on every poll; the full snapshot is four statements")
|
||||
.containsExactly(1L, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void theProjectionLagReadsOpenEventsThroughTheirIndex() throws SQLException {
|
||||
// given
|
||||
insertProviderEvent("PENDING", NOW.minus(Duration.ofMinutes(45)), false);
|
||||
|
||||
// when
|
||||
String plan;
|
||||
try (Connection connection = support.connection()) {
|
||||
RecordingJdbc jdbc = new RecordingJdbc(new SingleConnectionDataSource(connection, true));
|
||||
var unused = new JdbcNotificationServingState(jdbc, CLOCK).currentState();
|
||||
RecordedStatement lag =
|
||||
jdbc.statements.stream()
|
||||
.filter(statement -> statement.sql().contains("notification_provider_event"))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
plan = explainWithoutSequentialScans(connection, lag);
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(plan)
|
||||
.as("the event table is never purged; only the index over the open events bounds this read")
|
||||
.contains("ix_notification_provider_event_open");
|
||||
}
|
||||
|
||||
/** The plan PostgreSQL picks for {@code statement} when it may not scan a table sequentially. */
|
||||
private static String explainWithoutSequentialScans(
|
||||
Connection connection, RecordedStatement statement) throws SQLException {
|
||||
boolean autoCommit = connection.getAutoCommit();
|
||||
connection.setAutoCommit(false);
|
||||
try {
|
||||
try (Statement off = connection.createStatement()) {
|
||||
off.execute("SET LOCAL enable_seqscan = off");
|
||||
}
|
||||
StringBuilder plan = new StringBuilder();
|
||||
try (PreparedStatement explain = connection.prepareStatement("EXPLAIN " + statement.sql())) {
|
||||
for (int index = 0; index < statement.args().size(); index++) {
|
||||
explain.setObject(index + 1, statement.args().get(index));
|
||||
}
|
||||
try (ResultSet rows = explain.executeQuery()) {
|
||||
while (rows.next()) {
|
||||
plan.append(rows.getString(1)).append('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
return plan.toString();
|
||||
} finally {
|
||||
connection.rollback();
|
||||
connection.setAutoCommit(autoCommit);
|
||||
}
|
||||
}
|
||||
|
||||
/** A statement the serving state ran, as it ran it. */
|
||||
private record RecordedStatement(String sql, List<Object> args) {}
|
||||
|
||||
/** Plain JDBC that keeps the statements it was given, so a case reads the shipped SQL. */
|
||||
private static final class RecordingJdbc extends JdbcTemplate {
|
||||
|
||||
private final List<RecordedStatement> statements = new ArrayList<>();
|
||||
|
||||
private RecordingJdbc(DataSource dataSource) {
|
||||
super(dataSource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T queryForObject(String sql, RowMapper<T> rowMapper, Object... args) {
|
||||
statements.add(new RecordedStatement(sql, Arrays.asList(args)));
|
||||
return super.queryForObject(sql, rowMapper, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T queryForObject(String sql, Class<T> requiredType, Object... args) {
|
||||
statements.add(new RecordedStatement(sql, Arrays.asList(args)));
|
||||
return super.queryForObject(sql, requiredType, args);
|
||||
}
|
||||
|
||||
/** The named-parameter backlog read arrives here; its parameters stay inside the creator. */
|
||||
@Override
|
||||
public <T> List<T> query(PreparedStatementCreator creator, RowMapper<T> rowMapper) {
|
||||
statements.add(new RecordedStatement(((SqlProvider) creator).getSql(), List.of()));
|
||||
return super.query(creator, rowMapper);
|
||||
}
|
||||
}
|
||||
|
||||
private NotificationServingState measure() throws SQLException {
|
||||
// try-with-resources: the pool holds four connections, and a measurement that keeps one is a
|
||||
// measurement that starves the next test.
|
||||
@@ -174,6 +320,16 @@ class ServingStateContractTest {
|
||||
private void insertDelivery(
|
||||
String state, Instant nextDispatchAt, String leaseOwner, Instant leaseUntil)
|
||||
throws SQLException {
|
||||
insertDelivery(state, nextDispatchAt, leaseOwner, leaseUntil, null);
|
||||
}
|
||||
|
||||
private void insertDelivery(
|
||||
String state,
|
||||
Instant nextDispatchAt,
|
||||
String leaseOwner,
|
||||
Instant leaseUntil,
|
||||
Instant expiresAt)
|
||||
throws SQLException {
|
||||
UUID requestId = UUID.randomUUID();
|
||||
try (Connection connection = support.connection()) {
|
||||
try (PreparedStatement statement =
|
||||
@@ -197,17 +353,18 @@ class ServingStateContractTest {
|
||||
+ " id, notification_id, tenant_id, recipient_ref, routing_plan_json,"
|
||||
+ " route_cursor, delivery_state, submission_outcome, delivery_outcome,"
|
||||
+ " evidence_level, attempt_count, next_dispatch_at, lease_owner, lease_until,"
|
||||
+ " created_at, updated_at)"
|
||||
+ " expires_at, created_at, updated_at)"
|
||||
+ " VALUES (?, ?, 'tenant-a', 'user-1', '[]'::jsonb, 0, ?, 'NOT_SUBMITTED',"
|
||||
+ " 'UNKNOWN', 'PLATFORM_QUEUED', 0, ?, ?, ?, ?, ?)")) {
|
||||
+ " 'UNKNOWN', 'PLATFORM_QUEUED', 0, ?, ?, ?, ?, ?, ?)")) {
|
||||
statement.setObject(1, UUID.randomUUID());
|
||||
statement.setObject(2, requestId);
|
||||
statement.setString(3, state);
|
||||
statement.setTimestamp(4, Timestamp.from(nextDispatchAt));
|
||||
statement.setString(5, leaseOwner);
|
||||
statement.setTimestamp(6, leaseUntil == null ? null : Timestamp.from(leaseUntil));
|
||||
statement.setTimestamp(7, Timestamp.from(NOW));
|
||||
statement.setTimestamp(7, expiresAt == null ? null : Timestamp.from(expiresAt));
|
||||
statement.setTimestamp(8, Timestamp.from(NOW));
|
||||
statement.setTimestamp(9, Timestamp.from(NOW));
|
||||
statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
+51
@@ -5,9 +5,17 @@ import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.NotificationJpaStack;
|
||||
import dev.caskeleton.adapter.outbound.persistence.notification.platform.entity.InboxItemEntity;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
@@ -74,6 +82,49 @@ class InboxBulkReadVersionContractTest {
|
||||
.isEqualTo("READ");
|
||||
}
|
||||
|
||||
@Test
|
||||
void anItemArchivedWhileTheBulkReadWaitsForItStaysArchived() throws Exception {
|
||||
// given
|
||||
UUID id = insertUnreadItem();
|
||||
ExecutorService pool = Executors.newSingleThreadExecutor();
|
||||
|
||||
// when
|
||||
List<Object> outcome;
|
||||
try (Connection archiving = stack.jdbc().getDataSource().getConnection()) {
|
||||
// The archive holds the row uncommitted while mark-all-read runs. The bulk read picks the id
|
||||
// from the committed unread row, waits for the lock, and used to mark READ the item the
|
||||
// user had just archived — putting it back in the inbox.
|
||||
archiving.setAutoCommit(false);
|
||||
try (PreparedStatement archive =
|
||||
archiving.prepareStatement(
|
||||
"UPDATE notification_inbox_item SET state = 'ARCHIVED', version = version + 1"
|
||||
+ " WHERE id = ?")) {
|
||||
archive.setObject(1, id);
|
||||
archive.executeUpdate();
|
||||
}
|
||||
CompletableFuture<Integer> bulkRead =
|
||||
CompletableFuture.supplyAsync(
|
||||
() ->
|
||||
stack.inTransaction(
|
||||
() -> items.markAllRead("tenant-a", "user-1", null, null, 100, NOW)),
|
||||
pool);
|
||||
stack.awaitDoneOrBlocked(bulkRead, Duration.ofSeconds(30));
|
||||
archiving.commit();
|
||||
outcome = List.of(bulkRead.get(30, TimeUnit.SECONDS), stateOf(id));
|
||||
} finally {
|
||||
pool.shutdownNow();
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(outcome).containsExactly(0, "ARCHIVED");
|
||||
}
|
||||
|
||||
private static String stateOf(UUID id) {
|
||||
return stack
|
||||
.jdbc()
|
||||
.queryForObject("SELECT state FROM notification_inbox_item WHERE id = ?", String.class, id);
|
||||
}
|
||||
|
||||
private static UUID insertUnreadItem() {
|
||||
UUID id = UUID.randomUUID();
|
||||
stack
|
||||
|
||||
+214
@@ -0,0 +1,214 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.operation;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* The {@code durable_operation} table's own constraints, against a real PostgreSQL.
|
||||
*
|
||||
* <p>What the store does is {@link JpaDurableOperationStoreContractTest}: the shipped adapter
|
||||
* through the shared {@code DurableOperationStoreContract}. This class holds only what that
|
||||
* contract cannot reach — the constraints a second writer meets when it bypasses the adapter. A
|
||||
* {@code CHECK} constraint a fake does not enforce is a comment, so each one is shown refusing a
|
||||
* row here, by writing the row directly.
|
||||
*
|
||||
* <p>It replaces a class that re-typed the adapter's SQL into the test and ran that copy (round9
|
||||
* review 2026-11-12, handed over from RR9 area 01): the copy passed whatever the shipped statements
|
||||
* did, and seven of its cases repeated the contract.
|
||||
*/
|
||||
@Tag("jpa-contract")
|
||||
class DurableOperationSchemaConstraintTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-08-25T09:00:00Z");
|
||||
|
||||
private static JpaPlatformContractSupport support;
|
||||
|
||||
@BeforeAll
|
||||
static void startServer() {
|
||||
support = JpaPlatformContractSupport.start();
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
static void stopServer() {
|
||||
if (support != null) {
|
||||
support.close();
|
||||
}
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void migrate() throws SQLException, IOException {
|
||||
try (Connection connection = support.connection();
|
||||
Statement statement = connection.createStatement()) {
|
||||
statement.execute("DROP SCHEMA public CASCADE");
|
||||
statement.execute("CREATE SCHEMA public");
|
||||
statement.execute(migration("V11__durable_operation.sql"));
|
||||
statement.execute(migration("V23__durable_operation_lease_fence.sql"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void identicalScopeIsRefusedByTheUniqueConstraint() throws SQLException {
|
||||
// given
|
||||
insertPending("op-1", "hash-1");
|
||||
|
||||
// when
|
||||
// The constraint is the arbiter the adapter's ON CONFLICT names, not a prior read: two
|
||||
// identical submissions can arrive at the same instant, and a read-then-insert lets both in.
|
||||
Throwable refused = catchThrowable(() -> insertPending("op-2", "hash-1"));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("uq_durable_operation_scope");
|
||||
}
|
||||
|
||||
@Test
|
||||
void succeededRowWithoutAResultIsRefused() throws SQLException {
|
||||
// given
|
||||
insertPending("op-1", "hash-1");
|
||||
|
||||
// when
|
||||
Throwable refused =
|
||||
catchThrowable(
|
||||
() ->
|
||||
execute(
|
||||
"UPDATE durable_operation SET state = 'SUCCEEDED', completed_at = now()"
|
||||
+ " WHERE operation_id = 'op-1'"));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("ck_durable_operation_succeeded_has_result");
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedRowWithoutAReasonIsRefused() throws SQLException {
|
||||
// given
|
||||
insertPending("op-1", "hash-1");
|
||||
|
||||
// when
|
||||
Throwable refused =
|
||||
catchThrowable(
|
||||
() ->
|
||||
execute(
|
||||
"UPDATE durable_operation SET state = 'FAILED', completed_at = now()"
|
||||
+ " WHERE operation_id = 'op-1'"));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("ck_durable_operation_failed_has_reason");
|
||||
}
|
||||
|
||||
@Test
|
||||
void runningRowWithoutALeaseIsRefused() throws SQLException {
|
||||
// given
|
||||
insertPending("op-1", "hash-1");
|
||||
|
||||
// when
|
||||
Throwable refused =
|
||||
catchThrowable(
|
||||
() ->
|
||||
execute(
|
||||
"UPDATE durable_operation SET state = 'RUNNING' WHERE operation_id = 'op-1'"));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("ck_durable_operation_running_has_lease");
|
||||
}
|
||||
|
||||
@Test
|
||||
void stateOutsideThePublishedVocabularyIsRefused() throws SQLException {
|
||||
// given
|
||||
insertPending("op-1", "hash-1");
|
||||
|
||||
// when
|
||||
// A worker that could write its own state would extend the published contract by writing a
|
||||
// row, and a client branching on status would meet a value that is in no version of the API.
|
||||
Throwable refused =
|
||||
catchThrowable(
|
||||
() ->
|
||||
execute(
|
||||
"UPDATE durable_operation SET state = 'ALMOST_DONE'"
|
||||
+ " WHERE operation_id = 'op-1'"));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("ck_durable_operation_state");
|
||||
}
|
||||
|
||||
@Test
|
||||
void expiryNotAfterSubmissionIsRefused() throws SQLException {
|
||||
// given
|
||||
insertPending("op-1", "hash-1");
|
||||
|
||||
// when
|
||||
Throwable refused =
|
||||
catchThrowable(
|
||||
() ->
|
||||
execute(
|
||||
"UPDATE durable_operation SET expires_at = submitted_at"
|
||||
+ " WHERE operation_id = 'op-1'"));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("ck_durable_operation_expiry_after_submission");
|
||||
}
|
||||
|
||||
private void insertPending(String operationId, String hash) throws SQLException {
|
||||
try (Connection connection = support.connection();
|
||||
PreparedStatement statement =
|
||||
connection.prepareStatement(
|
||||
"""
|
||||
INSERT INTO durable_operation
|
||||
(operation_id, tenant, principal, operation_name, request_hash, payload,
|
||||
state, submitted_at, expires_at)
|
||||
VALUES (?, '', 'alice', 'transfers.create', ?, '{}', 'PENDING', ?, ?)
|
||||
""")) {
|
||||
statement.setString(1, operationId);
|
||||
statement.setString(2, hash);
|
||||
statement.setTimestamp(3, Timestamp.from(NOW));
|
||||
statement.setTimestamp(4, Timestamp.from(NOW.plus(Duration.ofDays(1))));
|
||||
statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
private void execute(String sql) throws SQLException {
|
||||
try (Connection connection = support.connection();
|
||||
Statement statement = connection.createStatement()) {
|
||||
statement.execute(sql);
|
||||
}
|
||||
}
|
||||
|
||||
private static String migration(String file) throws IOException {
|
||||
try (InputStream resource =
|
||||
DurableOperationSchemaConstraintTest.class
|
||||
.getClassLoader()
|
||||
.getResourceAsStream("db/migration/postgresql/" + file)) {
|
||||
if (resource == null) {
|
||||
throw new IOException("the durable operation migration " + file + " is missing");
|
||||
}
|
||||
return new String(resource.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
}
|
||||
-474
@@ -1,474 +0,0 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.operation;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* The durable operation store against a real PostgreSQL.
|
||||
*
|
||||
* <p>Every claim in this store is about concurrency or about a constraint, and neither survives
|
||||
* being tested against a fake. {@code FOR UPDATE SKIP LOCKED} does not exist in H2's PostgreSQL
|
||||
* mode with the semantics that matter, a {@code CHECK} constraint a fake does not enforce is a
|
||||
* comment, and a lease race has no meaning where only one connection exists.
|
||||
*
|
||||
* <p>A dead worker is written the only way that is faithful here: the row a crash leaves behind — a
|
||||
* lease nobody will renew — and then recovery runs against it. That is precisely what a killed
|
||||
* process is from the database's side.
|
||||
*/
|
||||
@Tag("jpa-contract")
|
||||
class DurableOperationStoreContractTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-08-25T09:00:00Z");
|
||||
private static final Duration LEASE = Duration.ofMinutes(2);
|
||||
|
||||
private static JpaPlatformContractSupport support;
|
||||
|
||||
@BeforeAll
|
||||
static void startServer() {
|
||||
support = JpaPlatformContractSupport.start();
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
static void stopServer() {
|
||||
if (support != null) {
|
||||
support.close();
|
||||
}
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void migrate() throws SQLException, IOException {
|
||||
try (Connection connection = support.connection();
|
||||
Statement statement = connection.createStatement()) {
|
||||
statement.execute("DROP SCHEMA public CASCADE");
|
||||
statement.execute("CREATE SCHEMA public");
|
||||
statement.execute(migration());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a lease left behind by a dead worker is reclaimed by another")
|
||||
void expiredLeaseCanBeReclaimedAfterWorkerCrash() throws SQLException {
|
||||
submit("op-1", "hash-1");
|
||||
claim("worker-a", NOW);
|
||||
|
||||
// The worker is gone. Nothing renews the lease, and nothing else may touch the operation until
|
||||
// it lapses — that exclusivity is the point of the lease.
|
||||
assertThat(claim("worker-b", NOW.plusSeconds(30))).isEmpty();
|
||||
|
||||
assertThat(claim("worker-b", NOW.plus(LEASE).plusSeconds(1))).contains("op-1");
|
||||
assertThat(state("op-1")).isEqualTo("RUNNING");
|
||||
assertThat(leaseOwner("op-1")).isEqualTo("worker-b");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a reclaimed operation keeps the moment it first started")
|
||||
void reclaimKeepsTheOriginalStartTime() throws SQLException {
|
||||
submit("op-1", "hash-1");
|
||||
claim("worker-a", NOW);
|
||||
|
||||
claim("worker-b", NOW.plus(LEASE).plusSeconds(1));
|
||||
|
||||
// COALESCE, not an unconditional assignment. Overwriting it would make every retry look like a
|
||||
// fresh start and hide how long the operation has really been running.
|
||||
assertThat(startedAt("op-1")).isEqualTo(NOW);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("two workers claiming at once take different operations")
|
||||
void concurrentWorkersDoNotTakeTheSameOperation() throws Exception {
|
||||
submit("op-1", "hash-1");
|
||||
submit("op-2", "hash-2");
|
||||
|
||||
List<String> claimed = new ArrayList<>();
|
||||
try (Connection first = support.connection();
|
||||
Connection second = support.connection()) {
|
||||
first.setAutoCommit(false);
|
||||
second.setAutoCommit(false);
|
||||
// Interleaved on purpose and inside open transactions: with plain FOR UPDATE the second
|
||||
// claim blocks here until the first commits, and with no locking at all both take op-1.
|
||||
claimed.add(claimOn(first, "worker-a", NOW).orElseThrow());
|
||||
claimed.add(claimOn(second, "worker-b", NOW).orElseThrow());
|
||||
first.commit();
|
||||
second.commit();
|
||||
}
|
||||
|
||||
assertThat(claimed).containsExactlyInAnyOrder("op-1", "op-2");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a worker whose lease lapsed cannot record a result over its successor")
|
||||
void aStaleWorkerCannotOverwriteTheNewOwner() throws SQLException {
|
||||
submit("op-1", "hash-1");
|
||||
claim("worker-a", NOW);
|
||||
claim("worker-b", NOW.plus(LEASE).plusSeconds(1));
|
||||
|
||||
// worker-a is alive and still finishing. Letting it report here is how one operation ends up
|
||||
// with two answers, and which one survives would come down to timing.
|
||||
assertThat(succeed("op-1", "worker-a", "/results/a")).isZero();
|
||||
assertThat(succeed("op-1", "worker-b", "/results/b")).isOne();
|
||||
assertThat(resultReference("op-1")).isEqualTo("/results/b");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a heartbeat from a worker that lost the lease fails rather than extending it")
|
||||
void heartbeatFromAStaleWorkerFails() throws SQLException {
|
||||
submit("op-1", "hash-1");
|
||||
claim("worker-a", NOW);
|
||||
claim("worker-b", NOW.plus(LEASE).plusSeconds(1));
|
||||
|
||||
assertThat(heartbeat("op-1", "worker-a", NOW.plus(LEASE).plusSeconds(2))).isZero();
|
||||
assertThat(heartbeat("op-1", "worker-b", NOW.plus(LEASE).plusSeconds(2))).isOne();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("cancel works once and never reverses a finished operation")
|
||||
void cancelIsIdempotentAndNeverReversesTerminal() throws SQLException {
|
||||
submit("op-1", "hash-1");
|
||||
claim("worker-a", NOW);
|
||||
succeed("op-1", "worker-a", "/results/a");
|
||||
|
||||
// The work happened. Recording it as CANCELED would replace the only record of a real outcome
|
||||
// with a claim that it never occurred.
|
||||
assertThat(cancel("op-1", "alice")).isZero();
|
||||
assertThat(state("op-1")).isEqualTo("SUCCEEDED");
|
||||
|
||||
submit("op-2", "hash-2");
|
||||
assertThat(cancel("op-2", "alice")).isOne();
|
||||
assertThat(cancel("op-2", "alice")).isZero();
|
||||
assertThat(state("op-2")).isEqualTo("CANCELED");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("another principal cannot cancel an operation they did not submit")
|
||||
void cancelIsScopedToThePrincipal() throws SQLException {
|
||||
submit("op-1", "hash-1");
|
||||
|
||||
assertThat(cancel("op-1", "mallory")).isZero();
|
||||
assertThat(state("op-1")).isEqualTo("PENDING");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a cancelled operation is not handed to a worker")
|
||||
void cancelledOperationsAreNotClaimable() throws SQLException {
|
||||
submit("op-1", "hash-1");
|
||||
cancel("op-1", "alice");
|
||||
|
||||
assertThat(claim("worker-a", NOW)).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an identical resubmission is refused by the unique constraint")
|
||||
void resubmissionCollidesOnScope() throws SQLException {
|
||||
submit("op-1", "hash-1");
|
||||
|
||||
// The constraint is the arbiter rather than a prior read: two identical submissions can arrive
|
||||
// at the same instant, and a read-then-insert lets both through.
|
||||
assertThatThrownBy(() -> submit("op-2", "hash-1")).isInstanceOf(SQLException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a succeeded row with no result is refused by the database")
|
||||
void succeededRowRequiresAResult() throws SQLException {
|
||||
submit("op-1", "hash-1");
|
||||
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
execute(
|
||||
"UPDATE durable_operation SET state = 'SUCCEEDED', completed_at = now(),"
|
||||
+ " lease_owner = NULL WHERE operation_id = 'op-1'"))
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("ck_durable_operation_succeeded_has_result");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a running row with no lease is refused by the database")
|
||||
void runningRowRequiresALease() {
|
||||
assertThatThrownBy(
|
||||
() -> {
|
||||
submit("op-1", "hash-1");
|
||||
execute("UPDATE durable_operation SET state = 'RUNNING' WHERE operation_id = 'op-1'");
|
||||
})
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("ck_durable_operation_running_has_lease");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a state outside the published vocabulary is refused")
|
||||
void stateVocabularyIsClosed() {
|
||||
// A worker that could write its own state would extend the published contract by writing a
|
||||
// row, and a client branching on status would meet a value that is in no version of the API.
|
||||
assertThatThrownBy(
|
||||
() -> {
|
||||
submit("op-1", "hash-1");
|
||||
execute(
|
||||
"UPDATE durable_operation SET state = 'ALMOST_DONE' WHERE operation_id = 'op-1'");
|
||||
})
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("ck_durable_operation_state");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an aged-out operation is expired, and a finished one is left alone")
|
||||
void expirySweepLeavesRealOutcomesAlone() throws SQLException {
|
||||
submit("op-1", "hash-1");
|
||||
submit("op-2", "hash-2");
|
||||
claim("worker-a", NOW);
|
||||
|
||||
List<String> expired = expireStale(NOW.plus(Duration.ofDays(2)));
|
||||
|
||||
assertThat(expired).containsExactlyInAnyOrder("op-1", "op-2");
|
||||
submit("op-3", "hash-3");
|
||||
claim("worker-b", NOW);
|
||||
succeed("op-3", "worker-b", "/results/c");
|
||||
assertThat(expireStale(NOW.plus(Duration.ofDays(2)))).isEmpty();
|
||||
assertThat(state("op-3")).isEqualTo("SUCCEEDED");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("reclaim returns lapsed leases to the queue")
|
||||
void reclaimReturnsLapsedLeases() throws SQLException {
|
||||
submit("op-1", "hash-1");
|
||||
claim("worker-a", NOW);
|
||||
|
||||
assertThat(reclaim(NOW.plusSeconds(30))).isEmpty();
|
||||
assertThat(reclaim(NOW.plus(LEASE).plusSeconds(1))).containsExactly("op-1");
|
||||
assertThat(state("op-1")).isEqualTo("PENDING");
|
||||
assertThat(leaseOwner("op-1")).isNull();
|
||||
}
|
||||
|
||||
private void submit(String operationId, String hash) throws SQLException {
|
||||
try (Connection connection = support.connection();
|
||||
PreparedStatement statement =
|
||||
connection.prepareStatement(
|
||||
"""
|
||||
INSERT INTO durable_operation
|
||||
(operation_id, tenant, principal, operation_name, request_hash, payload,
|
||||
state, submitted_at, expires_at)
|
||||
VALUES (?, '', 'alice', 'transfers.create', ?, '{}', 'PENDING', ?, ?)
|
||||
""")) {
|
||||
statement.setString(1, operationId);
|
||||
statement.setString(2, hash);
|
||||
statement.setTimestamp(3, Timestamp.from(NOW));
|
||||
statement.setTimestamp(4, Timestamp.from(NOW.plus(Duration.ofDays(1))));
|
||||
statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
private Optional<String> claim(String workerId, Instant now) throws SQLException {
|
||||
try (Connection connection = support.connection()) {
|
||||
return claimOn(connection, workerId, now);
|
||||
}
|
||||
}
|
||||
|
||||
private Optional<String> claimOn(Connection connection, String workerId, Instant now)
|
||||
throws SQLException {
|
||||
try (PreparedStatement statement =
|
||||
connection.prepareStatement(
|
||||
"""
|
||||
UPDATE durable_operation
|
||||
SET state = 'RUNNING',
|
||||
lease_owner = ?,
|
||||
lease_acquired_at = ?,
|
||||
lease_expires_at = ?,
|
||||
started_at = COALESCE(started_at, ?),
|
||||
row_version = row_version + 1
|
||||
WHERE operation_id = (
|
||||
SELECT operation_id
|
||||
FROM durable_operation
|
||||
WHERE (state = 'PENDING'
|
||||
OR (state = 'RUNNING' AND lease_expires_at <= ?))
|
||||
AND expires_at > ?
|
||||
ORDER BY submitted_at
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1)
|
||||
RETURNING operation_id
|
||||
""")) {
|
||||
statement.setString(1, workerId);
|
||||
statement.setTimestamp(2, Timestamp.from(now));
|
||||
statement.setTimestamp(3, Timestamp.from(now.plus(LEASE)));
|
||||
statement.setTimestamp(4, Timestamp.from(now));
|
||||
statement.setTimestamp(5, Timestamp.from(now));
|
||||
statement.setTimestamp(6, Timestamp.from(now));
|
||||
try (ResultSet rows = statement.executeQuery()) {
|
||||
return rows.next() ? Optional.of(rows.getString(1)) : Optional.empty();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private int succeed(String operationId, String workerId, String resultReference)
|
||||
throws SQLException {
|
||||
try (Connection connection = support.connection();
|
||||
PreparedStatement statement =
|
||||
connection.prepareStatement(
|
||||
"""
|
||||
UPDATE durable_operation
|
||||
SET state = 'SUCCEEDED', result_reference = ?, completed_at = ?,
|
||||
lease_owner = NULL, lease_acquired_at = NULL, lease_expires_at = NULL,
|
||||
row_version = row_version + 1
|
||||
WHERE operation_id = ? AND state = 'RUNNING' AND lease_owner = ?
|
||||
""")) {
|
||||
statement.setString(1, resultReference);
|
||||
statement.setTimestamp(2, Timestamp.from(NOW.plusSeconds(10)));
|
||||
statement.setString(3, operationId);
|
||||
statement.setString(4, workerId);
|
||||
return statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
private int heartbeat(String operationId, String workerId, Instant now) throws SQLException {
|
||||
try (Connection connection = support.connection();
|
||||
PreparedStatement statement =
|
||||
connection.prepareStatement(
|
||||
"""
|
||||
UPDATE durable_operation
|
||||
SET lease_expires_at = ?, row_version = row_version + 1
|
||||
WHERE operation_id = ? AND state = 'RUNNING' AND lease_owner = ?
|
||||
AND lease_expires_at > ?
|
||||
""")) {
|
||||
statement.setTimestamp(1, Timestamp.from(now.plus(LEASE)));
|
||||
statement.setString(2, operationId);
|
||||
statement.setString(3, workerId);
|
||||
statement.setTimestamp(4, Timestamp.from(now));
|
||||
return statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
private int cancel(String operationId, String principal) throws SQLException {
|
||||
try (Connection connection = support.connection();
|
||||
PreparedStatement statement =
|
||||
connection.prepareStatement(
|
||||
"""
|
||||
UPDATE durable_operation
|
||||
SET state = 'CANCELED', completed_at = ?, lease_owner = NULL,
|
||||
lease_acquired_at = NULL, lease_expires_at = NULL,
|
||||
row_version = row_version + 1
|
||||
WHERE operation_id = ? AND principal = ? AND state IN ('PENDING', 'RUNNING')
|
||||
""")) {
|
||||
statement.setTimestamp(1, Timestamp.from(NOW.plusSeconds(5)));
|
||||
statement.setString(2, operationId);
|
||||
statement.setString(3, principal);
|
||||
return statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> reclaim(Instant now) throws SQLException {
|
||||
return idsFrom(
|
||||
"""
|
||||
UPDATE durable_operation
|
||||
SET state = 'PENDING', lease_owner = NULL, lease_acquired_at = NULL,
|
||||
lease_expires_at = NULL, row_version = row_version + 1
|
||||
WHERE state = 'RUNNING' AND lease_expires_at <= ?
|
||||
RETURNING operation_id
|
||||
""",
|
||||
now);
|
||||
}
|
||||
|
||||
private List<String> expireStale(Instant now) throws SQLException {
|
||||
return idsFrom(
|
||||
"""
|
||||
UPDATE durable_operation
|
||||
SET state = 'EXPIRED', completed_at = ?, lease_owner = NULL,
|
||||
lease_acquired_at = NULL, lease_expires_at = NULL,
|
||||
row_version = row_version + 1
|
||||
WHERE state IN ('PENDING', 'RUNNING') AND expires_at <= ?
|
||||
RETURNING operation_id
|
||||
""",
|
||||
now,
|
||||
now);
|
||||
}
|
||||
|
||||
private List<String> idsFrom(String sql, Instant... arguments) throws SQLException {
|
||||
try (Connection connection = support.connection();
|
||||
PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||
for (int index = 0; index < arguments.length; index++) {
|
||||
statement.setTimestamp(index + 1, Timestamp.from(arguments[index]));
|
||||
}
|
||||
List<String> ids = new ArrayList<>();
|
||||
try (ResultSet rows = statement.executeQuery()) {
|
||||
while (rows.next()) {
|
||||
ids.add(rows.getString(1));
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
}
|
||||
|
||||
private String state(String operationId) throws SQLException {
|
||||
return column(operationId, "state");
|
||||
}
|
||||
|
||||
private String leaseOwner(String operationId) throws SQLException {
|
||||
return column(operationId, "lease_owner");
|
||||
}
|
||||
|
||||
private String resultReference(String operationId) throws SQLException {
|
||||
return column(operationId, "result_reference");
|
||||
}
|
||||
|
||||
private Instant startedAt(String operationId) throws SQLException {
|
||||
try (Connection connection = support.connection();
|
||||
PreparedStatement statement =
|
||||
connection.prepareStatement(
|
||||
"SELECT started_at FROM durable_operation WHERE operation_id = ?")) {
|
||||
statement.setString(1, operationId);
|
||||
try (ResultSet rows = statement.executeQuery()) {
|
||||
rows.next();
|
||||
return rows.getTimestamp(1).toInstant();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String column(String operationId, String name) throws SQLException {
|
||||
try (Connection connection = support.connection();
|
||||
PreparedStatement statement =
|
||||
connection.prepareStatement(
|
||||
"SELECT " + name + " FROM durable_operation WHERE operation_id = ?")) {
|
||||
statement.setString(1, operationId);
|
||||
try (ResultSet rows = statement.executeQuery()) {
|
||||
rows.next();
|
||||
return rows.getString(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void execute(String sql) throws SQLException {
|
||||
try (Connection connection = support.connection();
|
||||
Statement statement = connection.createStatement()) {
|
||||
statement.execute(sql);
|
||||
}
|
||||
}
|
||||
|
||||
private static String migration() throws IOException {
|
||||
try (InputStream resource =
|
||||
DurableOperationStoreContractTest.class
|
||||
.getClassLoader()
|
||||
.getResourceAsStream("db/migration/postgresql/V11__durable_operation.sql")) {
|
||||
if (resource == null) {
|
||||
throw new IOException("the durable operation migration is missing from the classpath");
|
||||
}
|
||||
return new String(resource.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
}
|
||||
+92
-7
@@ -1,9 +1,15 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.operation;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.operation.entity.DurableOperationEntity;
|
||||
import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport;
|
||||
import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction;
|
||||
import dev.caskeleton.application.idempotency.RequestFingerprint;
|
||||
import dev.caskeleton.application.operation.DurableOperationClaim;
|
||||
import dev.caskeleton.application.operation.DurableOperationId;
|
||||
import dev.caskeleton.application.operation.DurableOperationStorePort;
|
||||
import dev.caskeleton.application.operation.DurableOperationSubmission;
|
||||
import dev.caskeleton.application.operation.testkit.DurableOperationStoreContract;
|
||||
import dev.caskeleton.application.transaction.TransactionPort;
|
||||
import jakarta.persistence.EntityManagerFactory;
|
||||
@@ -13,13 +19,22 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Supplier;
|
||||
import javax.sql.DataSource;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
@@ -32,9 +47,10 @@ import org.springframework.transaction.PlatformTransactionManager;
|
||||
/**
|
||||
* The shipped adapter, through Spring Data and a Spring transaction, against real PostgreSQL.
|
||||
*
|
||||
* <p>The raw-SQL {@link DurableOperationStoreContractTest} proves the schema; this proves the code
|
||||
* that talks to it. A resubmission only breaks where PostgreSQL aborts the transaction after a
|
||||
* constraint violation, so no fake can stand in here.
|
||||
* <p>{@link DurableOperationSchemaConstraintTest} proves the table's constraints; this proves the
|
||||
* code that talks to it. A resubmission only breaks where PostgreSQL aborts the transaction after a
|
||||
* constraint violation, and two claims only race where two transactions hold row locks at once, so
|
||||
* no fake can stand in here.
|
||||
*/
|
||||
@Tag("jpa-contract")
|
||||
class JpaDurableOperationStoreContractTest extends DurableOperationStoreContract {
|
||||
@@ -45,7 +61,8 @@ class JpaDurableOperationStoreContractTest extends DurableOperationStoreContract
|
||||
@BeforeAll
|
||||
static void start() throws SQLException, IOException {
|
||||
support = JpaPlatformContractSupport.start();
|
||||
execute(migration());
|
||||
execute(migration("V11__durable_operation.sql"));
|
||||
execute(migration("V23__durable_operation_lease_fence.sql"));
|
||||
context = new AnnotationConfigApplicationContext();
|
||||
context.getBeanFactory().registerSingleton("dataSource", support.dataSource());
|
||||
context.register(AdapterConfiguration.class);
|
||||
@@ -67,6 +84,74 @@ class JpaDurableOperationStoreContractTest extends DurableOperationStoreContract
|
||||
execute("TRUNCATE durable_operation");
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoWorkersClaimingAtOnceTakeDifferentOperations() throws Exception {
|
||||
// given
|
||||
Instant now = Instant.parse("2026-08-25T09:00:00Z");
|
||||
inTransaction(() -> store().submit(submission("op-1", "a", now)));
|
||||
inTransaction(() -> store().submit(submission("op-2", "b", now.plusSeconds(1))));
|
||||
CountDownLatch firstClaimed = new CountDownLatch(1);
|
||||
CountDownLatch secondDone = new CountDownLatch(1);
|
||||
ExecutorService pool = Executors.newSingleThreadExecutor();
|
||||
|
||||
// when
|
||||
List<String> claimed;
|
||||
try {
|
||||
// The first claim holds its row lock until the second has claimed. With plain FOR UPDATE the
|
||||
// second would block behind it; with no lock at all both would take op-1.
|
||||
CompletableFuture<DurableOperationClaim> first =
|
||||
CompletableFuture.supplyAsync(
|
||||
() ->
|
||||
inTransaction(
|
||||
() -> {
|
||||
DurableOperationClaim claim = claimNow("worker-a", now);
|
||||
firstClaimed.countDown();
|
||||
awaitOrFail(secondDone);
|
||||
return claim;
|
||||
}),
|
||||
pool);
|
||||
awaitOrFail(firstClaimed);
|
||||
DurableOperationClaim second = inTransaction(() -> claimNow("worker-b", now));
|
||||
secondDone.countDown();
|
||||
claimed =
|
||||
List.of(
|
||||
first.get(30, TimeUnit.SECONDS).operationId().value(), second.operationId().value());
|
||||
} finally {
|
||||
pool.shutdownNow();
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(claimed).containsExactlyInAnyOrder("op-1", "op-2");
|
||||
}
|
||||
|
||||
private DurableOperationClaim claimNow(String workerId, Instant now) {
|
||||
return store().claimNext(workerId, Duration.ofMinutes(2), now).orElseThrow();
|
||||
}
|
||||
|
||||
private static void awaitOrFail(CountDownLatch latch) {
|
||||
try {
|
||||
if (!latch.await(30, TimeUnit.SECONDS)) {
|
||||
throw new IllegalStateException("the other claim did not arrive within 30 seconds");
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("interrupted while waiting for the other claim", interrupted);
|
||||
}
|
||||
}
|
||||
|
||||
private static DurableOperationSubmission submission(
|
||||
String operationId, String hashLetter, Instant submittedAt) {
|
||||
return new DurableOperationSubmission(
|
||||
new DurableOperationId(operationId),
|
||||
"reports.generate",
|
||||
"alice",
|
||||
null,
|
||||
new RequestFingerprint(hashLetter.repeat(64)),
|
||||
"{}",
|
||||
submittedAt,
|
||||
Duration.ofDays(1));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected DurableOperationStorePort store() {
|
||||
return context.getBean(JpaDurableOperationStore.class);
|
||||
@@ -84,13 +169,13 @@ class JpaDurableOperationStoreContractTest extends DurableOperationStoreContract
|
||||
}
|
||||
}
|
||||
|
||||
private static String migration() throws IOException {
|
||||
private static String migration(String file) throws IOException {
|
||||
try (InputStream resource =
|
||||
JpaDurableOperationStoreContractTest.class
|
||||
.getClassLoader()
|
||||
.getResourceAsStream("db/migration/postgresql/V11__durable_operation.sql")) {
|
||||
.getResourceAsStream("db/migration/postgresql/" + file)) {
|
||||
if (resource == null) {
|
||||
throw new IOException("the durable operation migration is missing from the classpath");
|
||||
throw new IOException("the durable operation migration " + file + " is missing");
|
||||
}
|
||||
return new String(resource.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
+51
@@ -2,12 +2,16 @@ package dev.caskeleton.adapter.outbound.persistence.platform;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import dev.caskeleton.adapter.outbound.persistence.hibernate.stateless.HibernateStatelessSessionRunner;
|
||||
import dev.caskeleton.adapter.outbound.persistence.hibernate.stateless.StatelessRowCapExceededException;
|
||||
import dev.caskeleton.adapter.outbound.persistence.hibernate.stateless.StatelessWorkName;
|
||||
import dev.caskeleton.adapter.outbound.persistence.hibernate.stateless.StatelessWorkResult;
|
||||
import dev.caskeleton.adapter.outbound.persistence.testkit.id.SequenceEntity;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
@@ -104,6 +108,53 @@ class HibernateStatelessSessionRunnerIntegrationTest {
|
||||
.hasMessageContaining("positive row cap");
|
||||
}
|
||||
|
||||
@Test
|
||||
void workStillRunningWhenTheTransactionTimeIsUpIsCancelledAndRolledBack() {
|
||||
// given
|
||||
var runner =
|
||||
new HibernateStatelessSessionRunner(
|
||||
jpa.sessionFactory(), Set.of(WORK), Duration.ofSeconds(1));
|
||||
long before = countRows();
|
||||
|
||||
// when
|
||||
Throwable failure =
|
||||
catchThrowable(
|
||||
() ->
|
||||
runner.execute(
|
||||
WORK,
|
||||
10,
|
||||
session -> {
|
||||
session.insert(new SequenceEntity("timed-out"));
|
||||
var unused =
|
||||
session
|
||||
.createNativeQuery("SELECT 1 FROM pg_sleep(5)", Integer.class)
|
||||
.getSingleResult();
|
||||
return StatelessWorkResult.of(1, 1);
|
||||
}));
|
||||
|
||||
// then
|
||||
assertThat(List.of(failure != null && causedByTimeout(failure), countRows()))
|
||||
.as(
|
||||
"the transaction's bound cancels the statement; nothing the work wrote is kept: %s",
|
||||
failure)
|
||||
.containsExactly(true, before);
|
||||
}
|
||||
|
||||
/** Whether the failure is the database or Hibernate stopping work that ran out of time. */
|
||||
private static boolean causedByTimeout(Throwable failure) {
|
||||
for (Throwable current = failure; current != null; current = current.getCause()) {
|
||||
String message = String.valueOf(current.getMessage()).toLowerCase(Locale.ROOT);
|
||||
if (current instanceof org.hibernate.QueryTimeoutException
|
||||
|| current instanceof jakarta.persistence.QueryTimeoutException
|
||||
|| message.contains("canceling statement due to statement timeout")
|
||||
|| message.contains("canceling statement due to user request")
|
||||
|| message.contains("transaction timeout expired")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static long countRows() {
|
||||
return jpa.inTransaction(
|
||||
entityManager ->
|
||||
|
||||
+44
-24
@@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.platform.experimental;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import com.zaxxer.hikari.HikariConfig;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
@@ -12,6 +13,7 @@ import dev.caskeleton.application.tenant.TenantId;
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
@@ -51,9 +53,9 @@ class TenantPoolCapacityContractTest {
|
||||
void budgetRefusesTheTenantThatWouldExceedTheCeiling() {
|
||||
var budget = new TenantPoolBudget(4, 8);
|
||||
|
||||
budget.requireCapacity(3, 6);
|
||||
assertThat(budget.hasCapacity(4, 8)).isFalse();
|
||||
assertThatThrownBy(() -> budget.requireCapacity(4, 8))
|
||||
budget.requireCapacity(3, 6, 2);
|
||||
assertThat(budget.hasCapacity(3, 7, 2)).isFalse();
|
||||
assertThatThrownBy(() -> budget.requireCapacity(3, 7, 2))
|
||||
.isInstanceOf(IllegalStateException.class);
|
||||
}
|
||||
|
||||
@@ -77,30 +79,50 @@ class TenantPoolCapacityContractTest {
|
||||
void openingPastTheBudgetIsRefusedBeforeAPoolIsCreated() {
|
||||
var budget = new TenantPoolBudget(2, 4);
|
||||
try (var registry =
|
||||
new TenantDataSourceRegistry(
|
||||
TenantPoolCapacityContractTest::poolFor,
|
||||
TenantPoolCapacityContractTest::poolSizeOf,
|
||||
budget)) {
|
||||
new TenantDataSourceRegistry(this::poolFor, tenant -> POOL_SIZE_PER_TENANT, budget)) {
|
||||
registry.openTenants(2);
|
||||
assertThat(registry.openPools()).isEqualTo(2);
|
||||
assertThat(registry.allocatedConnections()).isEqualTo(4);
|
||||
|
||||
assertThatThrownBy(() -> registry.openTenants(3)).isInstanceOf(IllegalStateException.class);
|
||||
assertThat(registry.openPools())
|
||||
.as("a refused tenant must not leave a half-built pool behind")
|
||||
.isEqualTo(2);
|
||||
assertThat(List.of(registry.openPools(), opened.size()))
|
||||
.as("a refused tenant must not leave a half-built pool behind, or have one built at all")
|
||||
.containsExactly(2, 2);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void aTenantWhosePoolWouldPassTheConnectionCeilingIsRefusedBeforeItsPoolOpens() {
|
||||
// given
|
||||
// Two pools of two use four of five connections; the third pool would make six.
|
||||
var budget = new TenantPoolBudget(4, 5);
|
||||
List<Object> outcome;
|
||||
try (var registry =
|
||||
new TenantDataSourceRegistry(this::poolFor, tenant -> POOL_SIZE_PER_TENANT, budget)) {
|
||||
registry.openTenants(2);
|
||||
|
||||
// when
|
||||
Throwable refused = catchThrowable(() -> registry.require(new TenantId("tenant-2")));
|
||||
outcome =
|
||||
List.of(
|
||||
refused instanceof IllegalStateException,
|
||||
registry.openPools(),
|
||||
registry.allocatedConnections(),
|
||||
opened.size());
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(outcome)
|
||||
.as("the third pool is refused on its planned size, so it never opens a connection")
|
||||
.containsExactly(true, 2, 4, 2);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("every opened tenant pool reaches the real server")
|
||||
void everyOpenedTenantPoolReachesTheServer() throws SQLException {
|
||||
var budget = new TenantPoolBudget(3, 6);
|
||||
try (var registry =
|
||||
new TenantDataSourceRegistry(
|
||||
TenantPoolCapacityContractTest::poolFor,
|
||||
TenantPoolCapacityContractTest::poolSizeOf,
|
||||
budget)) {
|
||||
new TenantDataSourceRegistry(this::poolFor, tenant -> POOL_SIZE_PER_TENANT, budget)) {
|
||||
registry.openTenants(3);
|
||||
|
||||
for (TenantId tenant : List.copyOf(registry.openTenantIds())) {
|
||||
@@ -118,10 +140,7 @@ class TenantPoolCapacityContractTest {
|
||||
void evictingATenantReturnsItsConnectionsToTheBudget() {
|
||||
var budget = new TenantPoolBudget(2, 4);
|
||||
try (var registry =
|
||||
new TenantDataSourceRegistry(
|
||||
TenantPoolCapacityContractTest::poolFor,
|
||||
TenantPoolCapacityContractTest::poolSizeOf,
|
||||
budget)) {
|
||||
new TenantDataSourceRegistry(this::poolFor, tenant -> POOL_SIZE_PER_TENANT, budget)) {
|
||||
registry.openTenants(2);
|
||||
TenantId evicted = List.copyOf(registry.openTenantIds()).get(0);
|
||||
|
||||
@@ -133,18 +152,19 @@ class TenantPoolCapacityContractTest {
|
||||
}
|
||||
}
|
||||
|
||||
private static javax.sql.DataSource poolFor(TenantId tenant) {
|
||||
/** Every pool this test's factory built, so a test can see one was never opened. */
|
||||
private final List<HikariDataSource> opened = new ArrayList<>();
|
||||
|
||||
private javax.sql.DataSource poolFor(TenantId tenant) {
|
||||
var config = new HikariConfig();
|
||||
config.setJdbcUrl(support.jdbcUrl());
|
||||
config.setUsername(support.username());
|
||||
config.setPassword(support.password());
|
||||
config.setMaximumPoolSize(POOL_SIZE_PER_TENANT);
|
||||
config.setPoolName("tenant-" + tenant.value());
|
||||
return new HikariDataSource(config);
|
||||
}
|
||||
|
||||
private static int poolSizeOf(javax.sql.DataSource pool) {
|
||||
return ((HikariDataSource) pool).getMaximumPoolSize();
|
||||
HikariDataSource pool = new HikariDataSource(config);
|
||||
opened.add(pool);
|
||||
return pool;
|
||||
}
|
||||
|
||||
private static int serverMaxConnections() throws SQLException {
|
||||
|
||||
+30
@@ -96,6 +96,36 @@ final class FileserverPostgreSqlStack implements AutoCloseable {
|
||||
return fileId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits until some session of this database is blocked on a lock, so a case can hold one
|
||||
* transaction open until a second one is queued behind it without sleeping a guessed interval.
|
||||
*/
|
||||
void awaitASessionWaitingOnALock(Duration deadline) {
|
||||
Instant giveUp = Instant.now().plus(deadline);
|
||||
while (Instant.now().isBefore(giveUp)) {
|
||||
Integer waiting =
|
||||
jdbc.queryForObject(
|
||||
"select count(*) from pg_stat_activity"
|
||||
+ " where datname = current_database() and wait_event_type = 'Lock'",
|
||||
Integer.class);
|
||||
if (waiting != null && waiting > 0) {
|
||||
return;
|
||||
}
|
||||
Thread.onSpinWait();
|
||||
}
|
||||
throw new IllegalStateException("no session waited on a lock within " + deadline);
|
||||
}
|
||||
|
||||
/** Counts the rows of {@code fs_recovery_item} for one file, open and resolved. */
|
||||
int recoveryItemsOf(FileId fileId) {
|
||||
Integer rows =
|
||||
jdbc.queryForObject(
|
||||
"select count(*) from fs_recovery_item where file_id = ?",
|
||||
Integer.class,
|
||||
fileId.value());
|
||||
return rows == null ? -1 : rows;
|
||||
}
|
||||
|
||||
/** Deletes every Fileserver row, children first. */
|
||||
void empty() {
|
||||
jdbc.execute(
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user