From afdedabf2a396caa1446926af2e6ad20bc01a205 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 13 Nov 2026 13:17:33 +0900 Subject: [PATCH 1/4] =?UTF-8?q?fix(persistence)!:=20=EB=84=98=EA=B2=A8?= =?UTF-8?q?=EB=B0=9B=EC=9D=80=20claim=EB=A7=8C=20=EC=93=B0=EA=B2=8C=20fenc?= =?UTF-8?q?e=EC=99=80=20=EC=A1=B0=EA=B1=B4=EB=B6=80=20=EC=93=B0=EA=B8=B0?= =?UTF-8?q?=EB=A5=BC=20=EB=A7=9E=EC=B6=98=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lease를 넘겨받는 저장소 다섯 곳이 앞 실행의 쓰기를 막지 못한다. durable operation은 owner만 보고 써서 같은 worker id가 다시 claim하면 앞 실행의 결과가 남는다. 알림 lease 회복은 상태를 다시 읽은 뒤 조건 없이 되돌려 끝난 delivery를 다시 보낸다. 만료 sweep·전체 읽음은 기다린 행의 상태를 다시 보지 않는다. 멱등성 reaper는 앱 시계로 비교하고 한 번에 다 지우며, 같은 키 요청이 되살린 기록까지 지운다. fileserver recovery enqueue는 동시 삽입에서 25P02로 실패하고, cleanup은 claim lease가 끝난 뒤에도 항목을 시작한다. 저장소가 행에서 만드는 claim·lease record는 잘못된 값을 IllegalArgumentException으로 거절해 web이 400으로 바꾼다. 고른 방법: 모든 쓰기를 owner·fence(·상태·만료) 조건을 단 한 문장으로 바꾼다. durable operation은 V23 lease_fence를 더하고 DurableOperationClaim 을 돌려준다. 알림 회복은 requeueAbandoned 한 문장이다. 만료 sweep은 subquery가 SKIP LOCKED로 쥔 행을 건너뛰고 바깥 UPDATE가 상태·만료를 다시 본다. 전체 읽음은 바깥 UPDATE가 상태를 다시 본다. reaper는 DB now()로 1000행씩, 트랜잭션마다 30초(SpringTransaction.DEFAULT_TIMEOUT_SECONDS)이고 바깥 DELETE에도 만료를 둔다. recovery는 INSERT … ON CONFLICT, cleanup은 lease 마지막 2분에 시작하지 않고 시도를 세지 않는 release로 돌려준다. 알림 provider timeout은 lease보다 짧아야 기동한다(호출이 lease를 넘으면 lease 복구가 아직 도는 호출을 맞추기로 넘긴다). 저장소가 만드는 record 일곱은 IllegalStateException으로 거절한다. 버린 대안: 읽고 비교한 뒤 쓰기(경합이 그대로 남음), 앱 쪽 락(인스턴스 사이에서 무의미). 위험·한계: 포트 시그니처가 바뀐다(DurableOperationStorePort, RecipientLeaseStorePort.expiredDispatching, RecipientDeliveryStorePort .requeueAbandoned, CleanupQueuePort.release). owner가 없거나 fence가 0인 V4 전 DISPATCHING 행은 회복이 고르지 않는다(안내 4행에 고치는 SQL). provider timeout 검사는 필요조건일 뿐이고, 호출 전체를 묶는 것은 SES다. 검증: desktop RED(고치기 전 트리 + 새 시험, pr1-red3) 새 시험 13건 실패. GREEN은 PR 본문의 verify.sh pr·PostgreSQL lane 결과. Refs: RR9-01-02, RR9-01-04, RR9-01-06, RR9-01-12, RR9-01-13, RR9-01-23, RR9-01-24, RR9-01-29, R1 넘김 DurableOperationStoreContractTest BREAKING CHANGE: DurableOperationStorePort의 heartbeat·reportProgress·succeed·fail이 DurableOperationClaim을 받고 claimNext가 그것을 돌려준다. RecipientLeaseStorePort.expiredDispatching이 List를 돌려주고 RecipientDeliveryStorePort.requeueAbandoned, CleanupQueuePort.release가 새로 생긴다. IdempotencyReaper 생성자에서 Clock이 빠진다. 알림 provider timeout이 lease 이상이면 기동이 실패한다. 저장소가 만드는 claim·lease record가 IllegalStateException을 던진다. docs/upgrade/2026-11-round9-claim-fence.md --- docs/architecture/jpa-api-surface.txt | 3 +- docs/ledger/estimates.jsonl | 1 + docs/manual/04-persistence.md | 11 + docs/notification/configuration-reference.md | 9 +- docs/registries/env-keys.yaml | 3 +- docs/upgrade/2026-11-round9-claim-fence.md | 38 ++ .../NotificationPlatformSettings.java | 36 ++ .../NotificationDispatchProperties.java | 5 +- .../NotificationPlatformSettingsTest.java | 33 ++ .../fileserver/JpaCleanupQueue.java | 6 + .../fileserver/JpaRecoveryQueue.java | 19 +- .../fileserver/entity/RecoveryItemEntity.java | 11 - .../FileserverCleanupRepository.java | 26 + .../FileserverRecoveryRepository.java | 31 +- .../outbound/persistence-jpa/AGENTS.md | 3 + .../outbound/persistence-jpa/README.md | 29 +- .../autoconfigure/JpaAdapterBeansConfig.java | 4 +- .../idempotency/jpa/IdempotencyReaper.java | 23 +- .../jpa/IdempotencyRecordJpaRepository.java | 35 +- .../DurableOperationJpaRepository.java | 24 +- .../operation/JpaDurableOperationStore.java | 74 +-- .../outbox/jpa/JpaOutboxStore.java | 2 +- .../outbox/jpa/OutboxEventJpaRepository.java | 6 +- .../persistence/outbox/jpa/OutboxReaper.java | 10 +- .../transaction/BatchedDeletes.java | 36 ++ .../transaction/SpringTransaction.java | 13 +- .../V23__durable_operation_lease_fence.sql | 24 + .../IdempotencyReaperIntegrationTest.java | 205 ++++++++ ...ostgreSqlIdempotencyStoreContractTest.java | 3 +- .../notification/NotificationJpaStack.java | 30 ++ ...greSqlRecipientLeaseStoreContractTest.java | 72 +++ .../InboxBulkReadVersionContractTest.java | 51 ++ .../DurableOperationSchemaConstraintTest.java | 214 ++++++++ .../DurableOperationStoreContractTest.java | 474 ------------------ .../JpaDurableOperationStoreContractTest.java | 99 +++- .../readiness/FileserverPostgreSqlStack.java | 30 ++ .../PostgreSqlMigrationIntegrationTest.java | 2 +- ...ficationBackgroundPassIntegrationTest.java | 27 +- .../PostgreSqlRecoveryQueueContractTest.java | 69 +++ .../jpa/IdempotencyReaperTest.java | 56 ++- .../resources/db/applied-migrations.sha256 | 1 + .../persistence-notification/AGENTS.md | 3 +- .../platform/JpaRecipientDeliveryStore.java | 17 + .../platform/JpaRecipientLeaseStore.java | 31 +- .../repository/InboxItemJpaRepository.java | 21 +- .../platform/repository/InboxItemSql.java | 13 + .../repository/RecipientClaimSql.java | 51 +- .../RecipientDeliveryJpaRepository.java | 41 +- .../operation/DurableOperationClaim.java | 45 ++ .../operation/DurableOperationStorePort.java | 42 +- .../application/operation/OperationLease.java | 8 +- .../outbox/ClaimedOutboxEvent.java | 8 +- .../application/outbox/OutboxLease.java | 8 +- .../CancelDurableOperationUseCaseTest.java | 5 +- .../operation/StoreBuiltRecordsTest.java | 80 +++ .../ClaimedOutboxEventContractTest.java | 3 +- .../DurableOperationStoreContract.java | 180 ++++++- .../InMemoryDurableOperationStore.java | 71 +-- .../fileserver/cleanup/CleanupQueuePort.java | 10 + .../cleanup/DefaultCleanupService.java | 32 +- .../cleanup/CleanupServiceTest.java | 89 +++- .../testkit/CleanupQueuePortContract.java | 34 ++ .../testkit/FakeCleanupContentGateway.java | 13 + .../testkit/InMemoryCleanupQueue.java | 22 + .../platform/admin/AdminOperationClaim.java | 6 +- .../dispatch/LeaseRecoveryService.java | 37 +- .../dispatch/RecipientDeliveryStorePort.java | 24 + .../platform/dispatch/RecipientLease.java | 8 +- .../dispatch/RecipientLeaseStorePort.java | 24 +- .../platform/dispatch/ReconciliationJob.java | 8 +- .../dispatch/LeaseRecoveryServiceTest.java | 34 ++ .../dispatch/StoreBuiltRecordsTest.java | 76 +++ .../testkit/InMemoryDeliveryAttemptStore.java | 23 +- .../testkit/InMemoryRecipientQueue.java | 42 +- .../RecipientDeliveryStoreContract.java | 78 +++ .../testkit/RecipientLeaseStoreContract.java | 62 ++- .../config/module-defaults/notification.yml | 5 +- 77 files changed, 2345 insertions(+), 757 deletions(-) create mode 100644 docs/upgrade/2026-11-round9-claim-fence.md create mode 100644 src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/BatchedDeletes.java create mode 100644 src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V23__durable_operation_lease_fence.sql create mode 100644 src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/IdempotencyReaperIntegrationTest.java create mode 100644 src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSchemaConstraintTest.java delete mode 100644 src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationStoreContractTest.java create mode 100644 src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemSql.java create mode 100644 src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java create mode 100644 src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java create mode 100644 src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java diff --git a/docs/architecture/jpa-api-surface.txt b/docs/architecture/jpa-api-surface.txt index e2d3f24a..0063f347 100644 --- a/docs/architecture/jpa-api-surface.txt +++ b/docs/architecture/jpa-api-surface.txt @@ -5,7 +5,7 @@ # root yet. # Update only after review with: # ./gradlew :adapter:outbound:persistence-jpa:updateJpaApiSurface -PapproveJpaApiSurfaceChange -# types: 165 +# types: 166 dev.caskeleton.adapter.outbound.persistence.api.PersistenceOperationName dev.caskeleton.adapter.outbound.persistence.api.capability.CapabilitySupport dev.caskeleton.adapter.outbound.persistence.api.capability.JpaCapability @@ -169,5 +169,6 @@ dev.caskeleton.adapter.outbound.persistence.springdata.SafeSortField dev.caskeleton.adapter.outbound.persistence.springdata.SafeSortMapper dev.caskeleton.adapter.outbound.persistence.springdata.SafeSortRegistry dev.caskeleton.adapter.outbound.persistence.springdata.ScrollPolicy +dev.caskeleton.adapter.outbound.persistence.transaction.BatchedDeletes dev.caskeleton.adapter.outbound.persistence.transaction.CallerTransaction dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction diff --git a/docs/ledger/estimates.jsonl b/docs/ledger/estimates.jsonl index a506c749..8efac6e1 100644 --- a/docs/ledger/estimates.jsonl +++ b/docs/ledger/estimates.jsonl @@ -15,3 +15,4 @@ {"schema": 1, "kind": "actual", "task": "w2-be-ci-perf", "transcript": "a6b6b25f-a4f7-46c2-a746-5ab20a588c78", "minutes": 663, "active_minutes": 417, "calls": 464, "output_tokens": 438927, "cache_read_tokens": 256417797, "max_context": 911512, "within": "over", "result": "partial", "commits": ["ecc14004", "6325b518", "86488256", "c7aff0f4", "7054d179", "575aa1f7", "deeb1126", "96caf997", "cfc80ae8", "000a59ed", "5a95db45", "d237bacc", "7284b581", "14053060", "7dbf11ca", "2c180b90", "24dde586", "aeb60339", "083c6910"], "why": "시간이 p80을 넘었다: desktop 공유 잠금 대기자 10여 개(flock이 순서 없이 깨움)로 측정·검증이 2시간 넘게 밀렸고, runner 1대 대기열에서 늦게 풀린 job이 맨 뒤로 가 수동 실행 결과가 몇 시간씩 늦었다. 예측에 없던 결정: Gitea needs+if Blocked 재현(일회용 workflow 셋), Trivy가 처음 읽은 lockfile 라이선스와 이미지가 jar 라이선스를 못 읽는 것에 맞춘 판정 재설계. 결과는 partial: runner 초록은 NFS(소유자 sudo)·compose(수정 뒤 runner 미확인) 빼고 달성, 속도 목표 넷은 미달(캐시 효과는 병합 뒤, PG 컨테이너 159개 공유는 후속).", "recorded_at": "2026-11-12T08:34+09:00"} {"schema": 1, "kind": "actual", "task": "w2-be-ci-perf", "correction": true, "transcript": "a6b6b25f-a4f7-46c2-a746-5ab20a588c78", "minutes": 1266, "active_minutes": 555, "calls": 556, "output_tokens": 534063, "cache_read_tokens": 301335415, "max_context": 963079, "within": "over", "result": "partial", "commits": ["ecc14004", "6325b518", "86488256", "c7aff0f4", "7054d179", "575aa1f7", "deeb1126", "96caf997", "cfc80ae8", "000a59ed", "5a95db45", "d237bacc", "7284b581", "14053060", "7dbf11ca", "2c180b90", "24dde586", "aeb60339", "083c6910", "335fc967", "9682c52c", "3e06a809", "d17f8bcf", "7db40c9b"], "why": "앞 actual 줄을 고친다. 그 뒤 총괄 지시 둘(새 main 위로, main.yml concurrency와 취소 증거)과 main 두 번 병합으로 작업 중 138분·호출 92가 더 들었다. 대부분 runner 대기열(PR run이 한 번에 2~4시간 뒤로 밀림)과 desktop 공유 잠금 대기였다. 결과는 여전히 partial: runner PR 130 초록·release gate 125 초록, NFS는 소유자 sudo, 속도 목표 넷은 미달(PR 13.5→12.0분/목표 8, release gate 45.6→32.5분/목표 35는 이미지 job 포함), 캐시 효과는 병합 뒤 측정.", "recorded_at": "2026-11-12T18:36+09:00"} {"schema": 1, "kind": "estimate", "task": "w2-be-ci-perf-2", "transcript": "a6b6b25f-a4f7-46c2-a746-5ab20a588c78", "started_at": "2026-11-12T18:45+09:00", "base": "e7be517e", "task_kind": "improve", "size": "large", "minutes": {"p30": 150, "p50": 240, "p80": 400}, "calls": {"p30": 120, "p50": 200, "p80": 320}, "output_tokens": {"p30": 100000, "p50": 170000, "p80": 280000}, "basis": "직전 라운드 w2-be-ci-perf 실측 555분(p50 144의 3.9배, 대부분 runner·desktop 잠금 대기). 이번 판단 5개(B2·NFS·C2·job 합치기·main 줄 순서), desktop main 측정 1회(30분)+pr 1회, runner nightly 1회(2시간)+release 1회+PR 2회. 대기가 시간을 정한다", "recorded_at": "2026-11-12T18:58+09:00"} +{"schema": 1, "kind": "estimate", "task": "r9-group-b", "transcript": "0ef672ab-db3e-427a-9f53-b2e918462534", "started_at": "2026-11-13T07:00:00+09:00", "base": "d3a90479", "task_kind": "improve", "size": "large", "minutes": {"p30": 190, "p50": 286, "p80": 424}, "calls": {"p30": 378, "p50": 567, "p80": 819}, "output_tokens": {"p30": 348000, "p50": 522000, "p80": 754000}, "basis": "발견 42건·PR 3·결정 약 15·desktop 실행 약 20회(Testcontainers lane)·코드 약 3000줄; 원값 360/540/800분·600/900/1300호출·600k/900k/1300k에 보정 0.53/0.63/0.58 곱함; 선례 w2-be-ci-perf 555분", "recorded_at": "2026-11-13T06:52+09:00"} diff --git a/docs/manual/04-persistence.md b/docs/manual/04-persistence.md index 040425ef..88720690 100644 --- a/docs/manual/04-persistence.md +++ b/docs/manual/04-persistence.md @@ -91,6 +91,17 @@ store는 자기 트랜잭션을 열지 않는다. 유스케이스가 연 트랜 예: `DurableOperationStoreContract`(`InMemoryDurableOperationStore`, `JpaDurableOperationStoreContractTest`). - 여러 인스턴스가 집는 작업 → `FOR UPDATE SKIP LOCKED`는 **트랜잭션 안에서**, 그리고 lease(만료 시각)와 fence 값을 함께 저장한다. 예: `ReconciliationJobStorePort.claimDue`(application-notification). + - 연장·완료·되돌리기는 그 claim의 owner와 fence를 `WHERE`에 넣은 조건부 UPDATE 하나다. fence는 claim마다 + 오르므로 같은 worker id가 다시 claim해도 앞 실행의 쓰기는 0행이다(예: `DurableOperationClaim`). + - `WHERE id IN (SELECT … LIMIT n)`으로 고른 행을 바꿀 때는 subquery를 `FOR UPDATE SKIP LOCKED`로 잠가 남이 쥔 행을 + 고르지 않는다. 상태 조건도 바깥 `UPDATE … WHERE`에 한 번 더 둔다. 잠금 절이 빠지면 READ COMMITTED는 잠금을 기다린 + 행에 바깥 조건만 다시 보고 subquery 안 조건은 보지 않기 때문이다(예: `RecipientClaimSql.EXPIRE_OVERDUE`). 잠그지 않는 + subquery라면 바깥 조건이 유일한 방어다(예: inbox `markAllRead`). + - 만료는 그 시각을 쓴 시계로 본다. row lease(outbox, 알림 수신자, durable operation, fileserver cleanup, + Mongo migration lock)는 앱 `Clock`으로 쓰고 앱 `Clock`으로 비교한다. 시험이 시계를 옮길 수 있게 하려는 + 것이고, 대신 인스턴스 사이 시계 차이가 가장 짧은 lease보다 훨씬 작아야 한다(NTP 동기화가 배포 조건이다). + DB가 쓴 시각은 DB 시계로(멱등성 `expires_at`은 `clock_timestamp()`로 쓰고 `now()`로 지운다), Redis가 쓴 + 시각은 Redis `TIME`으로 본다. 한 저장소 안에서 두 시계를 섞지 않는다. - 수정 충돌 → 버전 열(낙관적 잠금). 예: `DurableOperationEntity`(persistence-jpa)의 `@Version` 열 `row_version`. - **예외**: DB가 볼 수 없는 규칙(외부 시스템 상태)은 분산 락(`DistributedLockPort`)이나 멱등성 키(05장). - 동시성 테스트는 실 DB lane에서 동시 요청 N개 → 행 1개를 단언한다. diff --git a/docs/notification/configuration-reference.md b/docs/notification/configuration-reference.md index 1b298505..ca79ccd6 100644 --- a/docs/notification/configuration-reference.md +++ b/docs/notification/configuration-reference.md @@ -26,7 +26,7 @@ support leaf의 `ModuleDefaultsEnvironmentPostProcessor`가 가장 낮은 우선 | 속성 | 환경 변수 | 기본값 | 범위 | |---|---|---|---| | `dispatch.claim-batch-size` | `APP_NOTIFICATION_PLATFORM_CLAIM_BATCH_SIZE` | `50` | 1..1000 | -| `dispatch.lease-duration` | `APP_NOTIFICATION_PLATFORM_LEASE_DURATION` | `2m` | 양수, 유한 | +| `dispatch.lease-duration` | `APP_NOTIFICATION_PLATFORM_LEASE_DURATION` | `2m` | 양수, 유한. `poll-interval`과 켜진 provider profile의 `timeout`보다 길어야 기동한다 | | `dispatch.poll-interval` | `APP_NOTIFICATION_PLATFORM_POLL_INTERVAL` | `1s` | 양수, 유한 | | `dispatch.max-global-concurrency` | `APP_NOTIFICATION_PLATFORM_MAX_CONCURRENCY` | `64` | 양수 | | `dispatch.max-additional-attempts` | `APP_NOTIFICATION_PLATFORM_MAX_ADDITIONAL_ATTEMPTS` | `4` | 0 이상 | @@ -35,8 +35,11 @@ support leaf의 `ModuleDefaultsEnvironmentPostProcessor`가 가장 낮은 우선 모든 값에 상한이 있다. 「무제한」은 받는 설정이 아니다. -lease는 provider 호출과 그 시간 상한보다 길어야 한다. 그보다 짧으면, 살아 있는 worker가 아직 기다리는 전달을 두 번째 -worker가 claim하고, 수신자는 알림을 두 번 받는다. +lease는 provider 호출 한 번이 delivery를 쥐는 시간보다 길어야 한다. 그보다 짧으면 아직 기다리는 호출이 멈춘 lease로 +보인다. lease가 끝난 뒤 lease만큼 더 지나도 호출이 끝나지 않았으면, lease 복구가 그 delivery를 버려진 것으로 보고 맞추기로 +넘긴다. 기본 제공 provider(SES·SMTP)는 결과 조회가 없어 운영자 확인(`RECONCILIATION_REQUIRED`)으로 넘어간다. 그 뒤에 +끝난 호출도 자기 결과를 기록한다(owner와 fence가 그대로다). 두 번째 worker가 다시 보내지는 않는다. claim은 +`DISPATCHING`인 delivery를 가져가지 않는다. ## 콜백 diff --git a/docs/registries/env-keys.yaml b/docs/registries/env-keys.yaml index 3df26fc2..99a9b670 100644 --- a/docs/registries/env-keys.yaml +++ b/docs/registries/env-keys.yaml @@ -5063,7 +5063,8 @@ env_keys: required_test: adapter-contract:notification-dispatch-bounds - name: APP_NOTIFICATION_PLATFORM_LEASE_DURATION - # source: NTF-025 — provider 호출과 그 timeout보다 길어야 한다. 아니면 살아 있는 worker의 전달을 다른 worker가 claim한다 + # source: NTF-025 — poll-interval과 켜진 provider profile의 timeout보다 길어야 기동한다(2026-11-13부터 검사). + # 짧으면 살아 있는 worker의 전달을 다른 worker가 claim한다. 넉넉히 길게 둔다: lease는 claim 때 시작한다 type: duration default: 2m allowed_values: null diff --git a/docs/upgrade/2026-11-round9-claim-fence.md b/docs/upgrade/2026-11-round9-claim-fence.md new file mode 100644 index 00000000..99ae53f9 --- /dev/null +++ b/docs/upgrade/2026-11-round9-claim-fence.md @@ -0,0 +1,38 @@ +# 업그레이드 안내 — 2026-11 claim·fence·트랜잭션 시한·정리 + +- 범위: durable operation·알림 수신자·fileserver cleanup·멱등성 기록의 lease, fence, 조건부 쓰기, 정리를 고친 + 변경. 포트 시그니처와 기동 조건이 바뀐 곳이 있다. 아래 표가 그 목록이다. +- 근거: 2026-11-12 전체 리뷰(`docs/reviews/2026-11-12-round9-review/`)의 claim·fence(B8) 발견 + +## 1. 누가 읽나 + +이 템플릿을 `upstream` remote로 두고 merge해 받는 파생 서비스가 읽는다. 특히 durable operation worker를 직접 쓴 서비스, +알림 recipient store를 직접 구현한 서비스, 알림 provider timeout을 설정한 서비스다. + +## 2. 할 일 + +| # | 할 일 | 안 하면 | 해당 서비스 | +| --- | --- | --- | --- | +| 1 | durable operation worker는 `claimNext`가 돌려주는 `DurableOperationClaim`을 들고 있다가 `heartbeat(claim, …)`·`reportProgress(claim, …)`·`succeed(claim, …)`·`fail(claim, …)`에 넘긴다. 예전에는 `(operationId, workerId, …)`였다 | 컴파일이 실패한다 | durable operation worker를 쓴 서비스 | +| 2 | `DurableOperationStorePort`를 직접 구현했으면 claim마다 오르는 fence를 저장하고, 네 쓰기가 owner와 fence를 함께 확인하게 한다. `DurableOperationStoreContract`의 새 사례(같은 worker id가 다시 claim한 뒤 앞 실행의 쓰기는 거절)로 확인한다 | 재시작한 worker나 한 프로세스의 두 실행이 같은 id를 쓰면 앞 실행의 결과가 넘겨받은 실행 위에 기록된다 | 자기 durable operation store를 둔 서비스 | +| 3 | persistence-jpa의 `V23__durable_operation_lease_fence.sql`(`durable_operation.lease_fence BIGINT NOT NULL DEFAULT 0`)이 기본 stream에 더해진다. 같은 번호의 서비스 마이그레이션이 있으면 번호를 옮긴다. `durable_operation`을 건드린 트랜잭션이 열려 있으면 5초 뒤 실패하고 V22에 남는다(다시 배포) | Flyway가 번호 충돌로 기동하지 않는다 | persistence-jpa를 쓰는 모든 서비스 | +| 4 | `RecipientLeaseStorePort.expiredDispatching`이 `List`(버려질 때의 owner·fence·만료 시각)를 돌려주고, `RecipientDeliveryStorePort`에 `requeueAbandoned(lease, nextDispatchAt)`가 생겼다. 직접 구현했으면 상태가 `DISPATCHING`이고 owner·fence가 같고 lease가 늘지 않았을 때만 `READY_TO_DISPATCH`로 바꾼다(`RecipientDeliveryStoreContract` 새 사례). owner가 없거나 fence가 0인 `DISPATCHING` 행(notification V4 전에 남은 행)은 복구가 더는 고르지 않는다. 그런 행은 `SELECT id FROM notification_recipient_delivery WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 찾는다. 배포 전에 `UPDATE notification_recipient_delivery SET lease_owner = COALESCE(lease_owner, 'pre-v4'), lease_fence = lease_fence + 1, lease_until = now() - interval '1 day', version = version + 1, updated_at = now() WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 owner를 채우고 fence를 올려 복구 패스에 넘긴다. 복구는 attempt를 보고 정한다: attempt가 없으면 다시 보낼 차례로 돌리고, 끝나지 않은 attempt가 있으면 맞추기로 넘긴다(그 뒤는 `docs/notification/configuration-reference.md` 「발송」의 lease 문단). 손으로 `READY_TO_DISPATCH`로 바꾸지 않는다. 발송 중에 멈춘 행이면 같은 알림이 한 번 더 나간다. owner가 없는 행은 stuck lease 지표에도 잡히지 않는다 | 컴파일이 실패한다. 조건 없이 바꾸면 끝났거나 다시 claim된 delivery가 다시 발송된다 | 알림 store를 직접 구현한 서비스 | +| 5 | 알림 provider profile의 `timeout`이 `app.notification.platform.dispatch.lease-duration`보다 짧은지 확인한다. 짧은 것은 필요조건일 뿐이다: lease는 claim 때 시작해서 attempt 기록·렌더링·provider 허가 대기도 lease를 쓴다. lease는 timeout보다 넉넉히 길게 둔다. 이 timeout이 provider 호출 전체를 묶는 것은 SES(API 호출 시간 상한)다. SMTP는 `spring.mail` socket timeout이 명령마다 묶을 뿐이라 이 검사만으로는 호출 전체가 lease 안에 끝난다고 말할 수 없다 | 짧지 않으면 기동이 실패한다(`notification provider profile '…': timeout (…) must be shorter than …lease-duration`). 전에는 provider 호출 하나가 lease보다 길어도 기동했다. 그때 생기는 일은 `docs/notification/configuration-reference.md` 「발송」의 lease 문단이다 | 알림 capability를 켠 서비스 | +| 6 | `IdempotencyReaper`를 직접 만들던 코드는 `new IdempotencyReaper(repository)`로 바꾼다(`Clock` 인자 삭제). 만료 비교는 DB 시계(`now()`)이고 1000행씩 지운다 | 컴파일이 실패한다 | reaper를 직접 조립한 서비스 | +| 7 | fileserver cleanup 묶음의 `max-items`를 정할 때, 한 묶음이 claim lease(10분)의 첫 8분 안에 시작할 수 있는 만큼만 처리된다는 것을 안다. 나머지는 시도하지 않은 채 queue에 돌아가 다음 묶음이 다시 claim한다. 시도 횟수는 그대로다(바이트 예산이 바닥나 돌려준 항목도 이제 같다. 전에는 시도 한 번으로 세어 8번째에 보류됐다). 직접 구현한 `CleanupQueuePort`는 새 연산 `release`를 구현한다(`CleanupQueuePortContract` 새 사례) | 느린 저장소에서 큰 `max-items`는 그 사유로 도는 항목이 늘어난다. 전에는 lease가 끝난 뒤에도 남은 항목을 처리해 다른 worker와 겹쳤다 | fileserver capability를 켠 서비스 | + +## 3. 바뀐 동작 (따라 할 일 없음) + +- reaper 둘(outbox·멱등성)의 트랜잭션은 `SpringTransaction.DEFAULT_TIMEOUT_SECONDS`(30초)로 묶인다. + `app.persistence-jpa.transaction.*`를 바꿔도 reaper 시한은 그대로다. 한 트랜잭션이 1000행 한 묶음만 지운다. +- 저장소가 행에서 만드는 `DurableOperationClaim`·`OperationLease`·`RecipientLease`·`OutboxLease`·`ClaimedOutboxEvent`· + `ReconciliationJob`·`AdminOperationClaim`이 값을 거절할 때 `IllegalStateException`을 던진다 + (전에는 `IllegalArgumentException`). 요청에서 오는 값이 아니라 서버 결함이다. 이 IAE를 잡던 코드는 ISE를 잡는다. +- 알림 lease 회복은 버려질 때의 lease 그대로일 때만 delivery를 다시 넣는다. 다른 인스턴스가 이미 다시 넣었거나 holder가 + 끝냈으면 아무것도 쓰지 않고 처리 수에 세지 않는다. +- 알림 만료 sweep(`EXPIRE_OVERDUE`)은 다른 트랜잭션이 쥔 행을 건너뛰고, 바깥 `UPDATE`에서도 상태·만료를 다시 본다. + inbox 전체 읽음(`markAllRead`)도 바깥 `UPDATE`에서 상태를 다시 본다. 기다리는 사이 claim된 delivery나 보관된 inbox 항목을 + 바꾸지 않는다. +- 멱등성 reaper는 다른 트랜잭션이 쥔 기록을 건너뛰고, 바깥 `DELETE`에서도 만료를 다시 본다. 같은 키 요청이 만료된 기록을 + 되살리는 동안 돈 reap이 그 claim을 지우지 않는다. +- fileserver recovery 대기열 enqueue는 `INSERT … ON CONFLICT` 한 문장이다. 같은 파일을 동시에 넣어도 25P02로 실패하지 않는다. diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettings.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettings.java index 15e9eb5d..19a86e65 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettings.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettings.java @@ -37,6 +37,8 @@ public record NotificationPlatformSettings( providers = providers == null ? Map.of() : Map.copyOf(providers); boolean callbacksEnabled = callbacks.enabled(); providers.forEach((id, provider) -> provider.validate(id, callbacksEnabled)); + Duration leaseDuration = dispatch.leaseDuration(); + providers.forEach((id, provider) -> provider.requireCallFitsInLease(id, leaseDuration)); } /** Source-compatible constructor for callers that rely on the original settings shape. */ @@ -264,6 +266,40 @@ public record NotificationPlatformSettings( } } + /** + * Fail the boot when one call of this profile alone would outlast the dispatch lease. + * + *

A worker holds a delivery's lease while it waits on the provider, and the profile's + * timeout is how long that wait can be. A lease that ends first shows a live call as a stuck + * lease; if the call is still running one lease later, lease recovery takes the delivery for + * abandoned and hands it to reconciliation, and the shipped families (SES, SMTP) have no status + * query, so it is parked for an operator. A call that finishes after that still records its + * result, since it holds the same owner and fence. No second worker claims it: the claim never + * takes a dispatching delivery. Each value is fine on its own, so only the pair can be checked + * (round9 review 2026-11-12, RR9-01-24). + * + *

A necessary condition, not a sufficient one: the lease starts at the claim, so writing the + * attempt, rendering and waiting for a provider permit spend it before the call begins. A + * deployment keeps the lease well above the timeout rather than just above it. + * + * @param profileId the profile id + * @param leaseDuration {@code app.notification.platform.dispatch.lease-duration} + */ + void requireCallFitsInLease(String profileId, Duration leaseDuration) { + if (!enabled) { + return; + } + require( + timeout.compareTo(leaseDuration) < 0, + profileId, + "timeout (" + + timeout + + ") must be shorter than app.notification.platform.dispatch.lease-duration (" + + leaseDuration + + "): a provider call that outlasts its lease shows as a stuck lease, and one lease" + + " later lease recovery hands its delivery to reconciliation while it may still run"); + } + private static void require(boolean condition, String profileId, String message) { if (!condition) { throw new IllegalArgumentException( diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationDispatchProperties.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationDispatchProperties.java index 7cafab9b..cf00636b 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationDispatchProperties.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationDispatchProperties.java @@ -42,8 +42,9 @@ public record NotificationDispatchProperties( requirePositive(estimatedDispatchDuration, "estimatedDispatchDuration"); requirePositive(maxQueueAge, "maxQueueAge"); if (leaseDuration.compareTo(pollInterval) <= 0) { - // A lease shorter than the poll interval expires before the worker can renew it, so two - // workers would routinely claim the same job. + // A sanity bound, not a renewal rule (see RecipientLeaseStorePort.renew). The lease is how + // long one dispatch may hold a delivery and the poll interval only how long an idle worker + // sleeps, so a lease no longer than that nap is too short for any real send. throw new IllegalArgumentException("leaseDuration must exceed pollInterval"); } } diff --git a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettingsTest.java b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettingsTest.java index 05182ad6..91a446d6 100644 --- a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettingsTest.java +++ b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettingsTest.java @@ -31,6 +31,39 @@ class NotificationPlatformSettingsTest { .hasMessageContaining("unknown provider type '" + removedType + "'"); } + @Test + void aProfileWhoseTimeoutIsNotShorterThanTheLeaseFailsStartup() { + // given + Duration lease = NotificationPlatformSettings.Dispatch.defaults().leaseDuration(); + NotificationPlatformSettings.Provider slow = + new NotificationPlatformSettings.Provider( + "SMTP", true, false, "PRODUCTION", "smtp-key", lease, 8, 20, null); + + // when + Throwable failure = catchThrowable(() -> properties(slow)); + + // then + assertThat(failure) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("notification provider profile 'profile'") + .hasMessageContaining("must be shorter than app.notification.platform.dispatch.lease"); + } + + @Test + void aDisabledProfileIsNotHeldToTheLease() { + // given + Duration lease = NotificationPlatformSettings.Dispatch.defaults().leaseDuration(); + NotificationPlatformSettings.Provider off = + new NotificationPlatformSettings.Provider( + "SMTP", false, false, "PRODUCTION", "smtp-key", lease.multipliedBy(2), 8, 20, null); + + // when + NotificationPlatformSettings settings = properties(off); + + // then + assertThat(settings.providers()).containsOnlyKeys("profile"); + } + @Test void enabledSesRequiresItsTypedTransportIdentity() { assertThatThrownBy( diff --git a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaCleanupQueue.java b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaCleanupQueue.java index 8912277d..56eed895 100644 --- a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaCleanupQueue.java +++ b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaCleanupQueue.java @@ -118,6 +118,12 @@ public class JpaCleanupQueue implements CleanupQueuePort { settle(item, STATUS_FAILED, nextAttemptAt, reasonCode, clock.instant()); } + @Override + public void release(CleanupItem item, Instant nextAttemptAt) { + CallerTransaction.require("cleanup queue release"); + items.releaseClaim(item.cleanupId(), item.claimToken(), nextAttemptAt, clock.instant()); + } + @Override public void markAbandoned(CleanupItem item, String reasonCode) { CallerTransaction.require("cleanup queue markAbandoned"); diff --git a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaRecoveryQueue.java b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaRecoveryQueue.java index e56faffe..caa3ce2b 100644 --- a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaRecoveryQueue.java +++ b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaRecoveryQueue.java @@ -7,7 +7,6 @@ import dev.caskeleton.application.fileserver.api.FileId; import dev.caskeleton.application.fileserver.recovery.ReconciliationStatus; import dev.caskeleton.application.fileserver.recovery.RecoveryQueuePort; import java.time.Clock; -import java.time.Instant; import java.util.List; import java.util.UUID; import org.springframework.data.domain.Limit; @@ -15,10 +14,11 @@ import org.springframework.data.domain.Limit; /** * Durable recovery queue over {@code fs_recovery_item}. * - *

An enqueue is an upsert: the same file reported twice updates the open item rather than adding - * a second one. Reconciliation runs on a schedule and re-raises whatever it still cannot settle, so - * an append-only queue would grow one row per sweep per unresolved file and bury the distinct - * problems under repetitions of the same one. + *

An enqueue is an upsert, one {@code INSERT … ON CONFLICT} statement: the same file reported + * twice — in sequence or in two concurrent transactions — updates the open item rather than adding + * a second one or failing on the open-item index. Reconciliation runs on a schedule and re-raises + * whatever it still cannot settle, so an append-only queue would grow one row per sweep per + * unresolved file and bury the distinct problems under repetitions of the same one. * *

Resolution keeps the outcome rather than deleting the row. {@code UNRESOLVED} and {@code * QUARANTINE_REQUIRED} are the two answers a human has to act on, and both are worthless if the @@ -26,8 +26,6 @@ import org.springframework.data.domain.Limit; */ public class JpaRecoveryQueue implements RecoveryQueuePort { - private static final String STATUS_PENDING = "PENDING"; - private final FileserverRecoveryRepository items; private final Clock clock; @@ -39,12 +37,7 @@ public class JpaRecoveryQueue implements RecoveryQueuePort { @Override public void enqueue(FileId fileId, String reasonCode) { CallerTransaction.require("recovery queue enqueue"); - Instant now = clock.instant(); - if (items.refreshPending(fileId.value(), reasonCode, now) > 0) { - return; - } - items.save( - new RecoveryItemEntity(UUID.randomUUID(), fileId.value(), reasonCode, STATUS_PENDING, now)); + items.upsertPending(UUID.randomUUID(), fileId.value(), reasonCode, clock.instant()); } @Override diff --git a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/entity/RecoveryItemEntity.java b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/entity/RecoveryItemEntity.java index 020901fe..5d4cd9e8 100644 --- a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/entity/RecoveryItemEntity.java +++ b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/entity/RecoveryItemEntity.java @@ -52,17 +52,6 @@ public class RecoveryItemEntity extends AssignedIdEntity { return recoveryId; } - public RecoveryItemEntity( - UUID recoveryId, UUID fileId, String reasonCode, String status, Instant createdAt) { - this.recoveryId = recoveryId; - this.fileId = fileId; - this.reasonCode = reasonCode; - this.status = status; - this.attempt = 0; - this.createdAt = createdAt; - this.updatedAt = createdAt; - } - public UUID getRecoveryId() { return recoveryId; } diff --git a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverCleanupRepository.java b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverCleanupRepository.java index 0e302f72..80105229 100644 --- a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverCleanupRepository.java +++ b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverCleanupRepository.java @@ -63,6 +63,32 @@ public interface FileserverCleanupRepository extends JpaRepositoryThe item goes back to {@code PENDING} when it was never tried and to {@code FAILED} when an + * earlier attempt was — the two statuses a claim takes from — and keeps its counter and its last + * error. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query( + """ + update CleanupItemEntity c + set c.status = case when c.attempt = 0 then 'PENDING' else 'FAILED' end, + c.nextAttemptAt = :nextAttemptAt, + c.claimOwner = null, + c.claimToken = null, + c.leaseUntil = null, + c.updatedAt = :now + where c.cleanupId = :cleanupId + and c.claimToken = :token + """) + int releaseClaim( + @Param("cleanupId") UUID cleanupId, + @Param("token") UUID token, + @Param("nextAttemptAt") Instant nextAttemptAt, + @Param("now") Instant now); + /** * Returns items whose claim has expired, for a reaper to take over. * diff --git a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverRecoveryRepository.java b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverRecoveryRepository.java index 88f8f885..49792a40 100644 --- a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverRecoveryRepository.java +++ b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverRecoveryRepository.java @@ -22,23 +22,34 @@ public interface FileserverRecoveryRepository extends JpaRepository findPending(Limit limit); /** - * Re-raises an open item instead of adding a second one. + * Opens an item for the file, or re-raises the open one instead of adding a second, in one + * statement. * *

Reconciliation is retried on a schedule, so the same file reaches the queue repeatedly. One * open item per file keeps the queue a worklist rather than a failure log; the newest reason wins * because it describes the most recent evidence. + * + *

One {@code INSERT … ON CONFLICT} on the partial unique index {@code uq_fs_recovery_open} + * rather than an update followed by an insert when it matched nothing. Two enqueues of the same + * file in concurrent transactions both saw no open row, both inserted, and the second hit the + * index — a unique violation that aborts its transaction ({@code 25P02}) and fails the sweep that + * reported it. Here the second waits for the first to commit and then updates the row it wrote. */ @Modifying(clearAutomatically = true, flushAutomatically = true) @Query( - """ - update RecoveryItemEntity r - set r.reasonCode = :reasonCode, - r.attempt = r.attempt + 1, - r.updatedAt = :now - where r.fileId = :fileId - and r.status = 'PENDING' - """) - int refreshPending( + value = + """ + INSERT INTO fs_recovery_item + (recovery_id, file_id, reason_code, status, attempt, created_at, updated_at) + VALUES (:recoveryId, :fileId, :reasonCode, 'PENDING', 0, :now, :now) + ON CONFLICT (file_id) WHERE status = 'PENDING' + DO UPDATE SET reason_code = EXCLUDED.reason_code, + attempt = fs_recovery_item.attempt + 1, + updated_at = EXCLUDED.updated_at + """, + nativeQuery = true) + int upsertPending( + @Param("recoveryId") UUID recoveryId, @Param("fileId") UUID fileId, @Param("reasonCode") String reasonCode, @Param("now") Instant now); diff --git a/src/adapter/outbound/persistence-jpa/AGENTS.md b/src/adapter/outbound/persistence-jpa/AGENTS.md index e34653d4..26c31492 100644 --- a/src/adapter/outbound/persistence-jpa/AGENTS.md +++ b/src/adapter/outbound/persistence-jpa/AGENTS.md @@ -88,6 +88,9 @@ lock TTL)과 계약 표(트랜잭션 모드, SQLState → 오류 코드, vendor, `lease_token`을 올린다(`V17`). `markPublished`/`markFailed`/`markDead`는 `event_id` + `IN_FLIGHT` + owner + token에 대한 조건부 UPDATE 하나이고 `APPLIED`/`STALE`을 돌려준다. 완료는 행을 읽어서 덮어쓰지 않는다. V2 outbox 세대는 2026-09-25에 지웠다(`docs/capability-status.md`). +- durable operation fencing: `claimNext`가 claim의 행 lock 아래에서 `lease_fence`를 올리고(`V23`) `DurableOperationClaim` + 으로 돌려준다. heartbeat·progress·succeed·fail은 operation id + `RUNNING` + owner + fence에 대한 조건부 UPDATE 하나다. + 같은 worker id가 다시 claim하면 앞 실행의 쓰기는 0행이다. - `TransactionPort` 구현(`SpringTransaction`) — 애플리케이션의 트랜잭션 의도와 Spring `PlatformTransactionManager` 사이의 다리. - 감사 metadata 기반 + actor 이음매(`audit/AuditableEntity`, `audit/AuditContextPort`, `audit/DomainContextAuditContext`) — diff --git a/src/adapter/outbound/persistence-jpa/README.md b/src/adapter/outbound/persistence-jpa/README.md index 403e60e6..9269264e 100644 --- a/src/adapter/outbound/persistence-jpa/README.md +++ b/src/adapter/outbound/persistence-jpa/README.md @@ -389,6 +389,23 @@ PUBLISHED backlog 를 한 transaction 으로 지우면 그 사이 모든 row loc cadence 측정 필요). retention 한 값은 reaper-local 이라 `@Value` 로 받지만, canonical 6-property 문서는 app-bootstrap `OutboxSettings` / `application.yml` 에 있다. +## operation — `JpaDurableOperationStore` + +### claim과 fence (`V23`) + +lease가 지난 실행은 아직 살아서 일을 끝내는 중일 수 있다. 예전 heartbeat·progress·succeed·fail은 +`lease_owner = :workerId`만 확인해서, 같은 worker id가 다시 claim하면(재시작한 worker, 한 프로세스의 두 실행) +앞 실행의 결과가 넘겨받은 실행 위에 기록됐다(round9 리뷰 2026-11-12, RR9-01-02). outbox(`lease_token`), +알림 수신자(`lease_fence`), reconciliation(`claim_fence`), fileserver cleanup(`claim_token`)과 같은 모양으로 맞췄다. + +- **claim**: `claimNext(workerId, …)`가 row lock(`FOR UPDATE SKIP LOCKED`)을 쥔 상태에서 `lease_owner`, + `lease_fence = lease_fence + 1`을 기록하고 `DurableOperationClaim(submission, workerId, fence)`을 돌려준다. +- **쓰기**: heartbeat·progress·succeed·fail은 한 문장의 조건부 UPDATE다 — + `WHERE operation_id = ? AND state = 'RUNNING' AND lease_owner = ? AND lease_fence = ?`(heartbeat는 + `lease_expires_at > now`도). 1행이면 적용, 0행이면 그 claim은 더 이상 이 operation을 쥐지 않는다. +- fence는 줄지 않는다. reclaim·cancel·expire는 owner를 비우고 fence는 그대로 둔다. `V23` 전에 쓴 행은 0이고, + claim은 1 이상을 나눠 주므로 fence 0을 쥔 worker는 없다. + ## JPA 운영 capability 후보 `src/adapter/outbound/persistence-jpa/release/readiness-cards.yaml`이 capability와 독립 schema stream의 @@ -533,7 +550,7 @@ provider 선택(flag → bean → registry)의 정본은 설정 코드 `lock/Dis 반환 핸들은 `lock::unlock` 이다. lease TTL 이 `close()` 전에 만료된 `JdbcLockRegistry` 의 경우, 내부 `JdbcLock.unlock()` 이 `ConcurrentModificationException` 을 던진다(row 가 이미 회수됨). 이 어댑터는 여기서 일부러 잡지 않는다 — metered `distributedLockProvider` 데코레이터 -(컴포지션 루트)가 SI-LOCK-C5 계약을 소유한다: 로그 + `lock.lease.expired` metric 후 `close()` 에서 +(이 모듈의 `DistributedLockPersistenceConfig`가 감싼 `MeteredDistributedLock`)가 SI-LOCK-C5 계약을 소유한다: 로그 + `lock.lease.expired` metric 후 `close()` 에서 정상 return 해, 만료가 호출자의 보호작업 예외를 가리지 않게 한다. in-process `DefaultLockRegistry` 경로는 만료가 없어 그 `close()` 가 이 예외를 던질 수 없다. @@ -554,11 +571,11 @@ provider 선택(flag → bean → registry)의 정본은 설정 코드 `lock/Dis `DistributedLock.tryLock(waitTime, leaseTtl)` 로 전달한다. ### `LockSettings` -`app.lock.*` 에서 바인딩되는 yaml-only 기본값이다(`config/module-defaults/persistence-jpa.yml`). -새 `APP_*` env 키가 아니므로 `env-keys.yaml` 엔트리가 필요 없다. `DistributedLockPersistenceConfig`의 -`@EnableConfigurationProperties`가 바인딩한다. cross-field invariant: `leaseTtl >= waitTime` -이어야 한다 — TTL 이 waitTime 보다 먼저 만료되면 첫 holder 의 보호작업이 끝나기 전에 두 번째 -holder 가 락을 잡을 수 있다. +`app.lock.lease-ttl` 하나를 바인딩하는 yaml-only 기본값이다(`config/module-defaults/persistence-jpa.yml`). +새 `APP_*` env 키가 아니므로 `env-keys.yaml` 엔트리가 필요 없다. `DistributedLockPersistenceConfig#lockSettings`가 +`StrictSettingsBinder`로 바인딩해, `app.lock` 아래 모르는 키는 기동을 실패시킨다. 기다릴 시간은 호출자가 +`tryAcquire(key, waitTime, leaseTtl)`마다 넘긴다. 예전 `app.lock.wait-time`은 lease와 비교만 하고 읽는 곳이 +없었다(round9 리뷰 2026-11-12, RR9-01-07). 지금은 모르는 키라 설정해 두면 기동이 실패한다. ### `MeteredDistributedLock` (app-bootstrap에서 이동) - **`DistributedLockPort`를 감싸는 얇은 Micrometer 데코레이터다.** `tryAcquire` 호출마다 diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java index 18964a00..193187e5 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java @@ -154,8 +154,8 @@ class JpaAdapterBeansConfig { static class IdempotencyExpiry { @Bean - IdempotencyReaper idempotencyReaper(IdempotencyRecordJpaRepository repository, Clock clock) { - return new IdempotencyReaper(repository, clock); + IdempotencyReaper idempotencyReaper(IdempotencyRecordJpaRepository repository) { + return new IdempotencyReaper(repository); } } } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaper.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaper.java index 3e754839..053baef9 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaper.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaper.java @@ -1,32 +1,41 @@ package dev.caskeleton.adapter.outbound.persistence.idempotency.jpa; -import java.time.Clock; +import dev.caskeleton.adapter.outbound.persistence.transaction.BatchedDeletes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.transaction.annotation.Transactional; /** * Periodically deletes idempotency rows whose {@code expires_at} has passed. The PostgreSQL store * moves {@code expires_at} forward on every completion and failure, and a claim is reset in place * rather than re-inserted, so this is the only thing that bounds table growth. + * + *

Deletes in batches of {@link #BATCH_SIZE}, each committed on its own, until a batch comes back + * short — the outbox reaper's shape. Expiry is the database's clock, not this process's: the store + * wrote {@code expires_at} from {@code clock_timestamp()} (see {@link + * IdempotencyRecordJpaRepository#deleteExpiredBatch}). */ public class IdempotencyReaper { + /** Rows per delete statement and transaction. */ + static final int BATCH_SIZE = BatchedDeletes.BATCH_SIZE; + private static final Logger log = LoggerFactory.getLogger(IdempotencyReaper.class); private final IdempotencyRecordJpaRepository repository; - private final Clock clock; - public IdempotencyReaper(IdempotencyRecordJpaRepository repository, Clock clock) { + public IdempotencyReaper(IdempotencyRecordJpaRepository repository) { this.repository = repository; - this.clock = clock; } + /** + * Deletes every record whose expiry has passed by the database clock; returns the count. + * + *

Not transactional itself: each batch is its own transaction. + */ @Scheduled(fixedDelayString = "${app.capabilities.idempotency.reaper-interval:PT10M}") - @Transactional public int reap() { - int purged = repository.deleteExpired(clock.instant()); + int purged = BatchedDeletes.untilShort(repository::deleteExpiredBatch); if (purged > 0) { log.debug("idempotency reaper purged {} expired record(s)", purged); } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyRecordJpaRepository.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyRecordJpaRepository.java index 22a07ca7..0a8183d7 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyRecordJpaRepository.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyRecordJpaRepository.java @@ -1,12 +1,13 @@ package dev.caskeleton.adapter.outbound.persistence.idempotency.jpa; import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.entity.IdempotencyRecordEntity; -import java.time.Instant; +import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; import java.util.UUID; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.Repository; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; /** * Spring Data repository for {@link IdempotencyRecordEntity}: the reaper's expiry purge, and @@ -18,8 +19,34 @@ import org.springframework.data.repository.query.Param; */ public interface IdempotencyRecordJpaRepository extends Repository { - /** Reaper / TTL boundary (§E): purge every record whose expiry is at/before {@code now}. */ + /** + * Reaper / TTL boundary (§E): deletes at most {@code limit} records whose expiry has passed, in a + * transaction of its own unless the caller already has one; returns the count. + * + *

Expiry is judged by the database clock ({@code now()}) because that is the clock that wrote + * it: the PostgreSQL store sets {@code expires_at} from {@code clock_timestamp()}. Compared with + * the application's clock instead, a host whose clock runs ahead purged a record whose lease or + * replay window was still open. + * + *

Bounded for the reason the outbox purge is: a backlog of expired rows is what a reaper + * outage leaves behind, and one unbounded {@code DELETE} over it is a long transaction holding + * every deleted row's lock. + * + *

A same-key request reclaims an expired record in place (the store's reset). The delete must + * not take that claim: the next retry would run the work again. Two things stop it. The subquery + * locks the rows it picks and skips rows another transaction holds, so a record being reclaimed + * is not picked. The expiry is also a condition of the {@code DELETE} itself, so if the lock + * clause is ever lost — and a delete waits on the row and, under READ COMMITTED, re-checks only + * its own predicate — the record it waited for is still kept once it is no longer expired. + */ + @Transactional(timeout = SpringTransaction.DEFAULT_TIMEOUT_SECONDS) @Modifying - @Query("delete from IdempotencyRecordEntity e where e.expiresAt <= :now") - int deleteExpired(@Param("now") Instant now); + @Query( + value = + "DELETE FROM idempotency_record WHERE id IN (" + + "SELECT id FROM idempotency_record WHERE expires_at <= now()" + + " LIMIT :limit FOR UPDATE SKIP LOCKED)" + + " AND expires_at <= now()", + nativeQuery = true) + int deleteExpiredBatch(@Param("limit") int limit); } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationJpaRepository.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationJpaRepository.java index 3d17b6ab..38924d6b 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationJpaRepository.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationJpaRepository.java @@ -17,10 +17,11 @@ import org.springframework.data.repository.query.Param; * claim-then-check has every worker fetch the same head row and all but one retry, and a plain * {@code FOR UPDATE} makes them queue behind each other so the pool serialises. * - *

Every state-changing statement carries the owning worker in its {@code WHERE} clause and - * returns the affected count. A worker whose lease lapsed while it was still working must not be - * able to record a result over the worker that took over; the update simply matches nothing, and - * the caller learns it lost the lease from the count. + *

Every state-changing statement carries the owning worker and the fence of its claim in its + * {@code WHERE} clause and returns the affected count. A run whose lease lapsed while it was still + * working must not be able to record a result over the run that took over — even when both runs use + * the same worker id, which only the fence tells apart; the update simply matches nothing, and the + * caller learns it lost the lease from the count. */ public interface DurableOperationJpaRepository extends Repository { @@ -79,6 +80,8 @@ public interface DurableOperationJpaRepository extends RepositoryRunnable is PENDING, or RUNNING whose lease has lapsed. The second half is the crash * recovery path: without it an operation claimed by a worker that then died is never picked up * again, and the queue stops draining while every row still reads RUNNING. + * + * @return one row of {@code [operation_id, lease_fence]} when a row was claimed, none otherwise */ @Modifying(clearAutomatically = true, flushAutomatically = true) @Query( @@ -90,6 +93,7 @@ public interface DurableOperationJpaRepository extends Repository claimNext( + List claimNext( @Param("workerId") String workerId, @Param("leaseExpiresAt") Instant leaseExpiresAt, @Param("now") Instant now); @@ -119,12 +123,14 @@ public interface DurableOperationJpaRepository extends Repository :now """, nativeQuery = true) int heartbeat( @Param("operationId") String operationId, @Param("workerId") String workerId, + @Param("fence") long fence, @Param("leaseExpiresAt") Instant leaseExpiresAt, @Param("now") Instant now); @@ -141,11 +147,13 @@ public interface DurableOperationJpaRepository extends Repository claimNext( + public Optional claimNext( String workerId, Duration leaseDuration, Instant now) { requireCallerTransaction("claimNext"); - List claimed = repository.claimNext(workerId, now.plus(leaseDuration), now); + List claimed = repository.claimNext(workerId, now.plus(leaseDuration), now); if (claimed.isEmpty()) { return Optional.empty(); } + String operationId = (String) claimed.get(0)[0]; + long fence = ((Number) claimed.get(0)[1]).longValue(); return repository - .findById(claimed.get(0)) + .findById(operationId) .map( entity -> - new DurableOperationSubmission( - new DurableOperationId(entity.getOperationId()), - entity.getOperationName(), - entity.getPrincipal(), - entity.getTenant().isEmpty() ? null : entity.getTenant(), - new RequestFingerprint(entity.getRequestHash()), - entity.getPayload(), - entity.getSubmittedAt(), - Duration.between(entity.getSubmittedAt(), entity.getExpiresAt()))); + new DurableOperationClaim( + new DurableOperationSubmission( + new DurableOperationId(entity.getOperationId()), + entity.getOperationName(), + entity.getPrincipal(), + entity.getTenant().isEmpty() ? null : entity.getTenant(), + new RequestFingerprint(entity.getRequestHash()), + entity.getPayload(), + entity.getSubmittedAt(), + Duration.between(entity.getSubmittedAt(), entity.getExpiresAt())), + workerId, + fence)); } @Override - public boolean heartbeat( - DurableOperationId operationId, String workerId, Duration leaseDuration, Instant now) { + public boolean heartbeat(DurableOperationClaim claim, Duration leaseDuration, Instant now) { requireCallerTransaction("heartbeat"); - return repository.heartbeat(operationId.value(), workerId, now.plus(leaseDuration), now) == 1; + return repository.heartbeat( + claim.operationId().value(), + claim.workerId(), + claim.fence(), + now.plus(leaseDuration), + now) + == 1; } @Override - public boolean reportProgress( - DurableOperationId operationId, String workerId, OperationProgressSnapshot progress) { + public boolean reportProgress(DurableOperationClaim claim, OperationProgressSnapshot progress) { requireCallerTransaction("reportProgress"); return repository.reportProgress( - operationId.value(), - workerId, + claim.operationId().value(), + claim.workerId(), + claim.fence(), progress.completedUnits(), progress.totalUnits().orElse(null), progress.phase().orElse(null)) @@ -124,25 +135,24 @@ public class JpaDurableOperationStore implements DurableOperationStorePort { } @Override - public boolean succeed( - DurableOperationId operationId, - String workerId, - String resultReference, - Instant completedAt) { + public boolean succeed(DurableOperationClaim claim, String resultReference, Instant completedAt) { requireCallerTransaction("succeed"); - return repository.succeed(operationId.value(), workerId, resultReference, completedAt) == 1; + return repository.succeed( + claim.operationId().value(), + claim.workerId(), + claim.fence(), + resultReference, + completedAt) + == 1; } @Override - public boolean fail( - DurableOperationId operationId, - String workerId, - OperationFailure failure, - Instant completedAt) { + public boolean fail(DurableOperationClaim claim, OperationFailure failure, Instant completedAt) { requireCallerTransaction("fail"); return repository.fail( - operationId.value(), - workerId, + claim.operationId().value(), + claim.workerId(), + claim.fence(), failure.code(), failure.detail(), failure.retryable(), diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStore.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStore.java index e23e34ca..9d8158b1 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStore.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStore.java @@ -378,7 +378,7 @@ public class JpaOutboxStore OutboxEventStatus.valueOf(entity.getStatus()), entity.getAttemptCount(), entity.getDeferredAttemptCount()); - } catch (IllegalArgumentException exception) { + } catch (IllegalArgumentException | IllegalStateException exception) { throw new IllegalStateException( "invalid canonical outbox metadata for eventId=" + entity.getEventId(), exception); } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxEventJpaRepository.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxEventJpaRepository.java index 9f4d6d0c..65de4485 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxEventJpaRepository.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxEventJpaRepository.java @@ -1,6 +1,7 @@ package dev.caskeleton.adapter.outbound.persistence.outbox.jpa; import dev.caskeleton.adapter.outbound.persistence.outbox.jpa.entity.OutboxEventEntity; +import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; import java.time.Instant; import java.util.List; import java.util.Optional; @@ -34,9 +35,10 @@ public interface OutboxEventJpaRepository extends RepositoryBounded because a backlog of published rows is exactly what a relay outage leaves behind, * and one unbounded {@code DELETE} over it is a long transaction holding every deleted row's - * lock. The key subquery is plain SQL that PostgreSQL and H2 both run. + * lock. The key subquery is plain SQL that PostgreSQL and H2 both run. The transaction has the + * default bound, as the other reapers' do. */ - @Transactional + @Transactional(timeout = SpringTransaction.DEFAULT_TIMEOUT_SECONDS) @Modifying @Query( value = diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxReaper.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxReaper.java index e9042775..a90f66cf 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxReaper.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxReaper.java @@ -1,5 +1,6 @@ package dev.caskeleton.adapter.outbound.persistence.outbox.jpa; +import dev.caskeleton.adapter.outbound.persistence.transaction.BatchedDeletes; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -18,7 +19,7 @@ public class OutboxReaper { /** Rows per delete statement and transaction. */ // ponytail: fixed batch size; make it a setting if a deployment's backlog needs another value. - static final int BATCH_SIZE = 1_000; + static final int BATCH_SIZE = BatchedDeletes.BATCH_SIZE; private static final Logger log = LoggerFactory.getLogger(OutboxReaper.class); @@ -41,12 +42,7 @@ public class OutboxReaper { @Scheduled(fixedDelayString = "${app.outbox.reaper-interval:PT10M}") public int reap() { Instant cutoff = clock.instant().minus(retention); - int purged = 0; - int deleted; - do { - deleted = repository.deletePublishedBatch(cutoff, BATCH_SIZE); - purged += deleted; - } while (deleted == BATCH_SIZE); + int purged = BatchedDeletes.untilShort(limit -> repository.deletePublishedBatch(cutoff, limit)); if (purged > 0) { log.debug("outbox reaper purged {} published row(s)", purged); } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/BatchedDeletes.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/BatchedDeletes.java new file mode 100644 index 00000000..4889a740 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/BatchedDeletes.java @@ -0,0 +1,36 @@ +package dev.caskeleton.adapter.outbound.persistence.transaction; + +import java.util.function.IntUnaryOperator; + +/** + * Deletes a backlog in bounded batches, one transaction each, until a batch comes back short. + * + *

Every reaper here has the same reason to: a backlog of expired rows is what a reaper outage + * leaves behind, and one unbounded {@code DELETE} over it is a long transaction holding every + * deleted row's lock. The outbox, idempotency and live-event reapers each held their own copy of + * this loop; one copy is the one that cannot drift. + */ +public final class BatchedDeletes { + + /** The most rows one batch deletes. */ + public static final int BATCH_SIZE = 1_000; + + private BatchedDeletes() {} + + /** + * Runs {@code batch} until it deletes fewer than {@link #BATCH_SIZE} rows. + * + * @param batch deletes at most its argument's number of rows, in a transaction of its own, and + * returns how many it deleted + * @return how many rows all the batches deleted + */ + public static int untilShort(IntUnaryOperator batch) { + int purged = 0; + int deleted; + do { + deleted = batch.applyAsInt(BATCH_SIZE); + purged += deleted; + } while (deleted == BATCH_SIZE); + return purged; + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransaction.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransaction.java index aad85fd9..10475d66 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransaction.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransaction.java @@ -115,6 +115,15 @@ public final class SpringTransaction implements TransactionPort { return execute(requiresNewTemplate, action); } + /** + * The default bound, in whole seconds, of every transaction this leaf opens: {@link + * Timeouts#DEFAULT}, and the reapers' own transactions, whose {@code @Transactional(timeout)} + * needs a compile-time constant. A deployment's {@code app.persistence-jpa.transaction.*} moves + * the first and not the second; a reaper deletes one bounded batch per transaction, so the + * default is ample for it whatever the application's units of work need. + */ + public static final int DEFAULT_TIMEOUT_SECONDS = 30; + /** * How long one {@link SpringTransaction} write or read transaction may run (review 04-10). * @@ -140,7 +149,9 @@ public final class SpringTransaction implements TransactionPort { /** 30 seconds each; see the class comment for why. */ public static final Timeouts DEFAULT = - new Timeouts(Duration.ofSeconds(30), Duration.ofSeconds(30)); + new Timeouts( + Duration.ofSeconds(DEFAULT_TIMEOUT_SECONDS), + Duration.ofSeconds(DEFAULT_TIMEOUT_SECONDS)); public Timeouts { requireWholePositiveSeconds(write, "write"); diff --git a/src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V23__durable_operation_lease_fence.sql b/src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V23__durable_operation_lease_fence.sql new file mode 100644 index 00000000..1aef212e --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V23__durable_operation_lease_fence.sql @@ -0,0 +1,24 @@ +-- A fence on the durable operation lease. +-- +-- Until now a heartbeat, a progress report and a completion checked only lease_owner. A worker that +-- restarts under the same id, or one process running two executions under one id, claims the +-- operation again, and the earlier run — still alive and still finishing — then matches the same +-- lease_owner and records its result over the run that took over. The outbox (lease_token), the +-- notification recipient lease (lease_fence), the reconciliation claim (claim_fence) and the +-- fileserver claim (claim_token) all fence; this was the one store that did not (round9 review +-- 2026-11-12, RR9-01-02). +-- +-- Every claim increments lease_fence and returns it; every later write names the fence it holds in +-- its WHERE clause. NOT NULL DEFAULT 0 is a constant default, so PostgreSQL 11+ records it in the +-- catalog and rewrites no row; rows written before this keep 0 until their next claim, and no +-- worker holds fence 0 because a claim hands out 1 or more. + +-- Bound the wait for the lock this migration takes (ACCESS EXCLUSIVE on durable_operation, held +-- until it commits). Unbounded, it queues behind any open transaction that has touched the table, +-- and every submit, poll and claim queues behind the migration. Five seconds, then the migration +-- fails, rolls back whole and the database stays at V22: deploy again once that transaction has +-- ended (docs/jpa/migration-guide.md "Bound the lock wait"). +SET LOCAL lock_timeout = '5s'; + +ALTER TABLE durable_operation + ADD COLUMN lease_fence BIGINT NOT NULL DEFAULT 0; diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/IdempotencyReaperIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/IdempotencyReaperIntegrationTest.java new file mode 100644 index 00000000..ec92e02d --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/IdempotencyReaperIntegrationTest.java @@ -0,0 +1,205 @@ +package dev.caskeleton.adapter.outbound.persistence.idempotency.postgresql; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.IdempotencyReaper; +import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.IdempotencyRecordJpaRepository; +import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; +import jakarta.persistence.EntityManagerFactory; +import java.sql.Connection; +import java.sql.Statement; +import java.time.Duration; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.data.jpa.repository.support.JpaRepositoryFactory; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.SharedEntityManagerCreator; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import org.springframework.transaction.TransactionManager; +import org.springframework.transaction.annotation.AnnotationTransactionAttributeSource; +import org.springframework.transaction.interceptor.TransactionInterceptor; + +/** + * The idempotency reaper's delete against a real PostgreSQL: expiry is the database clock's, and a + * backlog larger than one batch is drained. + * + *

The store writes {@code expires_at} from {@code clock_timestamp()}, so the rows here are + * written the same way, relative to the database's {@code now()}. A reaper that compared them with + * the application's clock purged a live record whenever the host clock ran ahead of the database's + * (round9 review 2026-11-12, RR9-01-04); the reaper now has no clock of its own to compare with. + * + *

The repository is the production shape: its own {@code @Transactional} opens each batch's + * transaction, through the same interceptor Spring Data installs, and no test transaction wraps the + * reap. + */ +@Tag("jpa-contract") +class IdempotencyReaperIntegrationTest { + + private static JpaPlatformContractSupport support; + private static JdbcTemplate jdbc; + private static LocalContainerEntityManagerFactoryBean persistence; + private static IdempotencyReaper reaper; + + @BeforeAll + static void start() { + support = JpaPlatformContractSupport.start(); + Flyway.configure() + .dataSource(support.dataSource()) + .locations("classpath:db/migration/postgresql") + .table("flyway_schema_history") + .baselineOnMigrate(false) + .outOfOrder(false) + .load() + .migrate(); + jdbc = new JdbcTemplate(support.dataSource()); + persistence = new LocalContainerEntityManagerFactoryBean(); + persistence.setDataSource(support.dataSource()); + persistence.setJpaVendorAdapter(new HibernateJpaVendorAdapter()); + persistence.setPackagesToScan( + "dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.entity"); + Properties properties = new Properties(); + properties.setProperty("hibernate.jdbc.time_zone", "UTC"); + persistence.setJpaProperties(properties); + persistence.afterPropertiesSet(); + EntityManagerFactory entityManagers = persistence.getObject(); + JpaTransactionManager transactionManager = new JpaTransactionManager(entityManagers); + JpaRepositoryFactory repositories = + new JpaRepositoryFactory( + SharedEntityManagerCreator.createSharedEntityManager(entityManagers)); + repositories.addRepositoryProxyPostProcessor( + (proxy, information) -> + proxy.addAdvice( + new TransactionInterceptor( + (TransactionManager) transactionManager, + new AnnotationTransactionAttributeSource()))); + reaper = + new IdempotencyReaper(repositories.getRepository(IdempotencyRecordJpaRepository.class)); + } + + @AfterAll + static void stop() { + if (persistence != null) { + persistence.destroy(); + } + if (support != null) { + support.close(); + } + } + + @BeforeEach + void empty() { + jdbc.update("delete from idempotency_record"); + } + + @Test + void aRecordTheDatabaseClockHasNotExpiredSurvivesTheReap() { + // given + insert("live", 1, "now() + interval '1 hour'"); + insert("expired", 1, "now() - interval '1 second'"); + + // when + reap(); + + // then + assertThat(jdbc.queryForList("select idempotency_key from idempotency_record", String.class)) + .containsExactly("live-0"); + } + + @Test + void backlogLargerThanOneBatchIsDrainedInOneReap() { + // given + insert("expired", 2_501, "now() - interval '1 minute'"); + + // when + int purged = reap(); + + // then + assertThat(List.of(purged, count())).containsExactly(2_501, 0); + } + + @Test + void aRecordReclaimedWhileTheReapRunsSurvivesIt() throws Exception { + // given + insert("reset", 1, "now() - interval '1 second'"); + try (Connection reclaim = support.dataSource().getConnection()) { + reclaim.setAutoCommit(false); + try (Statement statement = reclaim.createStatement()) { + // What the store's reset does to an expired record a same-key request finds: it holds the + // row and claims it again in place. + statement.executeUpdate( + "update idempotency_record set status = 'CLAIMED'," + + " expires_at = now() + interval '1 hour' where idempotency_key = 'reset-0'"); + } + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + Future reaping = executor.submit(() -> reap()); + awaitDoneOrBlocked(reaping, Duration.ofSeconds(5)); + reclaim.commit(); + + // when + reaping.get(30, TimeUnit.SECONDS); + } + } + + // then + assertThat(jdbc.queryForList("select status from idempotency_record", String.class)) + .as("the reap deleted the claim a same-key request had just taken") + .containsExactly("CLAIMED"); + } + + private static int reap() { + return reaper.reap(); + } + + /** + * Returns once the reap has finished or is waiting on a row lock, whichever comes first, and + * fails the test when neither happened in time: committing the reclaim before the reap had run + * would pass without the race this test exists for. + */ + private static void awaitDoneOrBlocked(Future reaping, Duration within) + throws InterruptedException { + long deadline = System.nanoTime() + within.toNanos(); + while (!reaping.isDone()) { + Integer waiting = + jdbc.queryForObject( + "select count(*) from pg_locks where not granted and locktype = 'transactionid'", + Integer.class); + if (waiting != null && waiting > 0) { + return; + } + if (System.nanoTime() >= deadline) { + throw new AssertionError( + "the reap neither finished nor waited on the reclaimed row within " + within); + } + Thread.sleep(20); + } + } + + private static int count() { + Integer rows = jdbc.queryForObject("select count(*) from idempotency_record", Integer.class); + return rows == null ? -1 : rows; + } + + private static void insert(String keyPrefix, int rows, String expiresAtSql) { + jdbc.update( + "insert into idempotency_record (id, principal, idempotency_key, use_case_name," + + " request_hash, status, created_at, expires_at)" + + " select gen_random_uuid(), 'alice', ? || '-' || n, 'orders.create'," + + " repeat('a', 64), 'COMPLETED', now(), " + + expiresAtSql + + " from generate_series(0, ? - 1) as n", + keyPrefix, + rows); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/PostgreSqlIdempotencyStoreContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/PostgreSqlIdempotencyStoreContractTest.java index cac6ff47..8825edf8 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/PostgreSqlIdempotencyStoreContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/PostgreSqlIdempotencyStoreContractTest.java @@ -7,7 +7,6 @@ import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction import dev.caskeleton.application.idempotency.IdempotencyStorePort; import dev.caskeleton.application.idempotency.testkit.IdempotencyStoreContract; import jakarta.persistence.EntityManagerFactory; -import java.time.Clock; import java.time.Duration; import java.util.Properties; import java.util.function.Supplier; @@ -85,7 +84,7 @@ class PostgreSqlIdempotencyStoreContractTest extends IdempotencyStoreContract { new JpaRepositoryFactory( SharedEntityManagerCreator.createSharedEntityManager(entityManagers)) .getRepository(IdempotencyRecordJpaRepository.class); - reaper = new IdempotencyReaper(repository, Clock.systemUTC()); + reaper = new IdempotencyReaper(repository); reaperTransactions = new TransactionTemplate(new JpaTransactionManager(entityManagers)); } diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationJpaStack.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationJpaStack.java index 8408e920..72b52075 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationJpaStack.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationJpaStack.java @@ -3,7 +3,10 @@ package dev.caskeleton.adapter.outbound.persistence.notification; import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; import jakarta.persistence.EntityManager; import jakarta.persistence.EntityManagerFactory; +import java.time.Duration; +import java.time.Instant; import java.util.Map; +import java.util.concurrent.Future; import java.util.function.Supplier; import org.flywaydb.core.Flyway; import org.springframework.data.jpa.repository.support.JpaRepositoryFactory; @@ -69,6 +72,33 @@ public final class NotificationJpaStack implements AutoCloseable { return new JdbcTemplate(database.dataSource()); } + /** + * Waits until {@code work} has finished or some session of this database is blocked on a lock. + * + *

A race case holds one transaction open and starts a second statement that may or may not + * need the first's row lock; this is how it knows the second has either finished or queued behind + * the first before it lets the first commit, without sleeping a guessed interval. + */ + public void awaitDoneOrBlocked(Future work, Duration deadline) { + Instant giveUp = Instant.now().plus(deadline); + JdbcTemplate jdbc = jdbc(); + while (Instant.now().isBefore(giveUp)) { + if (work.isDone()) { + return; + } + Integer waiting = + jdbc.queryForObject( + "SELECT count(*) FROM pg_stat_activity" + + " WHERE datname = current_database() AND wait_event_type = 'Lock'", + Integer.class); + if (waiting != null && waiting > 0) { + return; + } + Thread.onSpinWait(); + } + throw new IllegalStateException("work neither finished nor waited on a lock in " + deadline); + } + /** * Deletes every row of the delivery tables, children first, so a contract case starts empty. * diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/PostgreSqlRecipientLeaseStoreContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/PostgreSqlRecipientLeaseStoreContractTest.java index fe4134f7..a4bec546 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/PostgreSqlRecipientLeaseStoreContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/PostgreSqlRecipientLeaseStoreContractTest.java @@ -15,11 +15,17 @@ import dev.caskeleton.application.notification.platform.dispatch.NotificationReq import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryRecord; import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort; import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort; +import dev.caskeleton.application.notification.platform.dispatch.testkit.ContractClock; import dev.caskeleton.application.notification.platform.dispatch.testkit.NotificationRows; import dev.caskeleton.application.notification.platform.dispatch.testkit.RecipientLeaseStoreContract; import java.time.Duration; import java.util.List; import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -114,6 +120,72 @@ class PostgreSqlRecipientLeaseStoreContractTest extends RecipientLeaseStoreContr .isGreaterThan(before); } + @Test + void anExpirySweepRacingAClaimLeavesTheClaimedJobDispatching() throws Exception { + // given + NotificationRequestRecord request = + NotificationRows.request(new TenantId("tenant-a"), "idem-race", NOW); + RecipientDeliveryRecord job = + NotificationRows.job( + request, + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(NOW), + Optional.of(NOW.plusSeconds(10)), + NOW); + inTransaction(() -> requests().insert(request, List.of(job))); + CountDownLatch claimed = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + ExecutorService pool = Executors.newFixedThreadPool(2); + + // when + List outcome; + try { + // The claim holds its row lock, uncommitted, while the sweep runs at an instant past the + // job's expiry. The sweep used to pick the id from the committed READY row, wait for the + // lock, and then retire the row the claim had just made DISPATCHING. + CompletableFuture claim = + CompletableFuture.runAsync( + () -> + inTransaction( + () -> { + var unused = leases().claim("worker-a", 10, Duration.ofMinutes(5)); + claimed.countDown(); + awaitOrFail(release); + return null; + }), + pool); + awaitOrFail(claimed); + ((ContractClock) clock()).moveTo(NOW.plusSeconds(11)); + CompletableFuture sweep = + CompletableFuture.supplyAsync( + () -> inTransaction(() -> leases().expireOverdue(10)), pool); + stack.awaitDoneOrBlocked(sweep, Duration.ofSeconds(30)); + release.countDown(); + claim.get(30, TimeUnit.SECONDS); + outcome = List.of(sweep.get(30, TimeUnit.SECONDS), stateOf(job.id())); + } finally { + pool.shutdownNow(); + } + + // then + assertThat(outcome).containsExactly(0, RecipientDeliveryState.DISPATCHING); + } + + private RecipientDeliveryState stateOf(RecipientDeliveryId id) { + return deliveries().find(id).orElseThrow().state(); + } + + private static void awaitOrFail(CountDownLatch latch) { + try { + if (!latch.await(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("the other transaction did not arrive within 30 seconds"); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while waiting", interrupted); + } + } + private static long versionOf(RecipientDeliveryId id) { return stack .jdbc() diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxBulkReadVersionContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxBulkReadVersionContractTest.java index 76863346..4851dadc 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxBulkReadVersionContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxBulkReadVersionContractTest.java @@ -5,9 +5,17 @@ import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.persistence.notification.NotificationJpaStack; import dev.caskeleton.adapter.outbound.persistence.notification.platform.entity.InboxItemEntity; +import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.Timestamp; +import java.time.Duration; import java.time.Instant; +import java.util.List; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; @@ -74,6 +82,49 @@ class InboxBulkReadVersionContractTest { .isEqualTo("READ"); } + @Test + void anItemArchivedWhileTheBulkReadWaitsForItStaysArchived() throws Exception { + // given + UUID id = insertUnreadItem(); + ExecutorService pool = Executors.newSingleThreadExecutor(); + + // when + List outcome; + try (Connection archiving = stack.jdbc().getDataSource().getConnection()) { + // The archive holds the row uncommitted while mark-all-read runs. The bulk read picks the id + // from the committed unread row, waits for the lock, and used to mark READ the item the + // user had just archived — putting it back in the inbox. + archiving.setAutoCommit(false); + try (PreparedStatement archive = + archiving.prepareStatement( + "UPDATE notification_inbox_item SET state = 'ARCHIVED', version = version + 1" + + " WHERE id = ?")) { + archive.setObject(1, id); + archive.executeUpdate(); + } + CompletableFuture bulkRead = + CompletableFuture.supplyAsync( + () -> + stack.inTransaction( + () -> items.markAllRead("tenant-a", "user-1", null, null, 100, NOW)), + pool); + stack.awaitDoneOrBlocked(bulkRead, Duration.ofSeconds(30)); + archiving.commit(); + outcome = List.of(bulkRead.get(30, TimeUnit.SECONDS), stateOf(id)); + } finally { + pool.shutdownNow(); + } + + // then + assertThat(outcome).containsExactly(0, "ARCHIVED"); + } + + private static String stateOf(UUID id) { + return stack + .jdbc() + .queryForObject("SELECT state FROM notification_inbox_item WHERE id = ?", String.class, id); + } + private static UUID insertUnreadItem() { UUID id = UUID.randomUUID(); stack diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSchemaConstraintTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSchemaConstraintTest.java new file mode 100644 index 00000000..501e79c7 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSchemaConstraintTest.java @@ -0,0 +1,214 @@ +package dev.caskeleton.adapter.outbound.persistence.operation; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +/** + * The {@code durable_operation} table's own constraints, against a real PostgreSQL. + * + *

What the store does is {@link JpaDurableOperationStoreContractTest}: the shipped adapter + * through the shared {@code DurableOperationStoreContract}. This class holds only what that + * contract cannot reach — the constraints a second writer meets when it bypasses the adapter. A + * {@code CHECK} constraint a fake does not enforce is a comment, so each one is shown refusing a + * row here, by writing the row directly. + * + *

It replaces a class that re-typed the adapter's SQL into the test and ran that copy (round9 + * review 2026-11-12, handed over from RR9 area 01): the copy passed whatever the shipped statements + * did, and seven of its cases repeated the contract. + */ +@Tag("jpa-contract") +class DurableOperationSchemaConstraintTest { + + private static final Instant NOW = Instant.parse("2026-08-25T09:00:00Z"); + + private static JpaPlatformContractSupport support; + + @BeforeAll + static void startServer() { + support = JpaPlatformContractSupport.start(); + } + + @AfterAll + static void stopServer() { + if (support != null) { + support.close(); + } + } + + @BeforeEach + void migrate() throws SQLException, IOException { + try (Connection connection = support.connection(); + Statement statement = connection.createStatement()) { + statement.execute("DROP SCHEMA public CASCADE"); + statement.execute("CREATE SCHEMA public"); + statement.execute(migration("V11__durable_operation.sql")); + statement.execute(migration("V23__durable_operation_lease_fence.sql")); + } + } + + @Test + void identicalScopeIsRefusedByTheUniqueConstraint() throws SQLException { + // given + insertPending("op-1", "hash-1"); + + // when + // The constraint is the arbiter the adapter's ON CONFLICT names, not a prior read: two + // identical submissions can arrive at the same instant, and a read-then-insert lets both in. + Throwable refused = catchThrowable(() -> insertPending("op-2", "hash-1")); + + // then + assertThat(refused) + .isInstanceOf(SQLException.class) + .hasMessageContaining("uq_durable_operation_scope"); + } + + @Test + void succeededRowWithoutAResultIsRefused() throws SQLException { + // given + insertPending("op-1", "hash-1"); + + // when + Throwable refused = + catchThrowable( + () -> + execute( + "UPDATE durable_operation SET state = 'SUCCEEDED', completed_at = now()" + + " WHERE operation_id = 'op-1'")); + + // then + assertThat(refused) + .isInstanceOf(SQLException.class) + .hasMessageContaining("ck_durable_operation_succeeded_has_result"); + } + + @Test + void failedRowWithoutAReasonIsRefused() throws SQLException { + // given + insertPending("op-1", "hash-1"); + + // when + Throwable refused = + catchThrowable( + () -> + execute( + "UPDATE durable_operation SET state = 'FAILED', completed_at = now()" + + " WHERE operation_id = 'op-1'")); + + // then + assertThat(refused) + .isInstanceOf(SQLException.class) + .hasMessageContaining("ck_durable_operation_failed_has_reason"); + } + + @Test + void runningRowWithoutALeaseIsRefused() throws SQLException { + // given + insertPending("op-1", "hash-1"); + + // when + Throwable refused = + catchThrowable( + () -> + execute( + "UPDATE durable_operation SET state = 'RUNNING' WHERE operation_id = 'op-1'")); + + // then + assertThat(refused) + .isInstanceOf(SQLException.class) + .hasMessageContaining("ck_durable_operation_running_has_lease"); + } + + @Test + void stateOutsideThePublishedVocabularyIsRefused() throws SQLException { + // given + insertPending("op-1", "hash-1"); + + // when + // A worker that could write its own state would extend the published contract by writing a + // row, and a client branching on status would meet a value that is in no version of the API. + Throwable refused = + catchThrowable( + () -> + execute( + "UPDATE durable_operation SET state = 'ALMOST_DONE'" + + " WHERE operation_id = 'op-1'")); + + // then + assertThat(refused) + .isInstanceOf(SQLException.class) + .hasMessageContaining("ck_durable_operation_state"); + } + + @Test + void expiryNotAfterSubmissionIsRefused() throws SQLException { + // given + insertPending("op-1", "hash-1"); + + // when + Throwable refused = + catchThrowable( + () -> + execute( + "UPDATE durable_operation SET expires_at = submitted_at" + + " WHERE operation_id = 'op-1'")); + + // then + assertThat(refused) + .isInstanceOf(SQLException.class) + .hasMessageContaining("ck_durable_operation_expiry_after_submission"); + } + + private void insertPending(String operationId, String hash) throws SQLException { + try (Connection connection = support.connection(); + PreparedStatement statement = + connection.prepareStatement( + """ + INSERT INTO durable_operation + (operation_id, tenant, principal, operation_name, request_hash, payload, + state, submitted_at, expires_at) + VALUES (?, '', 'alice', 'transfers.create', ?, '{}', 'PENDING', ?, ?) + """)) { + statement.setString(1, operationId); + statement.setString(2, hash); + statement.setTimestamp(3, Timestamp.from(NOW)); + statement.setTimestamp(4, Timestamp.from(NOW.plus(Duration.ofDays(1)))); + statement.executeUpdate(); + } + } + + private void execute(String sql) throws SQLException { + try (Connection connection = support.connection(); + Statement statement = connection.createStatement()) { + statement.execute(sql); + } + } + + private static String migration(String file) throws IOException { + try (InputStream resource = + DurableOperationSchemaConstraintTest.class + .getClassLoader() + .getResourceAsStream("db/migration/postgresql/" + file)) { + if (resource == null) { + throw new IOException("the durable operation migration " + file + " is missing"); + } + return new String(resource.readAllBytes(), StandardCharsets.UTF_8); + } + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationStoreContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationStoreContractTest.java deleted file mode 100644 index 0e3b7a3d..00000000 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationStoreContractTest.java +++ /dev/null @@ -1,474 +0,0 @@ -package dev.caskeleton.adapter.outbound.persistence.operation; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Timestamp; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -/** - * The durable operation store against a real PostgreSQL. - * - *

Every claim in this store is about concurrency or about a constraint, and neither survives - * being tested against a fake. {@code FOR UPDATE SKIP LOCKED} does not exist in H2's PostgreSQL - * mode with the semantics that matter, a {@code CHECK} constraint a fake does not enforce is a - * comment, and a lease race has no meaning where only one connection exists. - * - *

A dead worker is written the only way that is faithful here: the row a crash leaves behind — a - * lease nobody will renew — and then recovery runs against it. That is precisely what a killed - * process is from the database's side. - */ -@Tag("jpa-contract") -class DurableOperationStoreContractTest { - - private static final Instant NOW = Instant.parse("2026-08-25T09:00:00Z"); - private static final Duration LEASE = Duration.ofMinutes(2); - - private static JpaPlatformContractSupport support; - - @BeforeAll - static void startServer() { - support = JpaPlatformContractSupport.start(); - } - - @AfterAll - static void stopServer() { - if (support != null) { - support.close(); - } - } - - @BeforeEach - void migrate() throws SQLException, IOException { - try (Connection connection = support.connection(); - Statement statement = connection.createStatement()) { - statement.execute("DROP SCHEMA public CASCADE"); - statement.execute("CREATE SCHEMA public"); - statement.execute(migration()); - } - } - - @Test - @DisplayName("a lease left behind by a dead worker is reclaimed by another") - void expiredLeaseCanBeReclaimedAfterWorkerCrash() throws SQLException { - submit("op-1", "hash-1"); - claim("worker-a", NOW); - - // The worker is gone. Nothing renews the lease, and nothing else may touch the operation until - // it lapses — that exclusivity is the point of the lease. - assertThat(claim("worker-b", NOW.plusSeconds(30))).isEmpty(); - - assertThat(claim("worker-b", NOW.plus(LEASE).plusSeconds(1))).contains("op-1"); - assertThat(state("op-1")).isEqualTo("RUNNING"); - assertThat(leaseOwner("op-1")).isEqualTo("worker-b"); - } - - @Test - @DisplayName("a reclaimed operation keeps the moment it first started") - void reclaimKeepsTheOriginalStartTime() throws SQLException { - submit("op-1", "hash-1"); - claim("worker-a", NOW); - - claim("worker-b", NOW.plus(LEASE).plusSeconds(1)); - - // COALESCE, not an unconditional assignment. Overwriting it would make every retry look like a - // fresh start and hide how long the operation has really been running. - assertThat(startedAt("op-1")).isEqualTo(NOW); - } - - @Test - @DisplayName("two workers claiming at once take different operations") - void concurrentWorkersDoNotTakeTheSameOperation() throws Exception { - submit("op-1", "hash-1"); - submit("op-2", "hash-2"); - - List claimed = new ArrayList<>(); - try (Connection first = support.connection(); - Connection second = support.connection()) { - first.setAutoCommit(false); - second.setAutoCommit(false); - // Interleaved on purpose and inside open transactions: with plain FOR UPDATE the second - // claim blocks here until the first commits, and with no locking at all both take op-1. - claimed.add(claimOn(first, "worker-a", NOW).orElseThrow()); - claimed.add(claimOn(second, "worker-b", NOW).orElseThrow()); - first.commit(); - second.commit(); - } - - assertThat(claimed).containsExactlyInAnyOrder("op-1", "op-2"); - } - - @Test - @DisplayName("a worker whose lease lapsed cannot record a result over its successor") - void aStaleWorkerCannotOverwriteTheNewOwner() throws SQLException { - submit("op-1", "hash-1"); - claim("worker-a", NOW); - claim("worker-b", NOW.plus(LEASE).plusSeconds(1)); - - // worker-a is alive and still finishing. Letting it report here is how one operation ends up - // with two answers, and which one survives would come down to timing. - assertThat(succeed("op-1", "worker-a", "/results/a")).isZero(); - assertThat(succeed("op-1", "worker-b", "/results/b")).isOne(); - assertThat(resultReference("op-1")).isEqualTo("/results/b"); - } - - @Test - @DisplayName("a heartbeat from a worker that lost the lease fails rather than extending it") - void heartbeatFromAStaleWorkerFails() throws SQLException { - submit("op-1", "hash-1"); - claim("worker-a", NOW); - claim("worker-b", NOW.plus(LEASE).plusSeconds(1)); - - assertThat(heartbeat("op-1", "worker-a", NOW.plus(LEASE).plusSeconds(2))).isZero(); - assertThat(heartbeat("op-1", "worker-b", NOW.plus(LEASE).plusSeconds(2))).isOne(); - } - - @Test - @DisplayName("cancel works once and never reverses a finished operation") - void cancelIsIdempotentAndNeverReversesTerminal() throws SQLException { - submit("op-1", "hash-1"); - claim("worker-a", NOW); - succeed("op-1", "worker-a", "/results/a"); - - // The work happened. Recording it as CANCELED would replace the only record of a real outcome - // with a claim that it never occurred. - assertThat(cancel("op-1", "alice")).isZero(); - assertThat(state("op-1")).isEqualTo("SUCCEEDED"); - - submit("op-2", "hash-2"); - assertThat(cancel("op-2", "alice")).isOne(); - assertThat(cancel("op-2", "alice")).isZero(); - assertThat(state("op-2")).isEqualTo("CANCELED"); - } - - @Test - @DisplayName("another principal cannot cancel an operation they did not submit") - void cancelIsScopedToThePrincipal() throws SQLException { - submit("op-1", "hash-1"); - - assertThat(cancel("op-1", "mallory")).isZero(); - assertThat(state("op-1")).isEqualTo("PENDING"); - } - - @Test - @DisplayName("a cancelled operation is not handed to a worker") - void cancelledOperationsAreNotClaimable() throws SQLException { - submit("op-1", "hash-1"); - cancel("op-1", "alice"); - - assertThat(claim("worker-a", NOW)).isEmpty(); - } - - @Test - @DisplayName("an identical resubmission is refused by the unique constraint") - void resubmissionCollidesOnScope() throws SQLException { - submit("op-1", "hash-1"); - - // The constraint is the arbiter rather than a prior read: two identical submissions can arrive - // at the same instant, and a read-then-insert lets both through. - assertThatThrownBy(() -> submit("op-2", "hash-1")).isInstanceOf(SQLException.class); - } - - @Test - @DisplayName("a succeeded row with no result is refused by the database") - void succeededRowRequiresAResult() throws SQLException { - submit("op-1", "hash-1"); - - assertThatThrownBy( - () -> - execute( - "UPDATE durable_operation SET state = 'SUCCEEDED', completed_at = now()," - + " lease_owner = NULL WHERE operation_id = 'op-1'")) - .isInstanceOf(SQLException.class) - .hasMessageContaining("ck_durable_operation_succeeded_has_result"); - } - - @Test - @DisplayName("a running row with no lease is refused by the database") - void runningRowRequiresALease() { - assertThatThrownBy( - () -> { - submit("op-1", "hash-1"); - execute("UPDATE durable_operation SET state = 'RUNNING' WHERE operation_id = 'op-1'"); - }) - .isInstanceOf(SQLException.class) - .hasMessageContaining("ck_durable_operation_running_has_lease"); - } - - @Test - @DisplayName("a state outside the published vocabulary is refused") - void stateVocabularyIsClosed() { - // A worker that could write its own state would extend the published contract by writing a - // row, and a client branching on status would meet a value that is in no version of the API. - assertThatThrownBy( - () -> { - submit("op-1", "hash-1"); - execute( - "UPDATE durable_operation SET state = 'ALMOST_DONE' WHERE operation_id = 'op-1'"); - }) - .isInstanceOf(SQLException.class) - .hasMessageContaining("ck_durable_operation_state"); - } - - @Test - @DisplayName("an aged-out operation is expired, and a finished one is left alone") - void expirySweepLeavesRealOutcomesAlone() throws SQLException { - submit("op-1", "hash-1"); - submit("op-2", "hash-2"); - claim("worker-a", NOW); - - List expired = expireStale(NOW.plus(Duration.ofDays(2))); - - assertThat(expired).containsExactlyInAnyOrder("op-1", "op-2"); - submit("op-3", "hash-3"); - claim("worker-b", NOW); - succeed("op-3", "worker-b", "/results/c"); - assertThat(expireStale(NOW.plus(Duration.ofDays(2)))).isEmpty(); - assertThat(state("op-3")).isEqualTo("SUCCEEDED"); - } - - @Test - @DisplayName("reclaim returns lapsed leases to the queue") - void reclaimReturnsLapsedLeases() throws SQLException { - submit("op-1", "hash-1"); - claim("worker-a", NOW); - - assertThat(reclaim(NOW.plusSeconds(30))).isEmpty(); - assertThat(reclaim(NOW.plus(LEASE).plusSeconds(1))).containsExactly("op-1"); - assertThat(state("op-1")).isEqualTo("PENDING"); - assertThat(leaseOwner("op-1")).isNull(); - } - - private void submit(String operationId, String hash) throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = - connection.prepareStatement( - """ - INSERT INTO durable_operation - (operation_id, tenant, principal, operation_name, request_hash, payload, - state, submitted_at, expires_at) - VALUES (?, '', 'alice', 'transfers.create', ?, '{}', 'PENDING', ?, ?) - """)) { - statement.setString(1, operationId); - statement.setString(2, hash); - statement.setTimestamp(3, Timestamp.from(NOW)); - statement.setTimestamp(4, Timestamp.from(NOW.plus(Duration.ofDays(1)))); - statement.executeUpdate(); - } - } - - private Optional claim(String workerId, Instant now) throws SQLException { - try (Connection connection = support.connection()) { - return claimOn(connection, workerId, now); - } - } - - private Optional claimOn(Connection connection, String workerId, Instant now) - throws SQLException { - try (PreparedStatement statement = - connection.prepareStatement( - """ - UPDATE durable_operation - SET state = 'RUNNING', - lease_owner = ?, - lease_acquired_at = ?, - lease_expires_at = ?, - started_at = COALESCE(started_at, ?), - row_version = row_version + 1 - WHERE operation_id = ( - SELECT operation_id - FROM durable_operation - WHERE (state = 'PENDING' - OR (state = 'RUNNING' AND lease_expires_at <= ?)) - AND expires_at > ? - ORDER BY submitted_at - FOR UPDATE SKIP LOCKED - LIMIT 1) - RETURNING operation_id - """)) { - statement.setString(1, workerId); - statement.setTimestamp(2, Timestamp.from(now)); - statement.setTimestamp(3, Timestamp.from(now.plus(LEASE))); - statement.setTimestamp(4, Timestamp.from(now)); - statement.setTimestamp(5, Timestamp.from(now)); - statement.setTimestamp(6, Timestamp.from(now)); - try (ResultSet rows = statement.executeQuery()) { - return rows.next() ? Optional.of(rows.getString(1)) : Optional.empty(); - } - } - } - - private int succeed(String operationId, String workerId, String resultReference) - throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = - connection.prepareStatement( - """ - UPDATE durable_operation - SET state = 'SUCCEEDED', result_reference = ?, completed_at = ?, - lease_owner = NULL, lease_acquired_at = NULL, lease_expires_at = NULL, - row_version = row_version + 1 - WHERE operation_id = ? AND state = 'RUNNING' AND lease_owner = ? - """)) { - statement.setString(1, resultReference); - statement.setTimestamp(2, Timestamp.from(NOW.plusSeconds(10))); - statement.setString(3, operationId); - statement.setString(4, workerId); - return statement.executeUpdate(); - } - } - - private int heartbeat(String operationId, String workerId, Instant now) throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = - connection.prepareStatement( - """ - UPDATE durable_operation - SET lease_expires_at = ?, row_version = row_version + 1 - WHERE operation_id = ? AND state = 'RUNNING' AND lease_owner = ? - AND lease_expires_at > ? - """)) { - statement.setTimestamp(1, Timestamp.from(now.plus(LEASE))); - statement.setString(2, operationId); - statement.setString(3, workerId); - statement.setTimestamp(4, Timestamp.from(now)); - return statement.executeUpdate(); - } - } - - private int cancel(String operationId, String principal) throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = - connection.prepareStatement( - """ - UPDATE durable_operation - SET state = 'CANCELED', completed_at = ?, lease_owner = NULL, - lease_acquired_at = NULL, lease_expires_at = NULL, - row_version = row_version + 1 - WHERE operation_id = ? AND principal = ? AND state IN ('PENDING', 'RUNNING') - """)) { - statement.setTimestamp(1, Timestamp.from(NOW.plusSeconds(5))); - statement.setString(2, operationId); - statement.setString(3, principal); - return statement.executeUpdate(); - } - } - - private List reclaim(Instant now) throws SQLException { - return idsFrom( - """ - UPDATE durable_operation - SET state = 'PENDING', lease_owner = NULL, lease_acquired_at = NULL, - lease_expires_at = NULL, row_version = row_version + 1 - WHERE state = 'RUNNING' AND lease_expires_at <= ? - RETURNING operation_id - """, - now); - } - - private List expireStale(Instant now) throws SQLException { - return idsFrom( - """ - UPDATE durable_operation - SET state = 'EXPIRED', completed_at = ?, lease_owner = NULL, - lease_acquired_at = NULL, lease_expires_at = NULL, - row_version = row_version + 1 - WHERE state IN ('PENDING', 'RUNNING') AND expires_at <= ? - RETURNING operation_id - """, - now, - now); - } - - private List idsFrom(String sql, Instant... arguments) throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = connection.prepareStatement(sql)) { - for (int index = 0; index < arguments.length; index++) { - statement.setTimestamp(index + 1, Timestamp.from(arguments[index])); - } - List ids = new ArrayList<>(); - try (ResultSet rows = statement.executeQuery()) { - while (rows.next()) { - ids.add(rows.getString(1)); - } - } - return ids; - } - } - - private String state(String operationId) throws SQLException { - return column(operationId, "state"); - } - - private String leaseOwner(String operationId) throws SQLException { - return column(operationId, "lease_owner"); - } - - private String resultReference(String operationId) throws SQLException { - return column(operationId, "result_reference"); - } - - private Instant startedAt(String operationId) throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = - connection.prepareStatement( - "SELECT started_at FROM durable_operation WHERE operation_id = ?")) { - statement.setString(1, operationId); - try (ResultSet rows = statement.executeQuery()) { - rows.next(); - return rows.getTimestamp(1).toInstant(); - } - } - } - - private String column(String operationId, String name) throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = - connection.prepareStatement( - "SELECT " + name + " FROM durable_operation WHERE operation_id = ?")) { - statement.setString(1, operationId); - try (ResultSet rows = statement.executeQuery()) { - rows.next(); - return rows.getString(1); - } - } - } - - private void execute(String sql) throws SQLException { - try (Connection connection = support.connection(); - Statement statement = connection.createStatement()) { - statement.execute(sql); - } - } - - private static String migration() throws IOException { - try (InputStream resource = - DurableOperationStoreContractTest.class - .getClassLoader() - .getResourceAsStream("db/migration/postgresql/V11__durable_operation.sql")) { - if (resource == null) { - throw new IOException("the durable operation migration is missing from the classpath"); - } - return new String(resource.readAllBytes(), StandardCharsets.UTF_8); - } - } -} diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/JpaDurableOperationStoreContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/JpaDurableOperationStoreContractTest.java index 9cd7d16a..22fa4f0f 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/JpaDurableOperationStoreContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/JpaDurableOperationStoreContractTest.java @@ -1,9 +1,15 @@ package dev.caskeleton.adapter.outbound.persistence.operation; +import static org.assertj.core.api.Assertions.assertThat; + import dev.caskeleton.adapter.outbound.persistence.operation.entity.DurableOperationEntity; import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; +import dev.caskeleton.application.idempotency.RequestFingerprint; +import dev.caskeleton.application.operation.DurableOperationClaim; +import dev.caskeleton.application.operation.DurableOperationId; import dev.caskeleton.application.operation.DurableOperationStorePort; +import dev.caskeleton.application.operation.DurableOperationSubmission; import dev.caskeleton.application.operation.testkit.DurableOperationStoreContract; import dev.caskeleton.application.transaction.TransactionPort; import jakarta.persistence.EntityManagerFactory; @@ -13,13 +19,22 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.time.Duration; +import java.time.Instant; +import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import javax.sql.DataSource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.springframework.context.annotation.AnnotationConfigApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -32,9 +47,10 @@ import org.springframework.transaction.PlatformTransactionManager; /** * The shipped adapter, through Spring Data and a Spring transaction, against real PostgreSQL. * - *

The raw-SQL {@link DurableOperationStoreContractTest} proves the schema; this proves the code - * that talks to it. A resubmission only breaks where PostgreSQL aborts the transaction after a - * constraint violation, so no fake can stand in here. + *

{@link DurableOperationSchemaConstraintTest} proves the table's constraints; this proves the + * code that talks to it. A resubmission only breaks where PostgreSQL aborts the transaction after a + * constraint violation, and two claims only race where two transactions hold row locks at once, so + * no fake can stand in here. */ @Tag("jpa-contract") class JpaDurableOperationStoreContractTest extends DurableOperationStoreContract { @@ -45,7 +61,8 @@ class JpaDurableOperationStoreContractTest extends DurableOperationStoreContract @BeforeAll static void start() throws SQLException, IOException { support = JpaPlatformContractSupport.start(); - execute(migration()); + execute(migration("V11__durable_operation.sql")); + execute(migration("V23__durable_operation_lease_fence.sql")); context = new AnnotationConfigApplicationContext(); context.getBeanFactory().registerSingleton("dataSource", support.dataSource()); context.register(AdapterConfiguration.class); @@ -67,6 +84,74 @@ class JpaDurableOperationStoreContractTest extends DurableOperationStoreContract execute("TRUNCATE durable_operation"); } + @Test + void twoWorkersClaimingAtOnceTakeDifferentOperations() throws Exception { + // given + Instant now = Instant.parse("2026-08-25T09:00:00Z"); + inTransaction(() -> store().submit(submission("op-1", "a", now))); + inTransaction(() -> store().submit(submission("op-2", "b", now.plusSeconds(1)))); + CountDownLatch firstClaimed = new CountDownLatch(1); + CountDownLatch secondDone = new CountDownLatch(1); + ExecutorService pool = Executors.newSingleThreadExecutor(); + + // when + List claimed; + try { + // The first claim holds its row lock until the second has claimed. With plain FOR UPDATE the + // second would block behind it; with no lock at all both would take op-1. + CompletableFuture first = + CompletableFuture.supplyAsync( + () -> + inTransaction( + () -> { + DurableOperationClaim claim = claimNow("worker-a", now); + firstClaimed.countDown(); + awaitOrFail(secondDone); + return claim; + }), + pool); + awaitOrFail(firstClaimed); + DurableOperationClaim second = inTransaction(() -> claimNow("worker-b", now)); + secondDone.countDown(); + claimed = + List.of( + first.get(30, TimeUnit.SECONDS).operationId().value(), second.operationId().value()); + } finally { + pool.shutdownNow(); + } + + // then + assertThat(claimed).containsExactlyInAnyOrder("op-1", "op-2"); + } + + private DurableOperationClaim claimNow(String workerId, Instant now) { + return store().claimNext(workerId, Duration.ofMinutes(2), now).orElseThrow(); + } + + private static void awaitOrFail(CountDownLatch latch) { + try { + if (!latch.await(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("the other claim did not arrive within 30 seconds"); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while waiting for the other claim", interrupted); + } + } + + private static DurableOperationSubmission submission( + String operationId, String hashLetter, Instant submittedAt) { + return new DurableOperationSubmission( + new DurableOperationId(operationId), + "reports.generate", + "alice", + null, + new RequestFingerprint(hashLetter.repeat(64)), + "{}", + submittedAt, + Duration.ofDays(1)); + } + @Override protected DurableOperationStorePort store() { return context.getBean(JpaDurableOperationStore.class); @@ -84,13 +169,13 @@ class JpaDurableOperationStoreContractTest extends DurableOperationStoreContract } } - private static String migration() throws IOException { + private static String migration(String file) throws IOException { try (InputStream resource = JpaDurableOperationStoreContractTest.class .getClassLoader() - .getResourceAsStream("db/migration/postgresql/V11__durable_operation.sql")) { + .getResourceAsStream("db/migration/postgresql/" + file)) { if (resource == null) { - throw new IOException("the durable operation migration is missing from the classpath"); + throw new IOException("the durable operation migration " + file + " is missing"); } return new String(resource.readAllBytes(), StandardCharsets.UTF_8); } diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/FileserverPostgreSqlStack.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/FileserverPostgreSqlStack.java index a6e77b62..068f1dc0 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/FileserverPostgreSqlStack.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/FileserverPostgreSqlStack.java @@ -96,6 +96,36 @@ final class FileserverPostgreSqlStack implements AutoCloseable { return fileId; } + /** + * Waits until some session of this database is blocked on a lock, so a case can hold one + * transaction open until a second one is queued behind it without sleeping a guessed interval. + */ + void awaitASessionWaitingOnALock(Duration deadline) { + Instant giveUp = Instant.now().plus(deadline); + while (Instant.now().isBefore(giveUp)) { + Integer waiting = + jdbc.queryForObject( + "select count(*) from pg_stat_activity" + + " where datname = current_database() and wait_event_type = 'Lock'", + Integer.class); + if (waiting != null && waiting > 0) { + return; + } + Thread.onSpinWait(); + } + throw new IllegalStateException("no session waited on a lock within " + deadline); + } + + /** Counts the rows of {@code fs_recovery_item} for one file, open and resolved. */ + int recoveryItemsOf(FileId fileId) { + Integer rows = + jdbc.queryForObject( + "select count(*) from fs_recovery_item where file_id = ?", + Integer.class, + fileId.value()); + return rows == null ? -1 : rows; + } + /** Deletes every Fileserver row, children first. */ void empty() { jdbc.execute( diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java index bf5117a8..0439e9d5 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java @@ -50,7 +50,7 @@ class PostgreSqlMigrationIntegrationTest { assertThat(appliedVersions(postgres, "flyway_schema_history")) .containsExactly( "1", "3", "4", "5", "6", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", - "19", "20", "21", "22"); + "19", "20", "21", "22", "23"); assertThat( singleValue( postgres, diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlNotificationBackgroundPassIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlNotificationBackgroundPassIntegrationTest.java index 74d09587..09606e87 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlNotificationBackgroundPassIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlNotificationBackgroundPassIntegrationTest.java @@ -70,6 +70,7 @@ import org.flywaydb.core.Flyway; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.springframework.context.annotation.AnnotationConfigApplicationContext; @@ -189,7 +190,25 @@ class PostgreSqlNotificationBackgroundPassIntegrationTest { } @Test - @org.junit.jupiter.api.DisplayName("요청 저장은 collapse 와 dedup DROP 판정을 함께 남겨 같은 키 재조회가 같은 답을 준다") + @DisplayName("fence 없이 DISPATCHING으로 남은 옛 행이 있어도 복구 패스는 다른 행을 되돌린다") + void aDispatchingRowNoClaimWroteDoesNotStopTheRecoveryPass() { + // given + UUID legacy = insertDelivery("DISPATCHING", Instant.now().minus(Duration.ofMinutes(20)), null); + jdbc.update("update notification_recipient_delivery set lease_fence = 0 where id = ?", legacy); + UUID abandoned = + insertDelivery("DISPATCHING", Instant.now().minus(Duration.ofMinutes(10)), null); + + // when + recovery().recoverOnce(); + + // then + assertThat(List.of(deliveryState(legacy), deliveryState(abandoned))) + .as("a row before notification V4's fence is no lease the requeue can match on") + .containsExactly("DISPATCHING", "READY_TO_DISPATCH"); + } + + @Test + @DisplayName("요청 저장은 collapse 와 dedup DROP 판정을 함께 남겨 같은 키 재조회가 같은 답을 준다") void aStoredRequestKeepsItsCollapseAndItsDropVerdict() { // given JpaNotificationRequestStore requests = @@ -414,13 +433,15 @@ class PostgreSqlNotificationBackgroundPassIntegrationTest { "insert into notification_recipient_delivery (id, notification_id, tenant_id," + " recipient_ref, routing_plan_json, route_cursor, delivery_state, submission_outcome," + " delivery_outcome, evidence_level, attempt_count, next_dispatch_at, lease_owner," - + " lease_until, expires_at, created_at, updated_at)" + + " lease_fence, lease_until, expires_at, created_at, updated_at)" + " values (?, ?, 'tenant-a', 'user-1', '[]'::jsonb, 0, ?, 'NOT_SUBMITTED', 'UNKNOWN'," - + " 'PLATFORM_QUEUED', 0, now(), ?, ?, ?, now(), now())", + + " 'PLATFORM_QUEUED', 0, now(), ?, ?, ?, ?, now(), now())", delivery, request, state, leaseUntil == null ? null : "dead-worker", + // A claim writes the owner and raises the fence together, so a leased row has both. + leaseUntil == null ? 0L : 1L, leaseUntil == null ? null : Timestamp.from(leaseUntil), expiresAt == null ? null : Timestamp.from(expiresAt)); return delivery; diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlRecoveryQueueContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlRecoveryQueueContractTest.java index 9ba78957..a4c34b3c 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlRecoveryQueueContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlRecoveryQueueContractTest.java @@ -1,16 +1,25 @@ package dev.caskeleton.adapter.outbound.persistence.readiness; +import static org.assertj.core.api.Assertions.assertThat; + import dev.caskeleton.adapter.outbound.persistence.fileserver.JpaRecoveryQueue; import dev.caskeleton.application.fileserver.api.FileId; import dev.caskeleton.application.fileserver.api.StorageNamespace; import dev.caskeleton.application.fileserver.recovery.RecoveryQueuePort; import dev.caskeleton.application.fileserver.testkit.RecoveryQueuePortContract; import java.time.Duration; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * {@link JpaRecoveryQueue} held to the contract the in-memory queue passes, on real PostgreSQL @@ -38,6 +47,66 @@ class PostgreSqlRecoveryQueueContractTest extends RecoveryQueuePortContract { stack.empty(); } + @Test + void theSameFileEnqueuedInTwoConcurrentTransactionsLeavesOneOpenItem() throws Exception { + // given + FileId file = recordedFile(); + CountDownLatch firstEnqueued = new CountDownLatch(1); + CountDownLatch secondQueued = new CountDownLatch(1); + ExecutorService pool = Executors.newFixedThreadPool(2); + + // when + List outcome; + try { + // The first transaction holds its new row uncommitted until the second has queued behind the + // open-item index. Update-then-insert made the second insert too and fail on that index. + CompletableFuture first = + CompletableFuture.runAsync( + () -> + inTransaction( + () -> { + queue().enqueue(file, "FIRST_REPORT"); + firstEnqueued.countDown(); + awaitOrFail(secondQueued); + return null; + }), + pool); + awaitOrFail(firstEnqueued); + CompletableFuture second = + CompletableFuture.runAsync( + () -> + inTransaction( + () -> { + queue().enqueue(file, "SECOND_REPORT"); + return null; + }), + pool); + stack.awaitASessionWaitingOnALock(Duration.ofSeconds(30)); + secondQueued.countDown(); + first.get(30, TimeUnit.SECONDS); + second.get(30, TimeUnit.SECONDS); + outcome = + List.of(stack.recoveryItemsOf(file), inTransaction(() -> queue().pending(10)).size()); + } finally { + pool.shutdownNow(); + } + + // then + assertThat(outcome).containsExactly(1, 1); + } + + private static void awaitOrFail(CountDownLatch latch) { + try { + if (!latch.await(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("the other enqueue did not arrive within 30 seconds"); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "interrupted while waiting for the other enqueue", interrupted); + } + } + @Override protected RecoveryQueuePort queue() { return stack.bean(JpaRecoveryQueue.class); diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java index 72235d67..e80b313d 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java @@ -2,23 +2,61 @@ package dev.caskeleton.adapter.outbound.persistence.idempotency.jpa; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.time.Clock; -import java.time.Instant; -import java.time.ZoneOffset; import org.junit.jupiter.api.Test; +/** + * Unit tests for {@link IdempotencyReaper}: the delete runs in bounded batches until one comes back + * short, as the outbox reaper's does. + * + *

The repository is a Spring Data boundary, so Mockito stands in for it. The statement itself — + * expiry judged by the database clock — runs against PostgreSQL in {@code + * IdempotencyReaperIntegrationTest}. + */ class IdempotencyReaperTest { - private static final Instant NOW = Instant.parse("2026-06-09T12:00:00Z"); + private static final int BATCH = IdempotencyReaper.BATCH_SIZE; + + private final IdempotencyRecordJpaRepository repository = + mock(IdempotencyRecordJpaRepository.class); + private final IdempotencyReaper reaper = new IdempotencyReaper(repository); @Test - void reapDeletesRecordsExpiredAsOfNow() { - IdempotencyRecordJpaRepository repo = mock(IdempotencyRecordJpaRepository.class); - when(repo.deleteExpired(NOW)).thenReturn(3); - IdempotencyReaper reaper = new IdempotencyReaper(repo, Clock.fixed(NOW, ZoneOffset.UTC)); + void reapKeepsDeletingBatchesUntilOneComesBackShortAndReturnsTheTotal() { + // given + when(repository.deleteExpiredBatch(BATCH)).thenReturn(BATCH, BATCH, 3); - assertThat(reaper.reap()).isEqualTo(3); + // when + int purged = reaper.reap(); + + // then + assertThat(purged).isEqualTo(2 * BATCH + 3); + } + + @Test + void eachBatchIsItsOwnDeleteStatement() { + // given + when(repository.deleteExpiredBatch(BATCH)).thenReturn(BATCH, BATCH, 3); + + // when + var unused = reaper.reap(); + + // then + verify(repository, times(3)).deleteExpiredBatch(BATCH); + } + + @Test + void reapReturnsZeroWhenNothingHasExpired() { + // given + when(repository.deleteExpiredBatch(BATCH)).thenReturn(0); + + // when + int purged = reaper.reap(); + + // then + assertThat(purged).isZero(); } } diff --git a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 index 1eb57d14..32edf890 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 +++ b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 @@ -42,6 +42,7 @@ a0306e5b0fbcaf43984d256362dc4244aec57a0ce5031e4f5263be3762fe1caf postgresql/V19 25db296e55444636c8a017d9b736eecf4a839a9caf1733c0c4a5f37bae5a3161 postgresql/V20__outbox_event_deferred_attempt_count.sql 21744ea1aab2b29fbb3417b9a5c4389522199413048cc7035aa649ffcdd852cf postgresql/V21__outbox_aggregate_sequence.sql 20da5649a29bf2305f0e04ab14003b21b7f120d867439fb01f1658c98179dba0 postgresql/V22__outbox_event_last_failure_code.sql +9492ace50ab828144792b77c72d92fce50bec29a028ebc917f7e55facf58d85a postgresql/V23__durable_operation_lease_fence.sql 8ab106731d4d04d3719da0692069947736dc490ca46aa5a3ea9bdf65a5d956dc postgresql/V1__idempotency_record.sql 6b62a01a4eeb792b81015b4810ecc6f3f021e0a17dea50c871493f7694d4f0b9 postgresql/V3__outbox_event.sql c57ab8a35a10079190ad8a563be58fae4147c9406c5bb752debb6998324ccdf6 postgresql/V4__int_lock.sql diff --git a/src/adapter/outbound/persistence-notification/AGENTS.md b/src/adapter/outbound/persistence-notification/AGENTS.md index 94fa03ad..cb9ae8b1 100644 --- a/src/adapter/outbound/persistence-notification/AGENTS.md +++ b/src/adapter/outbound/persistence-notification/AGENTS.md @@ -8,7 +8,8 @@ - 패키지 루트: `dev.caskeleton.adapter.outbound.persistence.notification`. 패키지 규칙은 `persistence-fileserver`와 같아서 두 capability leaf를 같은 방식으로 읽는다: - `platform.entity` — JPA 행만. - - `platform.repository` — Spring Data 인터페이스와 그것이 돌리는 SQL 글(`RecipientClaimSql`). + - `platform.repository` — Spring Data 인터페이스와 그것이 돌리는 SQL 글(`RecipientClaimSql`). `platform`의 store도 + 이 SQL 조각을 쓴다. - `platform` — store 어댑터(`Jpa*`, `Jdbc*`, inbox의 `JpaNotificationInbox`와 그 outbox 도우미), `NotificationRecordMapper`, payload codec. - `configuration` — store를 bean으로 등록하는 자동 구성 창구. leaf 루트에는 scan 설정과 schema stream이 있다. diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java index 23fdb2dc..a2da23c2 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java @@ -99,6 +99,23 @@ public final class JpaRecipientDeliveryStore implements RecipientDeliveryStorePo return reread(written, record.id()); } + @Override + public Optional requeueAbandoned( + RecipientLease abandoned, Instant nextDispatchAt) { + CallerTransaction.require("recipient delivery requeueAbandoned"); + Objects.requireNonNull(abandoned, "abandoned"); + Objects.requireNonNull(nextDispatchAt, "nextDispatchAt"); + int written = + recipients.requeueAbandoned( + abandoned.recipientDeliveryId().value(), + abandoned.owner(), + abandoned.fence(), + abandoned.leaseUntil(), + nextDispatchAt, + clock.instant()); + return reread(written, abandoned.recipientDeliveryId()); + } + @Override public Optional transitionHeldBy( RecipientDeliveryId id, diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java index 65784bac..98cd4bef 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java @@ -5,9 +5,11 @@ import dev.caskeleton.adapter.outbound.persistence.transaction.CallerTransaction import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; import dev.caskeleton.application.notification.platform.dispatch.RecipientLease; import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort; +import java.sql.Timestamp; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.time.OffsetDateTime; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -109,13 +111,38 @@ public final class JpaRecipientLeaseStore implements RecipientLeaseStorePort { } @Override - public List expiredDispatching(int limit, Duration olderThan) { + public List expiredDispatching(int limit, Duration olderThan) { Objects.requireNonNull(olderThan, "olderThan"); if (limit < 1) { throw new IllegalArgumentException("limit"); } return recipients.selectExpiredDispatching(clock.instant().minus(olderThan), limit).stream() - .map(RecipientDeliveryId::new) + .map( + row -> + new RecipientLease( + new RecipientDeliveryId((UUID) row[0]), + (String) row[1], + ((Number) row[2]).longValue(), + instantOf(row[3]))) .toList(); } + + /** + * A {@code timestamptz} column of a native row. Hibernate hands it back as an {@link + * OffsetDateTime} or a {@link Timestamp} depending on its JDBC type mapping, never null here + * because the select only returns rows that have a lease. + */ + private static Instant instantOf(Object column) { + if (column instanceof Instant instant) { + return instant; + } + if (column instanceof OffsetDateTime offset) { + return offset.toInstant(); + } + if (column instanceof Timestamp timestamp) { + return timestamp.toInstant(); + } + throw new IllegalStateException( + "lease_until came back as " + (column == null ? "null" : column.getClass().getName())); + } } diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java index 15428027..e7ccfb32 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java @@ -60,18 +60,16 @@ public interface InboxItemJpaRepository extends Repository{@code read_at IS NULL} in the predicate is what makes the update idempotent under * concurrency: a second caller updates zero rows instead of overwriting the first caller's * timestamp. - */ - /** - * Marks everything up to and including a cursor position as read. * *

The cutoff is a tuple, matching the list's own {@code (created_at DESC, id DESC)} ordering. * It used to be a timestamp alone, and the list's order is not a total order on {@code @@ -95,14 +93,23 @@ public interface InboxItemJpaRepository extends RepositoryThe lease store contract ({@code RecipientLeaseStoreContract}, run on PostgreSQL by {@code * PostgreSqlRecipientLeaseStoreContractTest}) now drives this statement through the repository and * the store, so no test retypes it or runs it beside the adapter (review 2026-11-01 04-14). The - * named parameters are Spring Data's. + * named parameters are bound by Spring Data for the repository statements, and by a {@code + * NamedParameterJdbcTemplate} where the serving state reads {@link #CLAIMABLE_AT_NOW}. */ public final class RecipientClaimSql { + /** + * The states a delivery waits in to be sent, which the claim takes from, the expiry retires from + * and the serving state counts as backlog. One list so the statements cannot disagree about what + * is waiting. + */ + public static final String CLAIMABLE_STATES = "('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')"; + + /** + * Which deliveries the claim takes at {@code :now}: due, in a waiting state, not expired, and + * with no lease or one that has run out. The serving state counts its backlog with this same + * fragment, so the backlog is exactly what a claim would take. + */ + public static final String CLAIMABLE_AT_NOW = + "next_dispatch_at <= :now" + + " AND delivery_state IN " + + CLAIMABLE_STATES + + " AND (expires_at IS NULL OR expires_at > :now)" + + " AND (lease_until IS NULL OR lease_until < :now)"; + /** * Claims due deliveries and returns their fences. * @@ -28,10 +48,8 @@ public final class RecipientClaimSql { public static final String CLAIM_BATCH = "WITH claimable AS (" + " SELECT id FROM notification_recipient_delivery" - + " WHERE next_dispatch_at <= :now" - + " AND delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')" - + " AND (expires_at IS NULL OR expires_at > :now)" - + " AND (lease_until IS NULL OR lease_until < :now)" + + " WHERE " + + CLAIMABLE_AT_NOW + " ORDER BY next_dispatch_at, id" + " FOR UPDATE SKIP LOCKED" + " LIMIT :batchSize" @@ -51,7 +69,18 @@ public final class RecipientClaimSql { + " WHERE d.id = claimable.id" + " RETURNING d.id, d.lease_fence"; - /** Retires deliveries whose window closed before anyone claimed them. */ + /** + * Retires deliveries whose window closed before anyone claimed them. + * + *

A delivery a worker claimed while this statement ran must not be retired to {@code EXPIRED} + * under the worker dispatching it, which is what the unlocked subquery did (round9 review + * 2026-11-12, RR9-01-13). Two things stop it. The subquery locks the rows it picks and skips rows + * another transaction holds, so a delivery being claimed is not picked. The state and expiry + * conditions are also on the {@code UPDATE} itself: under READ COMMITTED PostgreSQL re-checks + * only the outer {@code WHERE} against a row it had to wait for, so if the lock clause is ever + * lost the claimed row is still left alone. A delivery being claimed right now is not expiring + * either: the claim refuses expired rows in the same statement. + */ public static final String EXPIRE_OVERDUE = "UPDATE notification_recipient_delivery SET" + " delivery_state = 'EXPIRED'," @@ -61,11 +90,17 @@ public final class RecipientClaimSql { + " updated_at = :now" + " WHERE id IN (" + " SELECT id FROM notification_recipient_delivery" - + " WHERE delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')" + + " WHERE delivery_state IN " + + CLAIMABLE_STATES + " AND expires_at IS NOT NULL" + " AND expires_at <= :now" + " ORDER BY expires_at" - + " LIMIT :batchSize)"; + + " FOR UPDATE SKIP LOCKED" + + " LIMIT :batchSize)" + + " AND delivery_state IN " + + CLAIMABLE_STATES + + " AND expires_at IS NOT NULL" + + " AND expires_at <= :now"; private RecipientClaimSql() {} } diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java index b5f1ee18..1240cf0f 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java @@ -183,13 +183,48 @@ public interface RecipientDeliveryJpaRepository extends RepositoryOnly rows a claim wrote: an owner and a fence of at least one. The requeue matches on both, + * and a row without them — one left dispatching before notification V4 added the fence — is no + * lease anyone holds. Read here it would be a lease the application refuses to build, and the one + * row would fail every recovery pass after it. + * + * @return one row per delivery: id, lease_owner, lease_fence, lease_until + */ @Query( value = - "SELECT id FROM notification_recipient_delivery " + "SELECT id, lease_owner, lease_fence, lease_until FROM notification_recipient_delivery " + "WHERE delivery_state = 'DISPATCHING' AND lease_until < :threshold " + + "AND lease_owner IS NOT NULL AND lease_fence > 0 " + "ORDER BY lease_until LIMIT :batchSize", nativeQuery = true) - List selectExpiredDispatching( + List selectExpiredDispatching( @Param("threshold") Instant threshold, @Param("batchSize") int batchSize); + + /** + * Returns an abandoned dispatch to the queue while it is still the one recovery found. + * + *

Owner, fence, state and expiry are all conditions. A new claim raises the fence, a finished + * holder moves the state, another instance's recovery moves the state, and a holder that renewed + * in time moves the expiry past what recovery saw; in each case the job is no longer abandoned + * and this matches nothing. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query( + value = + "UPDATE notification_recipient_delivery " + + "SET delivery_state = 'READY_TO_DISPATCH', next_dispatch_at = :nextDispatchAt, " + + "version = version + 1, updated_at = :now " + + "WHERE id = :id AND lease_owner = :owner AND lease_fence = :fence " + + "AND delivery_state = 'DISPATCHING' AND lease_until <= :leaseUntil", + nativeQuery = true) + int requeueAbandoned( + @Param("id") UUID id, + @Param("owner") String owner, + @Param("fence") long fence, + @Param("leaseUntil") Instant leaseUntil, + @Param("nextDispatchAt") Instant nextDispatchAt, + @Param("now") Instant now); } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java new file mode 100644 index 00000000..80bdbe0c --- /dev/null +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java @@ -0,0 +1,45 @@ +package dev.caskeleton.application.operation; + +import java.util.Objects; + +/** + * One worker's claim on one durable operation, as returned by {@link + * DurableOperationStorePort#claimNext}. + * + *

The {@code fence} is what tells one run from the next. Every claim increments it, and a + * heartbeat, progress report or completion names the fence it holds, so a run that stalled past its + * lease finds nothing left to change instead of overwriting the run that took over. The worker id + * alone cannot do this: a worker that restarts with the same id, or one process running two + * executions under one id, would otherwise have its earlier run accepted as the current one. + * + *

Kept apart from {@link DurableOperationSubmission} on purpose, as the outbox keeps its lease + * apart from the claimed event: the work to execute is the submission, and the claim is store + * bookkeeping the work has no use for. + * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * + * @param submission the claimed work + * @param workerId the worker that holds the claim + * @param fence the fencing token this claim was granted; at least 1 + */ +public record DurableOperationClaim( + DurableOperationSubmission submission, String workerId, long fence) { + + public DurableOperationClaim { + Objects.requireNonNull(submission, "submission"); + if (workerId == null || workerId.isBlank()) { + throw new IllegalStateException( + "a claim needs an owner; an anonymous one cannot be verified when the work completes"); + } + if (fence < 1) { + throw new IllegalStateException("fence must be >= 1, was " + fence); + } + } + + /** The claimed operation. */ + public DurableOperationId operationId() { + return submission.operationId(); + } +} diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java index 440d3fb2..66c718b2 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java @@ -11,9 +11,12 @@ import java.util.Optional; *

A port because the application owns what an operation is and nothing about how it is stored. * The JPA adapter behind it is one implementation; an in-memory one serves tests. * - *

Every method that changes a running operation takes the {@code workerId} and verifies it. A - * worker whose lease lapsed may still be alive and still finishing its work, and letting it report - * a result after another worker took over is how an operation ends up with two answers. + *

Every method that changes a running operation takes the {@link DurableOperationClaim} that + * {@link #claimNext} returned and verifies both its worker and its fence. A worker whose lease + * lapsed may still be alive and still finishing its work, and letting it report a result after + * another run took over is how an operation ends up with two answers. The worker id alone does not + * tell the runs apart: the same id may claim the operation again after a restart, so the fence — + * incremented by every claim — is what the store compares. */ public interface DurableOperationStorePort { @@ -43,48 +46,43 @@ public interface DurableOperationStorePort { * *

Runnable means PENDING, or RUNNING with a lapsed lease. The second case is the recovery * path: without it, an operation claimed by a worker that then died is never picked up again. + * Every claim gets a fence higher than any earlier claim of the same operation, whoever made it. * * @param workerId who is claiming * @param leaseDuration how long the claim holds without a heartbeat * @param now the current instant + * @return the claim the worker names on every later write, or empty when nothing is runnable */ - Optional claimNext( - String workerId, Duration leaseDuration, Instant now); + Optional claimNext(String workerId, Duration leaseDuration, Instant now); /** - * Extends a lease the worker still holds. + * Extends a lease the claim still holds. * - * @return false when the lease is no longer this worker's, which means it must stop + * @return false when the lease lapsed or another claim (by any worker, this one included) took + * the operation over, which means this run must stop */ - boolean heartbeat( - DurableOperationId operationId, String workerId, Duration leaseDuration, Instant now); + boolean heartbeat(DurableOperationClaim claim, Duration leaseDuration, Instant now); /** - * Publishes progress for an operation this worker holds. + * Publishes progress for an operation the claim holds. * - * @return false when the lease is no longer this worker's + * @return false when another claim took the operation over or it is no longer running */ - boolean reportProgress( - DurableOperationId operationId, String workerId, OperationProgressSnapshot progress); + boolean reportProgress(DurableOperationClaim claim, OperationProgressSnapshot progress); /** * Records success. * - * @return false when the lease is no longer this worker's, so the result is not recorded + * @return false when another claim took the operation over, so the result is not recorded */ - boolean succeed( - DurableOperationId operationId, String workerId, String resultReference, Instant completedAt); + boolean succeed(DurableOperationClaim claim, String resultReference, Instant completedAt); /** * Records failure. * - * @return false when the lease is no longer this worker's + * @return false when another claim took the operation over, so the failure is not recorded */ - boolean fail( - DurableOperationId operationId, - String workerId, - OperationFailure failure, - Instant completedAt); + boolean fail(DurableOperationClaim claim, OperationFailure failure, Instant completedAt); /** * Requests cancellation. diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java index 86328d84..618f23c1 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java @@ -11,6 +11,10 @@ import java.util.Objects; * ever with no worker behind it — the failure that makes a queue stop draining while every * dashboard reports it busy. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param workerId who holds it * @param acquiredAt when it was taken * @param expiresAt when another worker may take it @@ -22,11 +26,11 @@ public record OperationLease(String workerId, Instant acquiredAt, Instant expire Objects.requireNonNull(acquiredAt, "acquiredAt"); Objects.requireNonNull(expiresAt, "expiresAt"); if (workerId.isBlank()) { - throw new IllegalArgumentException( + throw new IllegalStateException( "a lease needs an owner; an anonymous one cannot be verified when the work completes"); } if (!expiresAt.isAfter(acquiredAt)) { - throw new IllegalArgumentException("a lease that expires when taken is not a lease"); + throw new IllegalStateException("a lease that expires when taken is not a lease"); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java b/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java index bafa1b5e..05c0062e 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java @@ -12,6 +12,10 @@ import java.util.Objects; * publishes the bytes that were validated rather than a re-rendering of them. The accessors below * name the few identities the relay and its diagnostics read without reaching into the envelope. * + *

A store builds it from a row, never from a request, so a count it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param event the appended event * @param status the row's lifecycle status; {@link OutboxEventStatus#IN_FLIGHT} right after a claim * @param attemptCount claims made, counting the claim that returned this row; the retry delay grows @@ -30,10 +34,10 @@ public record ClaimedOutboxEvent( Objects.requireNonNull(event, "event must not be null"); Objects.requireNonNull(status, "status must not be null"); if (attemptCount < 0) { - throw new IllegalArgumentException("attemptCount must be >= 0, was " + attemptCount); + throw new IllegalStateException("attemptCount must be >= 0, was " + attemptCount); } if (deferredAttemptCount < 0 || deferredAttemptCount > attemptCount) { - throw new IllegalArgumentException( + throw new IllegalStateException( "deferredAttemptCount must be between 0 and attemptCount (" + attemptCount + "), was " diff --git a/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java b/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java index eed397ab..ae783bc1 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java @@ -13,6 +13,10 @@ import java.util.Objects; *

Kept apart from {@link ClaimedOutboxEvent} on purpose: the broker adapter receives the event, * and the claim is store bookkeeping it has no use for. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param event the claimed row * @param owner the relay instance that holds the claim * @param token the fencing token this claim was granted @@ -22,10 +26,10 @@ public record OutboxLease(ClaimedOutboxEvent event, String owner, long token) { public OutboxLease { Objects.requireNonNull(event, "event must not be null"); if (owner == null || owner.isBlank()) { - throw new IllegalArgumentException("owner must not be blank"); + throw new IllegalStateException("owner must not be blank"); } if (token < 1) { - throw new IllegalArgumentException("token must be >= 1, was " + token); + throw new IllegalStateException("token must be >= 1, was " + token); } } diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java index 628ee3ef..8c845cac 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java @@ -39,8 +39,9 @@ class CancelDurableOperationUseCaseTest { void cancelOfAFinishedOperationReportsNoChange() { // given operations.submit(submission(null)); - operations.claimNext("worker-a", Duration.ofMinutes(1), NOW); - operations.succeed(ID, "worker-a", "/results/1", NOW); + DurableOperationClaim claim = + operations.claimNext("worker-a", Duration.ofMinutes(1), NOW).orElseThrow(); + operations.succeed(claim, "/results/1", NOW); // when Optional canceled = diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java new file mode 100644 index 00000000..13f6efaf --- /dev/null +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java @@ -0,0 +1,80 @@ +package dev.caskeleton.application.operation; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.idempotency.RequestFingerprint; +import dev.caskeleton.application.outbox.ClaimedOutboxEvent; +import dev.caskeleton.application.outbox.OutboxEventStatus; +import dev.caskeleton.application.outbox.OutboxLease; +import dev.caskeleton.application.outbox.testkit.OutboxEvents; +import java.time.Duration; +import java.time.Instant; +import java.util.stream.Stream; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** The claims and leases a store builds from its rows refuse a bad value as each record says. */ +class StoreBuiltRecordsTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + + static Stream refusedValues() { + DurableOperationSubmission submission = + new DurableOperationSubmission( + new DurableOperationId("op-1"), + "reports.generate", + "alice", + "tenant-a", + new RequestFingerprint("a".repeat(64)), + "{}", + NOW, + Duration.ofHours(1)); + return Stream.of( + Arguments.of( + "owner 없는 claim", + (ThrowingCallable) () -> new DurableOperationClaim(submission, " ", 1), + "a claim needs an owner"), + Arguments.of( + "fence 0인 claim", + (ThrowingCallable) () -> new DurableOperationClaim(submission, "worker-1", 0), + "fence must be >= 1, was 0"), + Arguments.of( + "owner 없는 lease", + (ThrowingCallable) () -> new OperationLease(" ", NOW, NOW.plusSeconds(30)), + "a lease needs an owner"), + Arguments.of( + "잡는 순간 끝나는 lease", + (ThrowingCallable) () -> new OperationLease("worker-1", NOW, NOW), + "a lease that expires when taken is not a lease"), + Arguments.of( + "owner 없는 outbox lease", + (ThrowingCallable) () -> new OutboxLease(OutboxEvents.claimed("evt-1", NOW, 1), " ", 1), + "owner must not be blank"), + Arguments.of( + "token 0인 outbox lease", + (ThrowingCallable) + () -> new OutboxLease(OutboxEvents.claimed("evt-1", NOW, 1), "relay-1", 0), + "token must be >= 1, was 0"), + Arguments.of( + "미룬 claim이 claim보다 많은 outbox 행", + (ThrowingCallable) + () -> + new ClaimedOutboxEvent( + OutboxEvents.event("evt-2", NOW), OutboxEventStatus.IN_FLIGHT, 1, 2), + "deferredAttemptCount must be between 0 and attemptCount (1), was 2")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("refusedValues") + void aValueAStoreCannotHaveBuiltIsAServerFault( + String name, ThrowingCallable build, String message) { + // when + Throwable refused = catchThrowable(build); + + // then + assertThat(refused).isInstanceOf(IllegalStateException.class).hasMessageContaining(message); + } +} diff --git a/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java b/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java index a5a6a3fa..b22ab546 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java @@ -49,7 +49,8 @@ class ClaimedOutboxEventContractTest { // then assertThat(thrown) - .isInstanceOf(IllegalArgumentException.class) + .as("a store builds the claim from its row, so a bad count is the store's fault") + .isInstanceOf(IllegalStateException.class) .hasMessage("attemptCount must be >= 0, was -1"); } } diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java index 4be958d2..38f896aa 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.application.idempotency.RequestFingerprint; import dev.caskeleton.application.operation.DurableOperation; +import dev.caskeleton.application.operation.DurableOperationClaim; import dev.caskeleton.application.operation.DurableOperationId; import dev.caskeleton.application.operation.DurableOperationState; import dev.caskeleton.application.operation.DurableOperationStorePort; @@ -25,6 +26,10 @@ import org.junit.jupiter.api.Test; * *

Every call goes through {@link #inTransaction}, because the real store refuses writes without * a caller transaction and because a resubmission must leave that transaction usable. + * + *

A lease write names the {@link DurableOperationClaim} it holds. The fence cases are why: the + * same worker id claiming an operation again must fence out its own earlier run, which a check on + * the worker id alone lets through. */ public abstract class DurableOperationStoreContract { @@ -33,6 +38,9 @@ public abstract class DurableOperationStoreContract { private static final Duration TTL = Duration.ofDays(1); private static final String HASH_A = "a".repeat(64); private static final String HASH_B = "b".repeat(64); + private static final String HASH_C = "c".repeat(64); + private static final OperationFailure TIMED_OUT = + new OperationFailure("DEPENDENCY_TIMEOUT", "timed out", true); /** The store under test, empty at the start of each test. */ protected abstract DurableOperationStorePort store(); @@ -182,10 +190,36 @@ public abstract class DurableOperationStoreContract { submit(submission); // when - Optional claimed = claim("worker-a", NOW); + Optional claimed = claim("worker-a", NOW); // then - assertThat(claimed).contains(submission); + assertThat(claimed).contains(new DurableOperationClaim(submission, "worker-a", 1)); + } + + @Test + void everyClaimOfAnOperationGetsAHigherFence() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim first = claim("worker-a", NOW).orElseThrow(); + + // when + DurableOperationClaim second = claim("worker-a", NOW.plus(LEASE).plusSeconds(1)).orElseThrow(); + + // then + assertThat(second.fence()).isGreaterThan(first.fence()); + } + + @Test + void canceledOperationIsNotHandedToAWorker() { + // given + submit(submission("op-1", null, HASH_A)); + cancel("op-1", "alice", NOW.plusSeconds(5)); + + // when + Optional claimed = claim("worker-a", NOW.plusSeconds(6)); + + // then + assertThat(claimed).isEmpty(); } @Test @@ -195,7 +229,7 @@ public abstract class DurableOperationStoreContract { claim("worker-a", NOW); // when - Optional claimed = claim("worker-b", NOW.plusSeconds(30)); + Optional claimed = claim("worker-b", NOW.plusSeconds(30)); // then assertThat(claimed).isEmpty(); @@ -224,11 +258,11 @@ public abstract class DurableOperationStoreContract { void heartbeatFromTheLeaseHolderExtendsTheLease() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); Instant beat = NOW.plusSeconds(60); // when - inTransaction(() -> store().heartbeat(id("op-1"), "worker-a", LEASE, beat)); + inTransaction(() -> store().heartbeat(held, LEASE, beat)); // then assertThat(find("op-1", "alice", beat).orElseThrow().lease()) @@ -239,13 +273,12 @@ public abstract class DurableOperationStoreContract { void heartbeatFromAWorkerThatLostTheLeaseIsRefused() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim stale = claim("worker-a", NOW).orElseThrow(); Instant later = NOW.plus(LEASE).plusSeconds(1); claim("worker-b", later); // when - boolean extended = - inTransaction(() -> store().heartbeat(id("op-1"), "worker-a", LEASE, later.plusSeconds(1))); + boolean extended = inTransaction(() -> store().heartbeat(stale, LEASE, later.plusSeconds(1))); // then assertThat(extended).isFalse(); @@ -255,12 +288,12 @@ public abstract class DurableOperationStoreContract { void progressFromTheLeaseHolderIsPublished() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); OperationProgressSnapshot progress = new OperationProgressSnapshot(3, Optional.of(10L), Optional.of("extracting")); // when - inTransaction(() -> store().reportProgress(id("op-1"), "worker-a", progress)); + inTransaction(() -> store().reportProgress(held, progress)); // then assertThat(find("op-1", "alice", NOW).orElseThrow().progress()).contains(progress); @@ -270,12 +303,13 @@ public abstract class DurableOperationStoreContract { void progressFromAnotherWorkerIsRefused() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); + DurableOperationClaim impostor = + new DurableOperationClaim(held.submission(), "worker-b", held.fence()); // when boolean published = - inTransaction( - () -> store().reportProgress(id("op-1"), "worker-b", OperationProgressSnapshot.of(1))); + inTransaction(() -> store().reportProgress(impostor, OperationProgressSnapshot.of(1))); // then assertThat(published).isFalse(); @@ -285,11 +319,11 @@ public abstract class DurableOperationStoreContract { void successFromTheLeaseHolderIsRecorded() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); Instant done = NOW.plusSeconds(10); // when - inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", done)); + inTransaction(() -> store().succeed(held, "/results/a", done)); // then assertThat(find("op-1", "alice", done).orElseThrow()) @@ -309,28 +343,106 @@ public abstract class DurableOperationStoreContract { void staleWorkerCannotRecordAResultOverItsSuccessor() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim stale = claim("worker-a", NOW).orElseThrow(); Instant later = NOW.plus(LEASE).plusSeconds(1); claim("worker-b", later); // when boolean recorded = - inTransaction( - () -> store().succeed(id("op-1"), "worker-a", "/results/a", later.plusSeconds(1))); + inTransaction(() -> store().succeed(stale, "/results/a", later.plusSeconds(1))); // then assertThat(recorded).isFalse(); } + @Test + void heartbeatOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = inTransaction(() -> store().heartbeat(earlier, LEASE, later.plusSeconds(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void progressOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = + inTransaction(() -> store().reportProgress(earlier, OperationProgressSnapshot.of(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void successOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = + inTransaction(() -> store().succeed(earlier, "/results/earlier", later.plusSeconds(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void failureOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = inTransaction(() -> store().fail(earlier, TIMED_OUT, later.plusSeconds(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void resultOfTheRunThatTookOverSurvivesItsSameWorkerPredecessor() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + DurableOperationClaim current = claim("worker-a", later).orElseThrow(); + inTransaction(() -> store().succeed(current, "/results/current", later.plusSeconds(1))); + + // when + inTransaction(() -> store().succeed(earlier, "/results/earlier", later.plusSeconds(2))); + + // then + assertThat(find("op-1", "alice", later.plusSeconds(2)).orElseThrow().resultReference()) + .contains("/results/current"); + } + @Test void failureFromTheLeaseHolderIsRecorded() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); OperationFailure failure = new OperationFailure("DEPENDENCY_TIMEOUT", "timed out", true); // when - inTransaction(() -> store().fail(id("op-1"), "worker-a", failure, NOW.plusSeconds(10))); + inTransaction(() -> store().fail(held, failure, NOW.plusSeconds(10))); // then assertThat(find("op-1", "alice", NOW.plusSeconds(10)).orElseThrow()) @@ -369,8 +481,8 @@ public abstract class DurableOperationStoreContract { void cancelNeverReversesASuccess() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); - inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", NOW.plusSeconds(10))); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); + inTransaction(() -> store().succeed(held, "/results/a", NOW.plusSeconds(10))); // when cancel("op-1", "alice", NOW.plusSeconds(20)); @@ -407,19 +519,35 @@ public abstract class DurableOperationStoreContract { } @Test - void expirySweepExpiresAgedOutWorkAndLeavesFinishedWorkAlone() { + void reclaimLeavesALiveLeaseAlone() { // given submit(submission("op-1", null, HASH_A)); claim("worker-a", NOW); - inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", NOW.plusSeconds(10))); + + // when + List reclaimed = + inTransaction(() -> store().reclaimExpiredLeases(NOW.plusSeconds(30))); + + // then + assertThat(reclaimed).isEmpty(); + } + + @Test + void expirySweepExpiresAgedOutWorkAndLeavesFinishedWorkAlone() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); + inTransaction(() -> store().succeed(held, "/results/a", NOW.plusSeconds(10))); submit(submission("op-2", null, HASH_B)); + submit(submission("op-3", null, HASH_C)); + claim("worker-b", NOW.plusSeconds(20)); // when List expired = inTransaction(() -> store().expireStaleOperations(NOW.plus(TTL).plus(TTL))); // then - assertThat(expired).containsExactly(id("op-2")); + assertThat(expired).containsExactlyInAnyOrder(id("op-2"), id("op-3")); } private DurableOperationSubmitResult submit(DurableOperationSubmission submission) { @@ -430,7 +558,7 @@ public abstract class DurableOperationStoreContract { return inTransaction(() -> store().find(id(operationId), principal, now)); } - private Optional claim(String workerId, Instant now) { + private Optional claim(String workerId, Instant now) { return inTransaction(() -> store().claimNext(workerId, LEASE, now)); } diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java index 019193c3..3c76c99a 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java @@ -1,6 +1,7 @@ package dev.caskeleton.application.operation.testkit; import dev.caskeleton.application.operation.DurableOperation; +import dev.caskeleton.application.operation.DurableOperationClaim; import dev.caskeleton.application.operation.DurableOperationId; import dev.caskeleton.application.operation.DurableOperationState; import dev.caskeleton.application.operation.DurableOperationStorePort; @@ -26,8 +27,10 @@ import java.util.concurrent.ConcurrentHashMap; *

Written to the same rules rather than to whatever makes the tests pass: a submission scope * that already exists returns the existing operation, a lease is checked on every write, and a * terminal operation is never reopened. A fake that was laxer than the adapter would let the use - * case tests certify behaviour the real store refuses. {@link DurableOperationStoreContract} holds - * this fake and the adapter to the same cases. + * case tests certify behaviour the real store refuses. Every claim increments the operation's + * fence, and a write is accepted only from the claim holding the current fence, as the adapter's + * {@code lease_fence} column does. {@link DurableOperationStoreContract} holds this fake and the + * adapter to the same cases. * *

Built over a {@link DirectTransactionPort}, every write made inside one of its transactions is * undone when that transaction rolls back, as the database undoes it; built without one, writes @@ -41,6 +44,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor private final Map byScope = new ConcurrentHashMap<>(); private final Map submissions = new ConcurrentHashMap<>(); + private final Map fences = new ConcurrentHashMap<>(); private final DirectTransactionPort transactions; /** A store whose writes stay as they are made. */ @@ -103,7 +107,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public Optional claimNext( + public Optional claimNext( String workerId, Duration leaseDuration, Instant now) { return byId.values().stream() .filter( @@ -126,19 +130,22 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor Optional.empty(), Optional.of(new OperationLease(workerId, now, now.plus(leaseDuration)))); write(claimed.operationId(), claimed); - return submissions.get(claimed.operationId()); + long previous = fences.getOrDefault(claimed.operationId(), 0L); + fences.put(claimed.operationId(), previous + 1); + onRollback(() -> fences.put(claimed.operationId(), previous)); + return new DurableOperationClaim( + submissions.get(claimed.operationId()), workerId, previous + 1); }); } @Override - public boolean heartbeat( - DurableOperationId operationId, String workerId, Duration leaseDuration, Instant now) { - DurableOperation operation = byId.get(operationId); - if (!holdsLease(operation, workerId, now)) { + public boolean heartbeat(DurableOperationClaim claim, Duration leaseDuration, Instant now) { + DurableOperation operation = byId.get(claim.operationId()); + if (!holdsLease(operation, claim, now)) { return false; } write( - operationId, + claim.operationId(), replace( operation, operation.state(), @@ -152,14 +159,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public boolean reportProgress( - DurableOperationId operationId, String workerId, OperationProgressSnapshot progress) { - DurableOperation operation = byId.get(operationId); - if (operation == null || !ownedBy(operation, workerId)) { + public boolean reportProgress(DurableOperationClaim claim, OperationProgressSnapshot progress) { + DurableOperation operation = byId.get(claim.operationId()); + if (!heldBy(operation, claim)) { return false; } write( - operationId, + claim.operationId(), replace( operation, operation.state(), @@ -173,17 +179,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public boolean succeed( - DurableOperationId operationId, - String workerId, - String resultReference, - Instant completedAt) { - DurableOperation operation = byId.get(operationId); - if (operation == null || !ownedBy(operation, workerId)) { + public boolean succeed(DurableOperationClaim claim, String resultReference, Instant completedAt) { + DurableOperation operation = byId.get(claim.operationId()); + if (!heldBy(operation, claim)) { return false; } write( - operationId, + claim.operationId(), replace( operation, DurableOperationState.SUCCEEDED, @@ -197,17 +199,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public boolean fail( - DurableOperationId operationId, - String workerId, - OperationFailure failure, - Instant completedAt) { - DurableOperation operation = byId.get(operationId); - if (operation == null || !ownedBy(operation, workerId)) { + public boolean fail(DurableOperationClaim claim, OperationFailure failure, Instant completedAt) { + DurableOperation operation = byId.get(claim.operationId()); + if (!heldBy(operation, claim)) { return false; } write( - operationId, + claim.operationId(), replace( operation, DurableOperationState.FAILED, @@ -316,6 +314,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor byId.clear(); byScope.clear(); submissions.clear(); + fences.clear(); } // Same as the adapter: a poll after the TTL sees EXPIRED even before the sweeper runs. @@ -334,13 +333,15 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor Optional.empty()); } - private static boolean holdsLease(DurableOperation operation, String workerId, Instant now) { - return operation != null && ownedBy(operation, workerId) && !operation.leaseExpiredAt(now); + private boolean holdsLease(DurableOperation operation, DurableOperationClaim claim, Instant now) { + return heldBy(operation, claim) && !operation.leaseExpiredAt(now); } - private static boolean ownedBy(DurableOperation operation, String workerId) { - return operation.state() == DurableOperationState.RUNNING - && operation.lease().map(lease -> lease.workerId().equals(workerId)).orElse(false); + private boolean heldBy(DurableOperation operation, DurableOperationClaim claim) { + return operation != null + && operation.state() == DurableOperationState.RUNNING + && operation.lease().map(lease -> lease.workerId().equals(claim.workerId())).orElse(false) + && fences.getOrDefault(claim.operationId(), 0L) == claim.fence(); } private static DurableOperation replace( diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java index 4577fec0..ad90e67d 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java @@ -44,6 +44,16 @@ public interface CleanupQueuePort { /** Records a failed attempt and when to try again, if {@code item}'s claim is still current. */ void markFailed(CleanupItem item, String reasonCode, Instant nextAttemptAt); + /** + * Gives a claimed item back unattempted, if {@code item}'s claim is still current. + * + *

For an item the batch claimed and then did not start — its byte budget or its claim's lease + * ran out first. The claim ends and the item is claimable again at {@code nextAttemptAt}; the + * attempt counter does not move, because nothing was tried. Recorded as a failure instead, every + * such release spent an attempt, and a busy queue's tail was abandoned without ever being tried. + */ + void release(CleanupItem item, Instant nextAttemptAt); + /** * Parks an item for good, if {@code item}'s claim is still current. * diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java index d28a7d40..71033678 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java @@ -49,6 +49,20 @@ public final class DefaultCleanupService implements CleanupService { */ public static final Duration CLAIM_LEASE = Duration.ofMinutes(10); + /** + * No claimed item is started once less than this is left of the batch's claim lease. + * + *

A batch claims up to {@code maxItems} under one lease and works them one after another, so + * the last item starts long after the claim. Past the lease, the reaper may already have handed + * that item to another worker, and both delete it and settle its quota. The outbox relay checks + * its batch against its lease when it is assembled and the notification scheduler claims only + * what it can start; cleanup has no bound on one delete to check against, so it stops starting + * items instead, and releases the rest for the next batch the way the byte budget does (round9 + * review 2026-11-12, RR9-01-23). Two minutes is the time one delete against slow storage is + * allowed to finish inside its lease. + */ + static final Duration LAST_START_BEFORE_LEASE_ENDS = Duration.ofMinutes(2); + private final CleanupQueuePort queue; private final CleanupContentPort contentGateway; private final FileMetadataPort metadataStore; @@ -95,6 +109,7 @@ public final class DefaultCleanupService implements CleanupService { // claiming means the recovered items are eligible in this same pass. transactions.inWrite(() -> queue.reclaimExpiredClaims(now, maxItems)); List due = transactions.inWrite(() -> queue.claimDue(now, maxItems, CLAIM_LEASE)); + Instant lastStart = now.plus(CLAIM_LEASE).minus(LAST_START_BEFORE_LEASE_ENDS); int deleted = 0; int skippedActiveLease = 0; @@ -113,10 +128,16 @@ public final class DefaultCleanupService implements CleanupService { failed++; continue; } + if (!clock.instant().isBefore(lastStart)) { + // The claim's lease is nearly spent; an item started now could still be running when the + // reaper hands it to another worker. Given back untried, like the byte budget's. + release(item, now); + continue; + } if (reclaimed >= maxBytes) { - // The byte budget is spent; the remaining claims are released for the next batch rather - // than being marked done, so nothing is silently dropped. - markFailed(item, "BATCH_BYTE_BUDGET_EXHAUSTED", now); + // The byte budget is spent; the remaining claims are given back untried rather than + // being marked done, so nothing is silently dropped and no attempt is spent on them. + release(item, now); continue; } Outcome outcome = process(item, now); @@ -330,6 +351,11 @@ public final class DefaultCleanupService implements CleanupService { }); } + /** Gives a claimed item back untried, in a write transaction of its own. */ + private void release(CleanupItem item, Instant nextAttemptAt) { + transactions.inWrite(() -> queue.release(item, nextAttemptAt)); + } + /** * Reschedules an item, or parks it once this failure spends its retry budget. * diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java index dd0bd88d..ead9afb2 100644 --- a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java +++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java @@ -26,7 +26,10 @@ import dev.caskeleton.application.fileserver.testkit.ReadyFileFixture; import dev.caskeleton.application.fileserver.testkit.RecordingQuotaReclaimGateway; import java.time.Clock; import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.List; import java.util.OptionalLong; import java.util.UUID; import org.junit.jupiter.api.DisplayName; @@ -321,7 +324,91 @@ class CleanupServiceTest { CleanupBatchResult result = cleanup.runBatch(100, 1); assertThat(result.deleted()).isEqualTo(1); - assertThat(queue.failed().values()).contains("BATCH_BYTE_BUDGET_EXHAUSTED"); + assertThat(List.of(queue.released().size(), queue.failed().size())) + .as("the deferred item is given back untried; recorded as failed it spent an attempt") + .containsExactly(1, 0); + } + + @Test + void itemsTheClaimLeaseNoLongerCoversAreReleasedNotStarted() { + // given + // Each staging delete takes just over half the time an item may still start in (the lease less + // its last two minutes): the first two items run, the third is handed back to the queue. + MovableClock slowStorage = new MovableClock(FileserverFixtures.NOW); + Duration perDelete = + DefaultCleanupService.CLAIM_LEASE + .minus(DefaultCleanupService.LAST_START_BEFORE_LEASE_ENDS) + .dividedBy(2) + .plusSeconds(1); + content.afterEachStagingDiscard(() -> slowStorage.advance(perDelete)); + CleanupService timed = + new DefaultCleanupService( + queue, + content, + metadata, + sessions, + quota, + Duration.ofMinutes(5), + FileserverMetricsPort.noop(), + transactions, + slowStorage); + for (int item = 0; item < 3; item++) { + cancelledStagingItem(); + } + + // when + CleanupBatchResult result = timed.runBatch(100, 1L << 30); + + // then + assertThat(List.of(result.deleted(), queue.released().size(), queue.failed().size())) + .as( + "the third item comes back untried rather than failed; that a release leaves the" + + " attempt counter where it was is the queue contract's") + .containsExactly(2, 1, 0); + } + + private void cancelledStagingItem() { + UploadId uploadId = UploadId.of(UUID.randomUUID()); + FileId fileId = FileId.of(UUID.randomUUID()); + sessions.create( + new UploadSessionDraft( + uploadId, + fileId, + UploadMode.SINGLE_SHOT, + OptionalLong.of(SIZE), + FileserverFixtures.NOW.plusSeconds(3600))); + sessions.terminate(uploadId); + content.storeStaging(uploadId); + queue.enqueue(CleanupRequest.forStaging(CleanupType.CANCELLED_STAGING, fileId, uploadId)); + } + + /** A clock the slow-storage gateway moves forward. */ + private static final class MovableClock extends Clock { + + private Instant now; + + private MovableClock(Instant start) { + this.now = start; + } + + private void advance(Duration duration) { + now = now.plus(duration); + } + + @Override + public ZoneId getZone() { + return ZoneOffset.UTC; + } + + @Override + public Clock withZone(ZoneId zone) { + return this; + } + + @Override + public Instant instant() { + return now; + } } @Test diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java index 0db36011..b9cc169b 100644 --- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java +++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java @@ -119,6 +119,40 @@ public abstract class CleanupQueuePortContract { }); } + @Test + void aReleasedItemIsClaimableAgainWithItsAttemptUnchanged() { + // given + enqueue(contentCleanup()); + CleanupItem item = claim(due(), 10).get(0); + settle(() -> queue().release(item, due())); + + // when + List again = claim(due(), 10); + + // then + assertThat(again) + .singleElement() + .extracting(CleanupItem::cleanupId, CleanupItem::attempt) + .containsExactly(item.cleanupId(), 0); + } + + @Test + void aReleaseUnderAReclaimedTokenChangesNothing() { + // given + enqueue(contentCleanup()); + CleanupItem stale = claim(due(), 10).get(0); + settle(() -> queue().release(stale, due())); + var unused = claim(due(), 10); + + // when + settle(() -> queue().release(stale, due())); + + // then + assertThat(claim(due(), 10)) + .as("the stale release would hand the current claim's item to a second worker") + .isEmpty(); + } + @Test @DisplayName("완료된 항목은 다시 claim되지 않는다") void aDoneItemIsNeverClaimedAgain() { diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java index 8a485a3c..3b842068 100644 --- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java +++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java @@ -7,6 +7,7 @@ import dev.caskeleton.application.fileserver.api.content.DeleteResult; import dev.caskeleton.application.fileserver.cleanup.CleanupContentPort; import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; import java.util.Set; /** Tracks which physical objects still exist so a test can prove nothing live was removed. */ @@ -16,6 +17,7 @@ public final class FakeCleanupContentGateway implements CleanupContentPort { private final Set stagingObjects = new LinkedHashSet<>(); private final Set quarantined = new LinkedHashSet<>(); private final List preconditions = new java.util.ArrayList<>(); + private Runnable afterStagingDiscard = () -> {}; public void store(ContentKey key) { objects.add(key); @@ -46,9 +48,20 @@ public final class FakeCleanupContentGateway implements CleanupContentPort { return DeleteResult.removed(precondition.expectedSize().orElse(0)); } + /** + * Runs {@code hook} after every staging discard, for a test that needs the storage to be slow — + * advancing its clock — or to fail mid-batch. + * + * @param hook what a discard does after removing the staging object + */ + public void afterEachStagingDiscard(Runnable hook) { + afterStagingDiscard = Objects.requireNonNull(hook, "hook"); + } + @Override public void discardStaging(UploadId uploadId) { stagingObjects.remove(uploadId); + afterStagingDiscard.run(); } public void quarantine(ContentKey key) { diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java index 50edd600..0dc81d84 100644 --- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java +++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java @@ -35,6 +35,7 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { private final Map rows = new LinkedHashMap<>(); private final List done = new ArrayList<>(); private final Map failed = new LinkedHashMap<>(); + private final List released = new ArrayList<>(); private final Map abandoned = new LinkedHashMap<>(); private final List reclaimSweeps = new ArrayList<>(); private final List claimSweeps = new ArrayList<>(); @@ -108,6 +109,15 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { } } + @Override + public void release(CleanupItem item, Instant nextAttemptAt) { + Row row = rows.get(item.cleanupId()); + if (row != null && item.claimToken().equals(row.claimToken)) { + row.release(nextAttemptAt); + released.add(item); + } + } + @Override public void markAbandoned(CleanupItem item, String reasonCode) { if (settle(item, Status.ABANDONED, clock.instant())) { @@ -151,6 +161,11 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { return Map.copyOf(failed); } + /** Items the caller gave back untried, in order. */ + public List released() { + return List.copyOf(released); + } + /** Items the caller parked for good, with the reason given. */ public Map abandoned() { return Map.copyOf(abandoned); @@ -180,6 +195,13 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { this.nextAttemptAt = nextAttemptAt; } + private void release(Instant nextAttemptAt) { + this.status = attempt == 0 ? Status.PENDING : Status.FAILED; + this.nextAttemptAt = nextAttemptAt; + this.claimToken = null; + this.leaseUntil = null; + } + private void settle(Status status, Instant nextAttemptAt) { this.status = status; this.attempt++; diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java index bd902930..40006855 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java @@ -11,6 +11,10 @@ import java.util.Optional; * the work is done; and the same id with a different command is a mistake worth reporting rather * than an idempotent repeat. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param outcome what the caller may do * @param existing the earlier result, when there is one */ @@ -37,7 +41,7 @@ public record AdminOperationClaim(Outcome outcome, OptionalA requeue is one write conditional on the abandoned lease recovery found (owner, fence, state, + * expiry), not a read followed by an unconditional transition. The pass runs on every instance, and + * between finding a job and writing it another instance's recovery may have requeued it and a new + * worker claimed it, or the holder may have finished; the unconditional write put such a job back + * to {@code READY_TO_DISPATCH} and it was dispatched again (round9 review 2026-11-12, RR9-01-12). A + * requeue that matches nothing is not counted as handled. + * *

Every write opens its own transaction; the pass itself runs on a bare scheduler thread. The * stores behind these ports are modifying JPA queries, which throw outside a transaction — so the * pass used to fail on its first statement, every time, and the scheduler only logged it. It lives @@ -110,14 +116,14 @@ public final class LeaseRecoveryService { // Retiring expired deliveries rides along with recovery because both answer the same operator // question — "why is this row still here?" — and neither needs a scheduler of its own. int handled = transactions.inWrite(() -> leases.expireOverdue(batchSize)); - List abandoned = leases.expiredDispatching(batchSize, staleAfter); - for (var recipientDeliveryId : abandoned) { + List abandoned = leases.expiredDispatching(batchSize, staleAfter); + for (var lease : abandoned) { + RecipientDeliveryId recipientDeliveryId = lease.recipientDeliveryId(); var attemptsOfDelivery = attempts.attemptsOf(recipientDeliveryId); if (attemptsOfDelivery.isEmpty()) { // Crash before the attempt row: no provider call can have happened, so this is the one // safe requeue. Leaving it DISPATCHING stranded the delivery permanently. - requeue(recipientDeliveryId); - handled++; + handled += requeue(lease); continue; } if (attemptsOfDelivery.stream().allMatch(attempt -> attempt.completedAt().isPresent())) { @@ -127,8 +133,7 @@ public final class LeaseRecoveryService { // the delivery, and dispatch derives the next step from exactly those facts — an accepted // submission stops as COMPLETED, an ambiguous one goes to reconciliation — so handing it // back to dispatch cannot send a second copy of something the provider accepted. - requeue(recipientDeliveryId); - handled++; + handled += requeue(lease); continue; } for (var attempt : attemptsOfDelivery) { @@ -141,25 +146,25 @@ public final class LeaseRecoveryService { // identically, so every abandoned attempt had to be reconciled. The pre-call row the // dispatcher writes never lands here: it records the start as unknown, because it is // still the stored state while the provider call runs. - requeue(recipientDeliveryId); + handled += requeue(lease); } else { // Outside any transaction: reconciliation asks the provider over the network and opens // its own boundaries for what it writes. reconciliation.reconcile(attempt.id()); + handled++; } - handled++; } } } return handled; } - private void requeue(RecipientDeliveryId recipientDeliveryId) { - transactions.inWrite( - () -> - deliveries.transition( - recipientDeliveryId, - RecipientDeliveryState.READY_TO_DISPATCH, - java.util.Optional.of(clock.instant()))); + /** Requeues the abandoned dispatch; 1 when it was still abandoned and moved, 0 otherwise. */ + private int requeue(RecipientLease abandoned) { + return transactions + .inWrite(() -> deliveries.requeueAbandoned(abandoned, clock.instant())) + .isPresent() + ? 1 + : 0; } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java index 101c9f35..c82fa5aa 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java @@ -49,4 +49,28 @@ public interface RecipientDeliveryStorePort { RecipientDeliveryState state, Optional nextDispatchAt, RecipientLease lease); + + /** + * Returns a job a dead worker left dispatching to the queue, only while it is still the abandoned + * dispatch recovery found. + * + *

One conditional write on the abandoned lease — same owner, same fence, still {@code + * DISPATCHING}, lease not extended past what recovery saw — rather than a read followed by an + * unconditional transition. Recovery runs on every instance, and between finding the job and + * writing it another instance may have requeued it and a new worker claimed it, or the original + * holder may have finished; an unconditional write put such a job back to {@code + * READY_TO_DISPATCH} and it was dispatched again. + * + *

The target state is the operation's meaning, as {@code EXPIRED} is {@code + * RecipientLeaseStorePort.expireOverdue}'s, so it is not an argument. What recovery decides is + * which operation applies: this one when no attempt may have reached the provider, the + * reconciliation path otherwise. + * + * @param abandoned the lease as {@link RecipientLeaseStorePort#expiredDispatching} returned it + * @param nextDispatchAt when the job becomes due again + * @return the requeued job, or empty when it is no longer that abandoned dispatch and nothing was + * written + */ + Optional requeueAbandoned( + RecipientLease abandoned, Instant nextDispatchAt); } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java index 0b9962a3..5660e4d3 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java @@ -16,6 +16,10 @@ import java.util.Objects; * that no longer exists and updates nothing. It travels with the lease rather than being re-read, * because re-reading is the race. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param recipientDeliveryId the job this lease is for * @param owner the holder, including the process incarnation that claimed it * @param fence the claim generation; a lower fence has been superseded @@ -29,11 +33,11 @@ public record RecipientLease( Objects.requireNonNull(owner, "owner"); Objects.requireNonNull(leaseUntil, "leaseUntil"); if (owner.isBlank()) { - throw new IllegalArgumentException("owner"); + throw new IllegalStateException("a lease needs an owner"); } if (fence < 1) { // A zero fence is the value an unclaimed row holds; a lease can never legitimately carry it. - throw new IllegalArgumentException("fence"); + throw new IllegalStateException("fence must be >= 1, was " + fence); } } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java index 634fabd7..da251a5d 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java @@ -1,6 +1,5 @@ package dev.caskeleton.application.notification.platform.dispatch; -import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; import java.time.Duration; import java.util.List; import java.util.Optional; @@ -25,6 +24,9 @@ public interface RecipientLeaseStorePort { /** * Extend a lease that is still being worked on. * + *

The template's dispatcher never calls it: a dispatch runs inside the lease it claimed. It is + * here for a deployment whose own dispatch outlasts one lease. + * * @return the extended lease, or empty when this holder no longer owns the job */ Optional renew(RecipientLease lease, Duration leaseDuration); @@ -40,8 +42,24 @@ public interface RecipientLeaseStorePort { /** Whether this lease is still the current one, for checking before an external side effect. */ boolean stillHeld(RecipientLease lease); - /** Jobs left {@code DISPATCHING} by a worker that died, for reconciliation. */ - List expiredDispatching(int limit, Duration olderThan); + /** + * Jobs left {@code DISPATCHING} by a worker that died, for recovery. + * + *

Each comes back as the lease it was abandoned under — owner, fence and expiry as they stood + * when it was found. Recovery decides what to do from attempts it reads afterwards, and the lease + * is what lets its write say "only if this is still that abandoned dispatch": another instance's + * recovery, a new claim or the holder itself may have moved the job in between. + * + *

Only jobs a claim leased: an owner and a fence of at least one. A {@code DISPATCHING} job + * without them is no lease anyone holds — one left before the fence existed, or one moved there + * without a claim — and is not returned; offered, it would be a lease nobody can build, and the + * pass would fail on it every time. + * + * @param limit how many to return + * @param olderThan how long a lease must have been expired before the job counts as abandoned + * @return the abandoned leases, oldest expiry first + */ + List expiredDispatching(int limit, Duration olderThan); /** * Retire deliveries whose window closed before anyone claimed them. diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java index c8545b62..e0d3e401 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java @@ -24,6 +24,10 @@ import java.util.UUID; * the store's writes after the claim match on it, so an instance that held the job past its lease * cannot reschedule or remove it once another instance has claimed it since. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param id the job identity * @param attemptId the attempt whose outcome is unknown * @param providerProfileId the profile that must answer, since capability is per profile @@ -55,10 +59,10 @@ public record ReconciliationJob( Objects.requireNonNull(createdAt, "createdAt"); Objects.requireNonNull(updatedAt, "updatedAt"); if (attempts < 0) { - throw new IllegalArgumentException("attempts must not be negative"); + throw new IllegalStateException("attempts must not be negative"); } if (claimFence < 0) { - throw new IllegalArgumentException("claimFence must not be negative"); + throw new IllegalStateException("claimFence must not be negative"); } } } diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java index 7ff29358..ed508150 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java @@ -186,6 +186,40 @@ class LeaseRecoveryServiceTest { assertThat(recovery.recoverOnce()).isEqualTo(3); } + @Test + void aDeliveryItsHolderFinishedAfterRecoveryFoundItIsNotRequeued() { + // given + RecipientDeliveryId id = jobOf(NOW, Optional.empty()); + RecipientLease lease = queue.claim("worker-a", 10, Duration.ofSeconds(30)).getFirst(); + workerDied(); + // The holder completes the job between recovery finding it abandoned and recovery writing it + // — the window another instance or a slow worker uses in production. + attempts.beforeAttemptsOf( + delivery -> { + var unused = + queue.transitionHeldBy( + delivery, RecipientDeliveryState.COMPLETED, Optional.empty(), lease); + }); + LeaseRecoveryService racing = + new LeaseRecoveryService( + queue, + attempts, + queue, + reconciled::add, + new DirectTransactionPort(), + CLOCK, + Duration.ofMinutes(1), + 10); + + // when + racing.recoverOnce(); + + // then + assertThat(stateOf(id)) + .as("the holder was only slow; recovery's requeue must not reopen what it finished") + .isEqualTo(RecipientDeliveryState.COMPLETED); + } + private static dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence notStarted() { return dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java new file mode 100644 index 00000000..dd33283f --- /dev/null +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java @@ -0,0 +1,76 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.notification.platform.admin.AdminOperationClaim; +import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; +import dev.caskeleton.application.notification.platform.api.ProviderProfileId; +import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * The leases, jobs and claims a notification store builds from its rows refuse a bad value as each + * record says. + */ +class StoreBuiltRecordsTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + + static Stream refusedValues() { + RecipientDeliveryId delivery = new RecipientDeliveryId(UUID.randomUUID()); + return Stream.of( + Arguments.of( + "owner 없는 수신자 lease", + (ThrowingCallable) () -> new RecipientLease(delivery, " ", 1, NOW), + "a lease needs an owner"), + Arguments.of( + "fence 0인 수신자 lease", + (ThrowingCallable) () -> new RecipientLease(delivery, "worker-1", 0, NOW), + "fence must be >= 1, was 0"), + Arguments.of( + "시도 수가 음수인 맞추기 job", + (ThrowingCallable) () -> job(-1, 0), + "attempts must not be negative"), + Arguments.of( + "fence가 음수인 맞추기 job", + (ThrowingCallable) () -> job(0, -1), + "claimFence must not be negative"), + Arguments.of( + "결과 없는 운영자 재생", + (ThrowingCallable) + () -> new AdminOperationClaim(AdminOperationClaim.Outcome.REPLAY, Optional.empty()), + "a replay must carry the result it is replaying")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("refusedValues") + void aValueAStoreCannotHaveBuiltIsAServerFault( + String name, ThrowingCallable build, String message) { + // when + Throwable refused = catchThrowable(build); + + // then + assertThat(refused).isInstanceOf(IllegalStateException.class).hasMessageContaining(message); + } + + private static ReconciliationJob job(int attempts, long claimFence) { + return new ReconciliationJob( + UUID.randomUUID(), + new DeliveryAttemptId(UUID.randomUUID()), + new ProviderProfileId("email-main"), + NOW, + attempts, + Optional.empty(), + NOW, + NOW, + claimFence); + } +} diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java index ae257085..65bc4d4d 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; /** * The fake of the attempt store and the two read models built on it, held to {@link @@ -47,6 +48,8 @@ public final class InMemoryDeliveryAttemptStore /** Stored attempts, in insertion order, as their rows read back. */ private final Map rows = new LinkedHashMap<>(); + private volatile Consumer beforeAttemptsOf = id -> {}; + private final Map requestIdHashes = new LinkedHashMap<>(); private final Map engagement = new LinkedHashMap<>(); private final Map suppression = new LinkedHashMap<>(); @@ -140,7 +143,23 @@ public final class InMemoryDeliveryAttemptStore } @Override - public synchronized List attemptsOf( + public List attemptsOf(RecipientDeliveryId recipientDeliveryId) { + // Outside the monitor: the hook writes to the recipient queue, as another worker would. + beforeAttemptsOf.accept(recipientDeliveryId); + return attemptsFor(recipientDeliveryId); + } + + /** + * Runs {@code hook} before every {@link #attemptsOf} read, for a test that needs another worker + * to act between a caller finding a delivery and reading its attempts. + * + * @param hook what happens, given the delivery being read + */ + public void beforeAttemptsOf(Consumer hook) { + beforeAttemptsOf = Objects.requireNonNull(hook, "hook"); + } + + private synchronized List attemptsFor( RecipientDeliveryId recipientDeliveryId) { return rows.values().stream() .filter(row -> row.recipientDeliveryId().equals(recipientDeliveryId)) @@ -155,7 +174,7 @@ public final class InMemoryDeliveryAttemptStore @Override public synchronized int nextAttemptNo(RecipientDeliveryId recipientDeliveryId) { - return attemptsOf(recipientDeliveryId).stream() + return attemptsFor(recipientDeliveryId).stream() .mapToInt(DeliveryAttemptRecord::attemptNo) .max() .orElse(0) diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java index 89517a47..7714a522 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java @@ -160,7 +160,7 @@ public final class InMemoryRecipientQueue } @Override - public List expiredDispatching(int limit, Duration olderThan) { + public List expiredDispatching(int limit, Duration olderThan) { Objects.requireNonNull(olderThan, "olderThan"); if (limit < 1) { throw new IllegalArgumentException("limit"); @@ -170,9 +170,18 @@ public final class InMemoryRecipientQueue return jobs() .filter(job -> job.state() == RecipientDeliveryState.DISPATCHING) .filter(job -> job.leaseUntil().filter(until -> until.isBefore(threshold)).isPresent()) + // Only what a claim wrote: an owner and a fence. The JPA store skips the rest the same + // way. + .filter(job -> job.leaseOwner().isPresent() && fences.getOrDefault(job.id(), 0L) > 0) .sorted(Comparator.comparing((RecipientDeliveryRecord job) -> job.leaseUntil().get())) .limit(limit) - .map(RecipientDeliveryRecord::id) + .map( + job -> + new RecipientLease( + job.id(), + job.leaseOwner().orElseThrow(), + fences.getOrDefault(job.id(), 0L), + job.leaseUntil().orElseThrow())) .toList(); } } @@ -294,6 +303,35 @@ public final class InMemoryRecipientQueue } } + @Override + public Optional requeueAbandoned( + RecipientLease abandoned, Instant nextDispatchAt) { + Objects.requireNonNull(abandoned, "abandoned"); + Objects.requireNonNull(nextDispatchAt, "nextDispatchAt"); + synchronized (requests) { + return heldBy(abandoned) + .filter(job -> job.state() == RecipientDeliveryState.DISPATCHING) + .filter( + job -> + job.leaseUntil() + .filter(until -> !until.isAfter(abandoned.leaseUntil())) + .isPresent()) + .map( + job -> { + RecipientDeliveryRecord moved = + copy( + job, + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(nextDispatchAt), + job.leaseOwner(), + job.leaseUntil(), + clock.instant()); + requests.replace(moved); + return moved; + }); + } + } + // ---- rows ------------------------------------------------------------------------------------- private Stream jobs() { diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java index 7ceefe82..5ec3dead 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java @@ -244,6 +244,84 @@ public abstract class RecipientDeliveryStoreContract { List.of(Optional.empty(), Optional.empty()), RecipientDeliveryState.DISPATCHING, 0); } + @Test + void anAbandonedDispatchIsRequeued() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease abandoned = claim(); + clock.moveTo(NOW.plus(LEASE).plusSeconds(1)); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(abandoned, clock.instant())); + + // then + assertThat( + requeued.map(stored -> List.of(stored.id(), stored.state(), stored.nextDispatchAt()))) + .contains( + List.of( + job.id(), + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(NOW.plus(LEASE).plusSeconds(1)))); + } + + @Test + void requeueOfAJobItsHolderFinishedWritesNothing() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease lease = claim(); + clock.moveTo(NOW.plus(LEASE).plusSeconds(1)); + inTransaction( + () -> + deliveries() + .transitionHeldBy( + job.id(), RecipientDeliveryState.COMPLETED, Optional.empty(), lease)); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(lease, clock.instant())); + + // then + assertThat(List.of(requeued, deliveries().find(job.id()).orElseThrow().state())) + .containsExactly(Optional.empty(), RecipientDeliveryState.COMPLETED); + } + + @Test + void requeueOfAJobAnotherClaimTookWritesNothing() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease abandoned = claim(); + clock.moveTo(NOW.plus(LEASE).plusSeconds(1)); + inTransaction(() -> deliveries().requeueAbandoned(abandoned, clock.instant())); + claim(); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(abandoned, clock.instant())); + + // then + assertThat(List.of(requeued, deliveries().find(job.id()).orElseThrow().state())) + .as("the second recovery found the same abandoned lease; the new claim raised the fence") + .containsExactly(Optional.empty(), RecipientDeliveryState.DISPATCHING); + } + + @Test + void requeueOfAJobWhoseLeaseWasRenewedWritesNothing() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease found = claim(); + clock.moveTo(NOW.plus(LEASE).minusSeconds(1)); + inTransaction(() -> leases().renew(found, LEASE)); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(found, clock.instant())); + + // then + assertThat(List.of(requeued, deliveries().find(job.id()).orElseThrow().state())) + .containsExactly(Optional.empty(), RecipientDeliveryState.DISPATCHING); + } + private RecipientDeliveryRecord givenAQueuedJob() { NotificationRequestRecord request = NotificationRows.request(TENANT, "idem-" + UUID.randomUUID(), NOW); diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java index 0a97a7c1..4ddfb077 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java @@ -278,25 +278,77 @@ public abstract class RecipientLeaseStoreContract { } @Test - @DisplayName("lease가 기준보다 오래 전에 끝난 DISPATCHING 작업만 복구 대상으로 나온다") + @DisplayName("lease가 기준보다 오래 전에 끝난 DISPATCHING 작업만, 버려질 때의 lease로 복구 대상에 나온다") void onlyJobsWhoseLeaseEndedLongEnoughAgoAreAbandoned() { // given - RecipientDeliveryId abandoned = given(RecipientDeliveryState.READY_TO_DISPATCH, NOW, null); - claim("worker-a"); + given(RecipientDeliveryState.READY_TO_DISPATCH, NOW, null); + RecipientLease abandoned = claim("worker-a").getFirst(); clock.moveTo(NOW.plus(LEASE).plusSeconds(30)); given(RecipientDeliveryState.READY_TO_DISPATCH, NOW.plus(LEASE), null); claim("worker-b"); clock.moveTo(NOW.plus(LEASE).plus(Duration.ofMinutes(2))); // when - List expired = leases().expiredDispatching(10, Duration.ofMinutes(1)); + List expired = leases().expiredDispatching(10, Duration.ofMinutes(1)); // then assertThat(expired) - .as("a job whose lease is still running belongs to a live worker") + .as( + "a job whose lease is still running belongs to a live worker; the abandoned one comes" + + " back as the lease it was abandoned under") .containsExactly(abandoned); } + @Test + @DisplayName("claim이 쓰지 않은 DISPATCHING 작업(owner는 있고 fence가 없음)은 복구 대상에 나오지 않는다") + void aDispatchingJobNoClaimWroteIsNotOfferedForRecovery() { + // given + // The shape a row left dispatching before the fence existed has: an owner and a lease expiry + // written by hand, and the fence column's default. + NotificationRequestRecord request = + NotificationRows.request(TENANT, "idem-" + UUID.randomUUID(), clock.instant()); + RecipientDeliveryRecord job = + NotificationRows.job( + request, + RecipientDeliveryState.DISPATCHING, + Optional.of(NOW), + Optional.empty(), + clock.instant()); + RecipientDeliveryRecord leasedByHand = + new RecipientDeliveryRecord( + job.id(), + job.notificationId(), + job.tenantId(), + job.recipientRef(), + job.locale(), + job.timeZone(), + job.routingPlan(), + job.routeCursor(), + job.state(), + job.submissionOutcome(), + job.deliveryOutcome(), + job.evidenceLevel(), + job.ambiguousAttemptExists(), + job.duplicateRisk(), + job.nextDispatchAt(), + job.expiresAt(), + Optional.of("legacy-worker"), + Optional.of(NOW.minus(Duration.ofHours(1))), + job.attemptCount(), + job.lastFailureCategory(), + job.createdAt(), + job.updatedAt()); + inTransaction(() -> requests().insert(request, List.of(leasedByHand))); + + // when + List expired = leases().expiredDispatching(10, Duration.ofMinutes(1)); + + // then + assertThat(expired) + .as("a lease the application cannot build would fail every recovery pass after it") + .isEmpty(); + } + private RecipientDeliveryId given( RecipientDeliveryState state, Instant dueAt, Instant expiresAt) { NotificationRequestRecord request = diff --git a/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml b/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml index 959da41d..1aaac56e 100644 --- a/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml +++ b/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml @@ -43,8 +43,9 @@ app: default-locale: ${APP_NOTIFICATION_PLATFORM_DEFAULT_LOCALE:ko-KR} dispatch: claim-batch-size: ${APP_NOTIFICATION_PLATFORM_CLAIM_BATCH_SIZE:50} - # The lease must outlast a provider call plus its timeout, or a live worker's delivery is - # claimed by a second one while the first is still waiting on the provider. + # The lease must outlast a provider call, or a live worker's delivery is claimed by a second + # one while the first is still waiting on the provider. Startup refuses a lease that is not + # longer than every enabled provider profile's timeout. lease-duration: ${APP_NOTIFICATION_PLATFORM_LEASE_DURATION:2m} poll-interval: ${APP_NOTIFICATION_PLATFORM_POLL_INTERVAL:1s} max-global-concurrency: ${APP_NOTIFICATION_PLATFORM_MAX_CONCURRENCY:64} From 4da5143906bdea2c08bb39ce44a8738b5a87f3d0 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 13 Nov 2026 13:17:33 +0900 Subject: [PATCH 2/4] =?UTF-8?q?fix(persistence)!:=20=ED=8A=B8=EB=9E=9C?= =?UTF-8?q?=EC=9E=AD=EC=85=98=20=EC=8B=9C=ED=95=9C=EA=B3=BC=20=EC=A0=95?= =?UTF-8?q?=EB=A6=AC=20=EA=B2=BD=EB=A1=9C=EB=A5=BC=20=EB=8B=AB=EB=8A=94?= =?UTF-8?q?=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 남은 트랜잭션·정리 결함 일곱. live event log는 지우는 곳이 없어 끝없이 자란다. app.lock.wait-time은 읽는 곳이 없다. 알림 scheduler는 poll마다 상태 쿼리 넷을 돌리고, projection 지연은 닫힌 event까지 읽는다. stateless runner 트랜잭션에 시한이 없다. tenant pool 예산은 새 pool의 크기를 모른 채 열어 연결 합계가 상한을 넘는다. runbook SQL 설명이 lane 넷 중 셋만 말한다. 고른 방법: LiveEventReaper(1000행씩, 트랜잭션 30초)와 공용 BatchedDeletes를 더한다. app.realtime.replay.*는 LiveEventReplaySettings가 엄격하게 묶고, 조립 설정이 SchedulingConfigurer로 reaper를 reaper-interval(기본 10분)마다 등록한다. LockSettings는 lease-ttl 하나만 받는다(모르는 키로 기동 실패). scheduler는 backlogDepth() 한 문장만 읽고, 그 WHERE는 claim과 같은 SQL 조각(RecipientClaimSql.CLAIMABLE_AT_NOW)이다. projection 지연은 V14 부분 인덱스로 읽는다(lock_timeout, V21 선례대로 CONCURRENTLY 아님). runner 기본 30초(SpringTransaction 쓰기 시한과 같음을 시험이 본다), tenant 예산은 계획된 pool 크기로 열기 전에 본다. 버린 대안: V14를 CONCURRENTLY로(Flyway 트랜잭션 안에서 못 돈다, 큰 DB는 안내대로 미리 만든다), reaper @Scheduled에 SpEL로 settings bean 이름 읽기(문자열 bean 조회는 검사가 못 본다). 위험·한계: LockSettings·TenantDataSourceRegistry·TenantPoolBudget의 공개 생성자가 바뀐다. V14는 큰 notification_provider_event에서 쓰기를 잠깐 막는다(안내 4행에 미리 만드는 법과 INVALID 인덱스 확인). backlog 수치는 창이 닫힌 delivery를 더는 세지 않는다. 검증: desktop RED(pr1-red3) 새 시험 실패 포함. GREEN은 PR 본문. Refs: RR9-01-03, RR9-01-07, RR9-01-09, RR9-01-10, RR9-01-14, RR9-01-25, RR9-01-26 BREAKING CHANGE: JpaLiveEventReplay.sweepExpired 삭제, app.realtime.replay 아래 모르는 키와 app.lock.wait-time은 기동 실패, HibernateStatelessSessionRunner 트랜잭션 30초 상한, TenantPoolBudget·TenantDataSourceRegistry 시그니처 변경. docs/upgrade/2026-11-round9-claim-fence.md --- docs/architecture/jpa-api-surface.txt | 4 +- docs/upgrade/2026-11-round9-claim-fence.md | 31 ++- .../dispatch/NotificationSchedulerWorker.java | 6 +- .../outbound/persistence-jpa/README.md | 16 ++ .../autoconfigure/JpaAdapterBeansConfig.java | 43 ++++- .../database/TenantDataSourceRegistry.java | 23 ++- .../database/TenantPoolBudget.java | 39 +++- .../HibernateStatelessSessionRunner.java | 47 ++++- .../liveevent/JpaLiveEventReplay.java | 10 - .../liveevent/LiveEventJpaRepository.java | 23 ++- .../liveevent/LiveEventReaper.java | 53 ++++++ .../liveevent/LiveEventReplaySettings.java | 55 ++++++ .../persistence/lock/LockSettings.java | 31 +-- .../module-defaults/persistence-jpa.yml | 10 +- .../JpaLiveEventReplayContractTest.java | 41 ++++ .../notification/NotificationFixtures.java | 3 +- .../ServingStateContractTest.java | 177 +++++++++++++++++- ...StatelessSessionRunnerIntegrationTest.java | 51 +++++ .../TenantPoolCapacityContractTest.java | 68 ++++--- .../PostgreSqlOptionalStreamLifecycle.java | 4 +- .../persistence/readiness/RunbookSql.java | 10 +- .../JpaLiveEventReplayActivationTest.java | 123 +++++++++++- .../ExperimentalFailClosedTest.java | 21 ++- .../stateless/StatelessRunnerTimeoutTest.java | 45 +++++ .../liveevent/LiveEventReaperTest.java | 53 ++++++ .../DistributedLockPersistenceConfigTest.java | 14 ++ .../resources/db/applied-migrations.sha256 | 1 + .../JdbcNotificationServingState.java | 88 ++++++--- ...notification_provider_event_open_index.sql | 26 +++ ...otificationServingStateOpenEventsTest.java | 42 +++++ .../observation/NotificationServingState.java | 2 +- .../NotificationServingStatePort.java | 18 ++ 32 files changed, 1038 insertions(+), 140 deletions(-) create mode 100644 src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaper.java create mode 100644 src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReplaySettings.java create mode 100644 src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java create mode 100644 src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java create mode 100644 src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql create mode 100644 src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java diff --git a/docs/architecture/jpa-api-surface.txt b/docs/architecture/jpa-api-surface.txt index 0063f347..06fc95b3 100644 --- a/docs/architecture/jpa-api-surface.txt +++ b/docs/architecture/jpa-api-surface.txt @@ -5,7 +5,7 @@ # root yet. # Update only after review with: # ./gradlew :adapter:outbound:persistence-jpa:updateJpaApiSurface -PapproveJpaApiSurfaceChange -# types: 166 +# types: 168 dev.caskeleton.adapter.outbound.persistence.api.PersistenceOperationName dev.caskeleton.adapter.outbound.persistence.api.capability.CapabilitySupport dev.caskeleton.adapter.outbound.persistence.api.capability.JpaCapability @@ -104,6 +104,8 @@ dev.caskeleton.adapter.outbound.persistence.idempotency.postgresql.IdempotencySc dev.caskeleton.adapter.outbound.persistence.idempotency.postgresql.PostgreSqlIdempotencyStore dev.caskeleton.adapter.outbound.persistence.liveevent.JpaLiveEventReplay dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventJpaRepository +dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReaper +dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReplaySettings dev.caskeleton.adapter.outbound.persistence.liveevent.entity.LiveEventEntity dev.caskeleton.adapter.outbound.persistence.lock.DistributedLockPersistenceConfig dev.caskeleton.adapter.outbound.persistence.lock.LockRegistryDistributedLock diff --git a/docs/upgrade/2026-11-round9-claim-fence.md b/docs/upgrade/2026-11-round9-claim-fence.md index 99ae53f9..940011b1 100644 --- a/docs/upgrade/2026-11-round9-claim-fence.md +++ b/docs/upgrade/2026-11-round9-claim-fence.md @@ -1,13 +1,14 @@ # 업그레이드 안내 — 2026-11 claim·fence·트랜잭션 시한·정리 -- 범위: durable operation·알림 수신자·fileserver cleanup·멱등성 기록의 lease, fence, 조건부 쓰기, 정리를 고친 - 변경. 포트 시그니처와 기동 조건이 바뀐 곳이 있다. 아래 표가 그 목록이다. -- 근거: 2026-11-12 전체 리뷰(`docs/reviews/2026-11-12-round9-review/`)의 claim·fence(B8) 발견 +- 범위: durable operation·알림 수신자·fileserver cleanup·멱등성 기록·live event·분산 락·stateless runner의 lease, + fence, 조건부 쓰기, 정리, 시간 상한을 고친 변경. 포트 시그니처와 기동 조건이 바뀐 곳이 있다. 아래 표가 그 목록이다. +- 근거: 2026-11-12 전체 리뷰(`docs/reviews/2026-11-12-round9-review/`)의 claim·fence(B8)와 트랜잭션 시한·정리(B9) 발견 ## 1. 누가 읽나 이 템플릿을 `upstream` remote로 두고 merge해 받는 파생 서비스가 읽는다. 특히 durable operation worker를 직접 쓴 서비스, -알림 recipient store를 직접 구현한 서비스, 알림 provider timeout을 설정한 서비스다. +알림 recipient store를 직접 구현한 서비스, `app.lock.wait-time`이나 알림 provider timeout을 설정한 서비스, experimental +tenant pool 예산을 쓰는 서비스다. ## 2. 할 일 @@ -16,17 +17,22 @@ | 1 | durable operation worker는 `claimNext`가 돌려주는 `DurableOperationClaim`을 들고 있다가 `heartbeat(claim, …)`·`reportProgress(claim, …)`·`succeed(claim, …)`·`fail(claim, …)`에 넘긴다. 예전에는 `(operationId, workerId, …)`였다 | 컴파일이 실패한다 | durable operation worker를 쓴 서비스 | | 2 | `DurableOperationStorePort`를 직접 구현했으면 claim마다 오르는 fence를 저장하고, 네 쓰기가 owner와 fence를 함께 확인하게 한다. `DurableOperationStoreContract`의 새 사례(같은 worker id가 다시 claim한 뒤 앞 실행의 쓰기는 거절)로 확인한다 | 재시작한 worker나 한 프로세스의 두 실행이 같은 id를 쓰면 앞 실행의 결과가 넘겨받은 실행 위에 기록된다 | 자기 durable operation store를 둔 서비스 | | 3 | persistence-jpa의 `V23__durable_operation_lease_fence.sql`(`durable_operation.lease_fence BIGINT NOT NULL DEFAULT 0`)이 기본 stream에 더해진다. 같은 번호의 서비스 마이그레이션이 있으면 번호를 옮긴다. `durable_operation`을 건드린 트랜잭션이 열려 있으면 5초 뒤 실패하고 V22에 남는다(다시 배포) | Flyway가 번호 충돌로 기동하지 않는다 | persistence-jpa를 쓰는 모든 서비스 | -| 4 | `RecipientLeaseStorePort.expiredDispatching`이 `List`(버려질 때의 owner·fence·만료 시각)를 돌려주고, `RecipientDeliveryStorePort`에 `requeueAbandoned(lease, nextDispatchAt)`가 생겼다. 직접 구현했으면 상태가 `DISPATCHING`이고 owner·fence가 같고 lease가 늘지 않았을 때만 `READY_TO_DISPATCH`로 바꾼다(`RecipientDeliveryStoreContract` 새 사례). owner가 없거나 fence가 0인 `DISPATCHING` 행(notification V4 전에 남은 행)은 복구가 더는 고르지 않는다. 그런 행은 `SELECT id FROM notification_recipient_delivery WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 찾는다. 배포 전에 `UPDATE notification_recipient_delivery SET lease_owner = COALESCE(lease_owner, 'pre-v4'), lease_fence = lease_fence + 1, lease_until = now() - interval '1 day', version = version + 1, updated_at = now() WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 owner를 채우고 fence를 올려 복구 패스에 넘긴다. 복구는 attempt를 보고 정한다: attempt가 없으면 다시 보낼 차례로 돌리고, 끝나지 않은 attempt가 있으면 맞추기로 넘긴다(그 뒤는 `docs/notification/configuration-reference.md` 「발송」의 lease 문단). 손으로 `READY_TO_DISPATCH`로 바꾸지 않는다. 발송 중에 멈춘 행이면 같은 알림이 한 번 더 나간다. owner가 없는 행은 stuck lease 지표에도 잡히지 않는다 | 컴파일이 실패한다. 조건 없이 바꾸면 끝났거나 다시 claim된 delivery가 다시 발송된다 | 알림 store를 직접 구현한 서비스 | -| 5 | 알림 provider profile의 `timeout`이 `app.notification.platform.dispatch.lease-duration`보다 짧은지 확인한다. 짧은 것은 필요조건일 뿐이다: lease는 claim 때 시작해서 attempt 기록·렌더링·provider 허가 대기도 lease를 쓴다. lease는 timeout보다 넉넉히 길게 둔다. 이 timeout이 provider 호출 전체를 묶는 것은 SES(API 호출 시간 상한)다. SMTP는 `spring.mail` socket timeout이 명령마다 묶을 뿐이라 이 검사만으로는 호출 전체가 lease 안에 끝난다고 말할 수 없다 | 짧지 않으면 기동이 실패한다(`notification provider profile '…': timeout (…) must be shorter than …lease-duration`). 전에는 provider 호출 하나가 lease보다 길어도 기동했다. 그때 생기는 일은 `docs/notification/configuration-reference.md` 「발송」의 lease 문단이다 | 알림 capability를 켠 서비스 | -| 6 | `IdempotencyReaper`를 직접 만들던 코드는 `new IdempotencyReaper(repository)`로 바꾼다(`Clock` 인자 삭제). 만료 비교는 DB 시계(`now()`)이고 1000행씩 지운다 | 컴파일이 실패한다 | reaper를 직접 조립한 서비스 | -| 7 | fileserver cleanup 묶음의 `max-items`를 정할 때, 한 묶음이 claim lease(10분)의 첫 8분 안에 시작할 수 있는 만큼만 처리된다는 것을 안다. 나머지는 시도하지 않은 채 queue에 돌아가 다음 묶음이 다시 claim한다. 시도 횟수는 그대로다(바이트 예산이 바닥나 돌려준 항목도 이제 같다. 전에는 시도 한 번으로 세어 8번째에 보류됐다). 직접 구현한 `CleanupQueuePort`는 새 연산 `release`를 구현한다(`CleanupQueuePortContract` 새 사례) | 느린 저장소에서 큰 `max-items`는 그 사유로 도는 항목이 늘어난다. 전에는 lease가 끝난 뒤에도 남은 항목을 처리해 다른 worker와 겹쳤다 | fileserver capability를 켠 서비스 | +| 4 | 알림 stream에 `V14__notification_provider_event_open_index.sql`(부분 인덱스 `ix_notification_provider_event_open`)이 더해진다. `notification_provider_event`가 이미 큰 운영 DB는 배포 전에 같은 인덱스를 `CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_notification_provider_event_open ON notification_provider_event (received_at) WHERE projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL;`로 먼저 만든다. 미리 만든 인덱스가 실패해 남았으면(`SELECT indisvalid FROM pg_index WHERE indexrelid = 'ix_notification_provider_event_open'::regclass`가 false) `DROP INDEX CONCURRENTLY ix_notification_provider_event_open` 뒤 다시 만든다. `IF NOT EXISTS`는 쓸 수 없는 인덱스도 있는 것으로 보고 넘어간다 | 마이그레이션이 인덱스를 만드는 동안 provider callback 저장이 기다린다(provider가 callback을 다시 보낸다) | 알림 capability를 켠 서비스 | +| 5 | `RecipientLeaseStorePort.expiredDispatching`이 `List`(버려질 때의 owner·fence·만료 시각)를 돌려주고, `RecipientDeliveryStorePort`에 `requeueAbandoned(lease, nextDispatchAt)`가 생겼다. 직접 구현했으면 상태가 `DISPATCHING`이고 owner·fence가 같고 lease가 늘지 않았을 때만 `READY_TO_DISPATCH`로 바꾼다(`RecipientDeliveryStoreContract` 새 사례). owner가 없거나 fence가 0인 `DISPATCHING` 행(notification V4 전에 남은 행)은 복구가 더는 고르지 않는다. 그런 행은 `SELECT id FROM notification_recipient_delivery WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 찾는다. 배포 전에 `UPDATE notification_recipient_delivery SET lease_owner = COALESCE(lease_owner, 'pre-v4'), lease_fence = lease_fence + 1, lease_until = now() - interval '1 day', version = version + 1, updated_at = now() WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 owner를 채우고 fence를 올려 복구 패스에 넘긴다. 복구는 attempt를 보고 정한다: attempt가 없으면 다시 보낼 차례로 돌리고, 끝나지 않은 attempt가 있으면 맞추기로 넘긴다(그 뒤는 `docs/notification/configuration-reference.md` 「발송」의 lease 문단). 손으로 `READY_TO_DISPATCH`로 바꾸지 않는다. 발송 중에 멈춘 행이면 같은 알림이 한 번 더 나간다. owner가 없는 행은 stuck lease 지표에도 잡히지 않는다 | 컴파일이 실패한다. 조건 없이 바꾸면 끝났거나 다시 claim된 delivery가 다시 발송된다 | 알림 store를 직접 구현한 서비스 | +| 6 | 알림 provider profile의 `timeout`이 `app.notification.platform.dispatch.lease-duration`보다 짧은지 확인한다. 짧은 것은 필요조건일 뿐이다: lease는 claim 때 시작해서 attempt 기록·렌더링·provider 허가 대기도 lease를 쓴다. lease는 timeout보다 넉넉히 길게 둔다. 이 timeout이 provider 호출 전체를 묶는 것은 SES(API 호출 시간 상한)다. SMTP는 `spring.mail` socket timeout이 명령마다 묶을 뿐이라 이 검사만으로는 호출 전체가 lease 안에 끝난다고 말할 수 없다 | 짧지 않으면 기동이 실패한다(`notification provider profile '…': timeout (…) must be shorter than …lease-duration`). 전에는 provider 호출 하나가 lease보다 길어도 기동했다. 그때 생기는 일은 `docs/notification/configuration-reference.md` 「발송」의 lease 문단이다 | 알림 capability를 켠 서비스 | +| 7 | `app.lock.wait-time`을 설정에서 지운다. 기다릴 시간은 `tryAcquire(key, waitTime, leaseTtl)`의 인자다 | `app.lock` 아래 모르는 키라 기동이 실패한다. 전에도 읽는 곳이 없었다 | `app.lock.wait-time`을 설정한 서비스 | +| 8 | `JpaLiveEventReplay.sweepExpired()`를 부르던 코드를 지운다. replay를 켜면 `LiveEventReaper`가 `app.realtime.replay.reaper-interval`(기본 `PT10M`)마다 `retained_until`이 지난 행을 1000행씩 지운다. `app.realtime.replay.*`는 이제 `LiveEventReplaySettings`가 엄격하게 묶는다: 그 아래 모르는 키, 0 이하의 `retention`·`reaper-interval`은 기동을 실패시킨다 | 컴파일이 실패한다. 잘못 쓴 키는 기동이 실패한다. 전에는 지우는 곳이 없어 `live_event_log`가 끝없이 자랐다 | `app.realtime.replay.enabled=true`인 서비스 | +| 9 | `IdempotencyReaper`를 직접 만들던 코드는 `new IdempotencyReaper(repository)`로 바꾼다(`Clock` 인자 삭제). 만료 비교는 DB 시계(`now()`)이고 1000행씩 지운다 | 컴파일이 실패한다 | reaper를 직접 조립한 서비스 | +| 10 | `HibernateStatelessSessionRunner`의 트랜잭션은 이제 30초 상한이 있다(`DEFAULT_TIMEOUT`). 더 긴 일이면 셋째 인자로 상한을 넘기거나 일을 나눈다 | 30초를 넘는 문장이 취소되고 일 전체가 롤백된다. 전에는 상한이 없었다 | stateless runner를 쓰는 서비스 | +| 11 | experimental `TenantPoolBudget.requireCapacity`·`hasCapacity`가 새 pool의 연결 수를 셋째 인자로 받는다. pool 수만 볼 때는 `requirePoolSlot(openPools)`를 쓴다. `TenantDataSourceRegistry`의 둘째 인자는 이제 열린 pool의 크기를 재는 함수가 아니라 tenant의 계획된 pool 크기(`ToIntFunction`)다. 예산은 pool을 열기 전에 그 크기로 본다 | 컴파일이 실패한다. 전에는 새 pool 크기를 모른 채 열어 연결 합계가 상한을 넘었다 | experimental tenant pool을 쓰는 서비스 | +| 12 | fileserver cleanup 묶음의 `max-items`를 정할 때, 한 묶음이 claim lease(10분)의 첫 8분 안에 시작할 수 있는 만큼만 처리된다는 것을 안다. 나머지는 시도하지 않은 채 queue에 돌아가 다음 묶음이 다시 claim한다. 시도 횟수는 그대로다(바이트 예산이 바닥나 돌려준 항목도 이제 같다. 전에는 시도 한 번으로 세어 8번째에 보류됐다). 직접 구현한 `CleanupQueuePort`는 새 연산 `release`를 구현한다(`CleanupQueuePortContract` 새 사례) | 느린 저장소에서 큰 `max-items`는 그 사유로 도는 항목이 늘어난다. 전에는 lease가 끝난 뒤에도 남은 항목을 처리해 다른 worker와 겹쳤다 | fileserver capability를 켠 서비스 | ## 3. 바뀐 동작 (따라 할 일 없음) -- reaper 둘(outbox·멱등성)의 트랜잭션은 `SpringTransaction.DEFAULT_TIMEOUT_SECONDS`(30초)로 묶인다. +- reaper 셋(outbox·멱등성·live event)의 트랜잭션은 `SpringTransaction.DEFAULT_TIMEOUT_SECONDS`(30초)로 묶인다. `app.persistence-jpa.transaction.*`를 바꿔도 reaper 시한은 그대로다. 한 트랜잭션이 1000행 한 묶음만 지운다. - 저장소가 행에서 만드는 `DurableOperationClaim`·`OperationLease`·`RecipientLease`·`OutboxLease`·`ClaimedOutboxEvent`· - `ReconciliationJob`·`AdminOperationClaim`이 값을 거절할 때 `IllegalStateException`을 던진다 + `ReconciliationJob`·`AdminOperationClaim`과 stateless runner의 timeout이 값을 거절할 때 `IllegalStateException`을 던진다 (전에는 `IllegalArgumentException`). 요청에서 오는 값이 아니라 서버 결함이다. 이 IAE를 잡던 코드는 ISE를 잡는다. - 알림 lease 회복은 버려질 때의 lease 그대로일 때만 delivery를 다시 넣는다. 다른 인스턴스가 이미 다시 넣었거나 holder가 끝냈으면 아무것도 쓰지 않고 처리 수에 세지 않는다. @@ -36,3 +42,8 @@ - 멱등성 reaper는 다른 트랜잭션이 쥔 기록을 건너뛰고, 바깥 `DELETE`에서도 만료를 다시 본다. 같은 키 요청이 만료된 기록을 되살리는 동안 돈 reap이 그 claim을 지우지 않는다. - fileserver recovery 대기열 enqueue는 `INSERT … ON CONFLICT` 한 문장이다. 같은 파일을 동시에 넣어도 25P02로 실패하지 않는다. +- 알림 scheduler는 poll마다 backlog 한 문장만 읽는다(전에는 넷). health snapshot의 projection 지연은 열린 provider event만 + 읽는다. +- 알림 backlog(`backlogDepth`, scheduler의 queue-depth gauge, health 임계값)는 claim이 지금 가져갈 delivery만 센다. claim과 + 같은 SQL 조각이다. 전에는 창이 닫힌(`expires_at`이 지난) delivery도 셌고, lease가 지금 막 끝나는 행(`lease_until = now`)도 + 셌다. 만료가 쌓인 플랫폼은 배포 직후 backlog 수치가 내려간다. 보낼 일이 줄어든 것이 아니라 보낼 수 없는 행을 빼고 센 것이다. diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java index d761c923..f54fa001 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java @@ -81,10 +81,10 @@ public final class NotificationSchedulerWorker implements AutoCloseable { // by the claim batch size — so a queue of ten and a queue of ten million published the same // number, and the one metric named "queue depth" was the one that could not show a queue // growing. + // The depth alone: the full serving snapshot is several statements and is the health + // endpoint's, not something to run on every poll. metrics.gauge( - NotificationMetricName.QUEUE_DEPTH, - Map.of(), - (double) servingState.currentState().backlogDepth()); + NotificationMetricName.QUEUE_DEPTH, Map.of(), (double) servingState.backlogDepth()); for (RecipientLease lease : claimed) { globalConcurrency.acquireUninterruptibly(); diff --git a/src/adapter/outbound/persistence-jpa/README.md b/src/adapter/outbound/persistence-jpa/README.md index 9269264e..299beed7 100644 --- a/src/adapter/outbound/persistence-jpa/README.md +++ b/src/adapter/outbound/persistence-jpa/README.md @@ -389,6 +389,22 @@ PUBLISHED backlog 를 한 transaction 으로 지우면 그 사이 모든 row loc cadence 측정 필요). retention 한 값은 reaper-local 이라 `@Value` 로 받지만, canonical 6-property 문서는 app-bootstrap `OutboxSettings` / `application.yml` 에 있다. +## liveevent — `JpaLiveEventReplay`와 `LiveEventReaper` + +`app.realtime.replay.enabled=true`이고 vendor가 PostgreSQL이면 replay store와 reaper가 함께 조립된다. 설정은 +`LiveEventReplaySettings`가 `app.realtime.replay.*`를 엄격하게 묶는다. 모르는 키는 기동을 실패시킨다. + +- `retention`: 한 event를 다시 받을 수 있는 시간. 기본값이 없다. 모듈 기본값이 `APP_REALTIME_REPLAY_RETENTION`(1시간)으로 + 채운다. 0이나 음수는 기동 실패다. +- `reaper-interval`: `LiveEventReaper`가 retention이 지난 행을 지우는 간격(기본 10분, 모듈 기본값이 같은 값을 적는다). 0이나 + 음수는 기동 실패다. reaper 자체에는 일정이 없다. 조립 설정이 이 값으로 `reap()`을 등록한다(첫 실행도 한 간격 뒤). + reaper를 직접 만드는 서비스는 일정도 직접 건다. + +reaper는 `BatchedDeletes`로 1,000행씩, 배치마다 따로 commit하고, 배치가 덜 차면 멈춘다. 배치 문장은 `SKIP LOCKED`라 여러 +replica가 같이 돌아도 서로 기다리지 않는다. 트랜잭션 시한은 `SpringTransaction.DEFAULT_TIMEOUT_SECONDS`(30초)다. replay 읽기는 +`retained_until`로 지난 행을 이미 거르므로, reaper가 늦어도 지난 event를 내주지는 않는다. reaper가 없으면 테이블이 끝없이 +자란다(2026-11-13 전에는 지우는 곳이 없었다). + ## operation — `JpaDurableOperationStore` ### claim과 fence (`V23`) diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java index 193187e5..438f0978 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java @@ -8,6 +8,8 @@ import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.IdempotencyRe import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.IdempotencyRecordJpaRepository; import dev.caskeleton.adapter.outbound.persistence.liveevent.JpaLiveEventReplay; import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventJpaRepository; +import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReaper; +import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReplaySettings; import dev.caskeleton.adapter.outbound.persistence.lock.DistributedLockPersistenceConfig; import dev.caskeleton.adapter.outbound.persistence.operation.DurableOperationJpaRepository; import dev.caskeleton.adapter.outbound.persistence.operation.JpaDurableOperationStore; @@ -26,7 +28,11 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.core.env.Environment; import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.SchedulingConfigurer; +import org.springframework.scheduling.config.FixedDelayTask; import org.springframework.transaction.PlatformTransactionManager; /** @@ -120,11 +126,15 @@ class JpaAdapterBeansConfig { } /** - * Retained live-event history: on when replay is switched on, and only on PostgreSQL (the - * position allocator is an {@code INSERT … ON CONFLICT … RETURNING}). The retention has no - * default on purpose: a replay window nobody chose is one nobody sized. + * Retained live-event history and the reaper that removes it past retention: on when replay is + * switched on, and only on PostgreSQL (the position allocator is an {@code INSERT … ON CONFLICT … + * RETURNING}). The settings are {@link LiveEventReplaySettings}, bound strictly: the retention is + * one hour unless the deployment sets {@code APP_REALTIME_REPLAY_RETENTION} (the module + * defaults), the reaper runs every {@code reaper-interval}. Scheduling is enabled here because + * the reaper is part of the store, as for the idempotency reaper. */ @Configuration(proxyBeanMethods = false) + @EnableScheduling @ConditionalOnProperty(prefix = "app.realtime.replay", name = "enabled", havingValue = "true") @ConditionalOnProperty( prefix = "app.persistence", @@ -133,13 +143,36 @@ class JpaAdapterBeansConfig { matchIfMissing = true) static class LiveEventReplay { + @Bean + LiveEventReplaySettings liveEventReplaySettings(Environment environment) { + return LiveEventReplaySettings.bind(environment); + } + @Bean JpaLiveEventReplay jpaLiveEventReplay( LiveEventJpaRepository repository, JdbcOperations jdbc, Clock clock, - @Value("${app.realtime.replay.retention}") Duration retention) { - return new JpaLiveEventReplay(repository, jdbc, clock, retention); + LiveEventReplaySettings settings) { + return new JpaLiveEventReplay(repository, jdbc, clock, settings.retention()); + } + + @Bean + LiveEventReaper liveEventReaper(LiveEventJpaRepository repository, Clock clock) { + return new LiveEventReaper(repository, clock); + } + + /** + * Runs the reaper every bound {@code reaper-interval}, first one interval after startup. The + * interval is handed over as the settings' value, so it has one default and a zero has already + * failed startup in the settings rather than scheduling a loop. + */ + @Bean + SchedulingConfigurer liveEventReaperSchedule( + LiveEventReaper reaper, LiveEventReplaySettings settings) { + Duration interval = settings.reaperInterval(); + return registrar -> + registrar.addFixedDelayTask(new FixedDelayTask(reaper::reap, interval, interval)); } } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantDataSourceRegistry.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantDataSourceRegistry.java index 90737a29..2551c93c 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantDataSourceRegistry.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantDataSourceRegistry.java @@ -23,26 +23,32 @@ import javax.sql.DataSource; public final class TenantDataSourceRegistry implements AutoCloseable { private final Map pools = new LinkedHashMap<>(); + private final Map poolSizes = new LinkedHashMap<>(); private final Function poolFactory; - private final ToIntFunction poolSize; + private final ToIntFunction plannedPoolSize; private final TenantPoolBudget budget; /** * @param poolFactory opens a pool for a tenant from its secret-backed connection profile - * @param poolSize reports a pool's maximum connection count, for the global budget + * @param plannedPoolSize the maximum connection count a tenant's pool will have, read from its + * connection profile before the pool is opened, for the global budget */ public TenantDataSourceRegistry( Function poolFactory, - ToIntFunction poolSize, + ToIntFunction plannedPoolSize, TenantPoolBudget budget) { this.poolFactory = Objects.requireNonNull(poolFactory, "poolFactory"); - this.poolSize = Objects.requireNonNull(poolSize, "poolSize"); + this.plannedPoolSize = Objects.requireNonNull(plannedPoolSize, "plannedPoolSize"); this.budget = Objects.requireNonNull(budget, "budget"); } /** * The tenant's pool, opening it if the budget allows. * + *

The budget is checked against the pool's planned size before the pool is opened: a pool + * opens its first connections as it is built, so one checked afterwards had already taken the + * server past the ceiling, if only until it was closed again. + * * @throws IllegalStateException when the global pool budget is exhausted */ public synchronized DataSource require(TenantId tenant) { @@ -51,9 +57,12 @@ public final class TenantDataSourceRegistry implements AutoCloseable { if (existing != null) { return existing; } - budget.requireCapacity(pools.size(), allocatedConnections()); + budget.requirePoolSlot(pools.size()); + int planned = plannedPoolSize.applyAsInt(tenant); + budget.requireCapacity(pools.size(), allocatedConnections(), planned); DataSource opened = poolFactory.apply(tenant); pools.put(tenant, opened); + poolSizes.put(tenant, planned); return opened; } @@ -72,6 +81,7 @@ public final class TenantDataSourceRegistry implements AutoCloseable { */ public synchronized void evict(TenantId tenant) { DataSource removed = pools.remove(tenant); + poolSizes.remove(tenant); if (removed instanceof AutoCloseable closeable) { TenantDataSourceLifecycle.drainAndClose(closeable); } @@ -84,7 +94,7 @@ public final class TenantDataSourceRegistry implements AutoCloseable { /** How many connections all open pools may hold between them. */ public synchronized int allocatedConnections() { - return pools.values().stream().mapToInt(poolSize).sum(); + return poolSizes.values().stream().mapToInt(Integer::intValue).sum(); } /** The tenants with an open pool. */ @@ -99,5 +109,6 @@ public final class TenantDataSourceRegistry implements AutoCloseable { .map(AutoCloseable.class::cast) .forEach(TenantDataSourceLifecycle::drainAndClose); pools.clear(); + poolSizes.clear(); } } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantPoolBudget.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantPoolBudget.java index b3abfb1d..c26fa39d 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantPoolBudget.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantPoolBudget.java @@ -23,19 +23,44 @@ public record TenantPoolBudget(int maxOpenPools, int maxConnectionsAcrossPools) } /** - * Fails when another tenant pool would exceed either ceiling. + * Fails when another tenant pool would exceed the pool count, before that pool is opened. * - * @throws IllegalStateException when the budget is exhausted + * @param openPools how many pools are open now + * @throws IllegalStateException when no other pool fits */ - public void requireCapacity(int openPools, int allocatedConnections) { - if (openPools >= maxOpenPools || allocatedConnections >= maxConnectionsAcrossPools) { + public void requirePoolSlot(int openPools) { + if (openPools >= maxOpenPools) { throw new IllegalStateException("tenant pool budget exhausted"); } } - /** Whether another tenant pool fits inside both ceilings. */ - public boolean hasCapacity(int openPools, int allocatedConnections) { - return openPools < maxOpenPools && allocatedConnections < maxConnectionsAcrossPools; + /** + * Fails when a new tenant pool of {@code newPoolConnections} would exceed either ceiling. + * + *

The new pool's own size is part of the question. Checking only that the pools already open + * stay under the ceiling let a ten-connection pool open with one connection to spare, and the + * total then stood nine above the ceiling the budget exists to keep (round9 review 2026-11-12, + * RR9-01-26). + * + * @param openPools how many pools are open now, the new one not included + * @param allocatedConnections how many connections the open pools may hold between them + * @param newPoolConnections how many the new pool may hold + * @throws IllegalStateException when the budget cannot take the new pool + */ + public void requireCapacity(int openPools, int allocatedConnections, int newPoolConnections) { + if (!hasCapacity(openPools, allocatedConnections, newPoolConnections)) { + throw new IllegalStateException("tenant pool budget exhausted"); + } + } + + /** Whether a new tenant pool of {@code newPoolConnections} fits inside both ceilings. */ + public boolean hasCapacity(int openPools, int allocatedConnections, int newPoolConnections) { + if (newPoolConnections < 1) { + // The size is the server's own configuration, not the caller's input: a state error. + throw new IllegalStateException("a tenant pool holds at least one connection"); + } + return openPools < maxOpenPools + && (long) allocatedConnections + newPoolConnections <= maxConnectionsAcrossPools; } /** The maximum number of tenants that can be open at once. */ diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/HibernateStatelessSessionRunner.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/HibernateStatelessSessionRunner.java index 00f8aa30..d80927af 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/HibernateStatelessSessionRunner.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/HibernateStatelessSessionRunner.java @@ -1,5 +1,6 @@ package dev.caskeleton.adapter.outbound.persistence.hibernate.stateless; +import java.time.Duration; import java.util.Objects; import java.util.Set; import java.util.function.Function; @@ -19,23 +20,63 @@ import org.hibernate.Transaction; * *

Only registered work names may run, and each carries a declared row cap. An admin bulk path * without a cap is how a maintenance job becomes an outage. + * + *

The transaction has the same time bound as the application's write transactions (30 seconds + * unless the constructor is given another, {@code docs/work/reference/decisions.md} 2.5). The row + * cap can only be checked once the work reports what it touched, so without a bound a runaway + * statement held its locks for as long as it ran before the cap could roll it back. Hibernate hands + * every statement of the work the time left, and the database cancels the one that runs past it + * (round9 review 2026-11-12, RR9-01-25). */ public final class HibernateStatelessSessionRunner { + /** + * The default write-transaction bound, which this runner's transactions share: {@code + * SpringTransaction.DEFAULT_TIMEOUT_SECONDS}. Restated rather than read from it, because the + * {@code hibernate} package does not depend on {@code transaction} (JpaModuleBoundaryTest); + * {@code StatelessRunnerTimeoutTest} holds the two equal. A deployment that moves its write bound + * passes the same value to the three-argument constructor. + */ + public static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30); + private final SessionFactory sessionFactory; private final Set registeredWork; + private final int timeoutSeconds; + /** A runner whose transactions are bounded by {@link #DEFAULT_TIMEOUT}. */ public HibernateStatelessSessionRunner( SessionFactory sessionFactory, Set registeredWork) { + this(sessionFactory, registeredWork, DEFAULT_TIMEOUT); + } + + /** + * A runner whose transactions are bounded by {@code timeout}. + * + * @param timeout a whole, positive number of seconds; Hibernate's transaction timeout has no + * finer unit, so a fraction would be dropped without a word + */ + public HibernateStatelessSessionRunner( + SessionFactory sessionFactory, Set registeredWork, Duration timeout) { this.sessionFactory = Objects.requireNonNull(sessionFactory, "sessionFactory"); this.registeredWork = Set.copyOf(Objects.requireNonNull(registeredWork, "registeredWork")); + Objects.requireNonNull(timeout, "timeout"); + if (timeout.isNegative() + || timeout.isZero() + || timeout.toNanosPart() != 0 + || timeout.toSeconds() > Integer.MAX_VALUE) { + throw new IllegalStateException( + "a stateless transaction timeout must be a positive whole number of seconds, was " + + timeout); + } + this.timeoutSeconds = Math.toIntExact(timeout.toSeconds()); } /** * Runs {@code work} in its own stateless session and transaction. * *

The work reports how many rows it touched, and exceeding {@code maxRows} rolls the - * transaction back. + * transaction back. A statement still running when the transaction's time is up is cancelled, and + * the failure rolls the transaction back. * * @param maxRows the row cap for this unit of work * @throws StatelessRowCapExceededException when the work touched more, after rolling back @@ -54,7 +95,9 @@ public final class HibernateStatelessSessionRunner { } try (StatelessSession session = sessionFactory.openStatelessSession()) { - Transaction transaction = session.beginTransaction(); + Transaction transaction = session.getTransaction(); + transaction.setTimeout(timeoutSeconds); + transaction.begin(); try { StatelessWorkResult result = work.apply(session); if (result.affectedRows() > maxRows) { diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplay.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplay.java index 26e345af..d2762a63 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplay.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplay.java @@ -134,16 +134,6 @@ public final class JpaLiveEventReplay implements LiveEventReplayPort { .toList(); } - /** - * Removes events past their retention. - * - * @return how many went - */ - public int sweepExpired() { - requireCallerTransaction("sweepExpired"); - return repository.sweepExpired(clock.instant()); - } - private static void requireCallerTransaction(String operation) { CallerTransaction.require("live-event " + operation); } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventJpaRepository.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventJpaRepository.java index cf9ea4ce..9df87a3f 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventJpaRepository.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventJpaRepository.java @@ -1,6 +1,7 @@ package dev.caskeleton.adapter.outbound.persistence.liveevent; import dev.caskeleton.adapter.outbound.persistence.liveevent.entity.LiveEventEntity; +import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; import java.time.Instant; import java.util.List; import org.springframework.data.domain.Limit; @@ -8,6 +9,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; /** * Spring Data access for {@code live_event_log}. @@ -66,12 +68,27 @@ public interface LiveEventJpaRepository Long latestRetained(@Param("streamId") String streamId, @Param("now") Instant now); /** - * Removes expired rows. + * Deletes at most {@code limit} rows retained no later than {@code now}, in a transaction of its + * own unless the caller already has one; returns the count. + * + *

Bounded, and skipping rows another replica's reaper is deleting, for the reason the outbox + * purge is bounded: a backlog of expired history is one long transaction holding every row lock + * otherwise. {@code retained_until} is written from the application clock, so it is compared with + * the application clock. * * @param now rows retained no later than this are removed + * @param limit the most rows one statement removes * @return how many went */ + // Bounded by the default transaction bound: a statement stuck behind a lock gives the connection + // back instead of holding it until the pool is empty. + @Transactional(timeout = SpringTransaction.DEFAULT_TIMEOUT_SECONDS) @Modifying - @Query("delete from LiveEventEntity e where e.retainedUntil <= :now") - int sweepExpired(@Param("now") Instant now); + @Query( + value = + "DELETE FROM live_event_log WHERE (stream_id, position) IN (" + + "SELECT stream_id, position FROM live_event_log WHERE retained_until <= :now" + + " FOR UPDATE SKIP LOCKED LIMIT :limit)", + nativeQuery = true) + int deleteExpiredBatch(@Param("now") Instant now, @Param("limit") int limit); } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaper.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaper.java new file mode 100644 index 00000000..6e57b633 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaper.java @@ -0,0 +1,53 @@ +package dev.caskeleton.adapter.outbound.persistence.liveevent; + +import dev.caskeleton.adapter.outbound.persistence.transaction.BatchedDeletes; +import java.time.Clock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Periodically deletes {@code live_event_log} rows past their retention. + * + *

A replay read already filters on {@code retained_until}, so an expired row is never served; + * without this nothing ever removed one, and the table grew by every event ever published for as + * long as replay was on (round9 review 2026-11-12, RR9-01-03). It is assembled with the replay + * store, as the idempotency and outbox reapers are with theirs, rather than left to each service to + * schedule: a duty that every deployment must remember is one some deployment forgets. + * + *

Deletes in batches of {@link #BATCH_SIZE}, each committed on its own, until a batch comes back + * short. Every replica runs it; the batch skips rows another replica's batch holds. + * + *

It carries no schedule of its own. The replay configuration registers {@link #reap()} at the + * bound {@link LiveEventReplaySettings#reaperInterval()}, so the interval reaches the schedule as a + * value rather than through a bean name an expression looks up. A deployment that builds a reaper + * itself schedules it too. + */ +public class LiveEventReaper { + + /** Rows per delete statement and transaction. */ + static final int BATCH_SIZE = BatchedDeletes.BATCH_SIZE; + + private static final Logger log = LoggerFactory.getLogger(LiveEventReaper.class); + + private final LiveEventJpaRepository repository; + private final Clock clock; + + public LiveEventReaper(LiveEventJpaRepository repository, Clock clock) { + this.repository = repository; + this.clock = clock; + } + + /** + * Deletes every row retained no later than now; returns the count. + * + *

Not transactional itself: each batch is its own transaction. + */ + public int reap() { + int purged = + BatchedDeletes.untilShort(limit -> repository.deleteExpiredBatch(clock.instant(), limit)); + if (purged > 0) { + log.debug("live event reaper purged {} expired row(s)", purged); + } + return purged; + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReplaySettings.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReplaySettings.java new file mode 100644 index 00000000..dd62e36b --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReplaySettings.java @@ -0,0 +1,55 @@ +package dev.caskeleton.adapter.outbound.persistence.liveevent; + +import dev.caskeleton.adapter.outbound.support.StrictSettingsBinder; +import java.time.Duration; +import org.springframework.core.env.Environment; + +/** + * The replay store's settings, bound strictly from {@code app.realtime.replay.*}: a misspelt key + * fails startup rather than leaving a default in force. + * + *

The reaper's interval was read only by the {@code @Scheduled} placeholder, with its default + * written twice and nothing to refuse a zero (round9 review 2026-11-12). It is bound here with the + * rest of the store's settings and read from this record by the reaper's schedule. + * + * @param enabled whether the replay store is assembled; the configuration's condition reads it + * @param retention how long an event stays replayable. No default: how long a reconnecting client + * may resume is the deployment's decision, made against what it is willing to store + * @param reaperInterval how often {@link LiveEventReaper} deletes rows past retention; 10 minutes + * unless set, which the module defaults mirror + */ +public record LiveEventReplaySettings( + boolean enabled, Duration retention, Duration reaperInterval) { + + static final String PREFIX = "app.realtime.replay"; + + private static final Duration DEFAULT_REAPER_INTERVAL = Duration.ofMinutes(10); + + public LiveEventReplaySettings { + if (retention == null) { + throw new IllegalArgumentException( + "app.realtime.replay.retention must be set when replay is enabled: how long a client may" + + " resume is the deployment's decision"); + } + if (retention.isNegative() || retention.isZero()) { + throw new IllegalArgumentException( + "app.realtime.replay.retention must be positive, was " + retention); + } + if (reaperInterval == null) { + reaperInterval = DEFAULT_REAPER_INTERVAL; + } else if (reaperInterval.isNegative() || reaperInterval.isZero()) { + throw new IllegalArgumentException( + "app.realtime.replay.reaper-interval must be positive, was " + reaperInterval); + } + } + + /** + * Binds {@link #PREFIX} strictly. + * + * @param environment the environment to bind from + * @return the bound settings + */ + public static LiveEventReplaySettings bind(Environment environment) { + return StrictSettingsBinder.bind(environment, PREFIX, LiveEventReplaySettings.class); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/lock/LockSettings.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/lock/LockSettings.java index 54cc10d6..5d19b24a 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/lock/LockSettings.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/lock/LockSettings.java @@ -5,41 +5,28 @@ import java.time.Duration; import org.springframework.core.env.Environment; /** - * Distributed-lock tuning knobs bound from {@code app.lock.*} (yaml-only defaults, not {@code - * APP_*} env keys). {@code waitTime} = max wait before timeout (default 3s); {@code leaseTtl} = max - * lock hold before auto-expiry (default 30s, must be ≥ {@code waitTime}). See README "lock". + * Distributed-lock tuning bound from {@code app.lock.*} (yaml-only defaults, not {@code APP_*} env + * keys). {@code leaseTtl} = the longest a lock may be held before the provider expires it (default + * 30s); a caller's {@code tryAcquire} may ask for that long or less. See README "lock". + * + *

There is no wait setting. How long to wait is every caller's own argument to {@code + * tryAcquire}; an {@code app.lock.wait-time} used to be bound here, checked against the lease, and + * read by nothing (round9 review 2026-11-12, RR9-01-07). It now fails startup as an unknown key, so + * a deployment that set it learns that it never did anything. */ -public record LockSettings(Duration waitTime, Duration leaseTtl) { +public record LockSettings(Duration leaseTtl) { static final String PREFIX = "app.lock"; - private static final Duration DEFAULT_WAIT_TIME = Duration.ofSeconds(3); private static final Duration DEFAULT_LEASE_TTL = Duration.ofSeconds(30); public LockSettings { - if (waitTime == null) { - waitTime = DEFAULT_WAIT_TIME; - } else if (waitTime.isNegative() || waitTime.isZero()) { - throw new IllegalArgumentException( - "app.lock.wait-time must be positive (> 0), was " + waitTime); - } if (leaseTtl == null) { leaseTtl = DEFAULT_LEASE_TTL; } else if (leaseTtl.isNegative() || leaseTtl.isZero()) { throw new IllegalArgumentException( "app.lock.lease-ttl must be positive (> 0), was " + leaseTtl); } - // Cross-field invariant: a lease shorter than the max wait is nonsensical. - // If the TTL fires before the waitTime elapses, a second holder could acquire - // the lock before the first holder's protected work is complete. - if (leaseTtl.compareTo(waitTime) < 0) { - throw new IllegalArgumentException( - "app.lock.lease-ttl (" - + leaseTtl - + ") must be >= app.lock.wait-time (" - + waitTime - + ") — a lease shorter than the max wait is nonsensical"); - } } /** diff --git a/src/adapter/outbound/persistence-jpa/src/main/resources/config/module-defaults/persistence-jpa.yml b/src/adapter/outbound/persistence-jpa/src/main/resources/config/module-defaults/persistence-jpa.yml index 9b9ff3ce..64a0eb39 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/resources/config/module-defaults/persistence-jpa.yml +++ b/src/adapter/outbound/persistence-jpa/src/main/resources/config/module-defaults/persistence-jpa.yml @@ -199,10 +199,10 @@ app: # true | false (whether to run Flyway migrations automatically at startup) migration-on-startup: ${APP_MIGRATION_ON_STARTUP:true} # feature-distributed-lock-contract D5 — distributed lock acquisition contract (try-lock + - # finite wait + lease TTL). Bound to LockSettings. Plain values, not APP_* env keys; the code - # defaults in LockSettings mirror these. + # finite wait + lease TTL). Bound to LockSettings. A plain value, not an APP_* env key; the code + # default in LockSettings mirrors it. The wait is each caller's tryAcquire argument, so there is + # no wait setting (app.lock.wait-time was read by nothing and is now an unknown key). lock: - wait-time: 3s lease-ttl: 30s realtime: replay: @@ -210,6 +210,10 @@ app: # events into the store and exposes replay/resume semantics. enabled: ${APP_REALTIME_REPLAY_ENABLED:false} retention: ${APP_REALTIME_REPLAY_RETENTION:1h} + # ISO-8601 duration — how often LiveEventReaper deletes rows past retention, 1 000 a batch. + # A plain value like the outbox reaper's interval. Bound strictly by LiveEventReplaySettings, + # whose default mirrors it; zero or negative fails startup. + reaper-interval: PT10M outbox: # feature-domain-event-outbox-contract I11 — the tuning values are literal defaults; # NO env placeholders (spec: 신규 env key 없음). Declared, validated and strictly bound by diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplayContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplayContractTest.java index b8b1b3cf..aa2e38ed 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplayContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplayContractTest.java @@ -1,5 +1,7 @@ package dev.caskeleton.adapter.outbound.persistence.liveevent; +import static org.assertj.core.api.Assertions.assertThat; + import dev.caskeleton.adapter.outbound.persistence.liveevent.entity.LiveEventEntity; import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; @@ -13,12 +15,17 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.time.Clock; import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; import java.util.Map; +import java.util.UUID; import javax.sql.DataSource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.springframework.context.annotation.AnnotationConfigApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -98,6 +105,40 @@ class JpaLiveEventReplayContractTest extends LiveEventReplayContract { return RETENTION; } + @Test + void theReaperRemovesRowsPastTheirRetentionAndKeepsTheRest() { + // given + String stream = "reaper-" + UUID.randomUUID(); + Instant start = Instant.parse("2026-11-03T00:00:00Z"); + appendAt(stream, "expired", start); + appendAt(stream, "retained", start.plus(RETENTION).minusSeconds(60)); + LiveEventReaper reaper = + new LiveEventReaper( + context.getBean(LiveEventJpaRepository.class), + Clock.fixed(start.plus(RETENTION).plusSeconds(1), ZoneOffset.UTC)); + + // when + var unused = reaper.reap(); + + // then + assertThat( + new JdbcTemplate(support.dataSource()) + .queryForList( + "SELECT payload FROM live_event_log WHERE stream_id = ?", String.class, stream)) + .containsExactly("retained"); + } + + private void appendAt(String streamId, String payload, Instant at) { + JpaLiveEventReplay writer = + new JpaLiveEventReplay( + context.getBean(LiveEventJpaRepository.class), + new JdbcTemplate(support.dataSource()), + Clock.fixed(at, ZoneOffset.UTC), + RETENTION); + var unused = + context.getBean(TransactionPort.class).inWrite(() -> writer.append(streamId, payload)); + } + private static void execute(String sql) throws SQLException { try (Connection connection = support.connection(); Statement statement = connection.createStatement()) { diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationFixtures.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationFixtures.java index d88d0217..12625a8f 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationFixtures.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationFixtures.java @@ -49,7 +49,8 @@ final class NotificationFixtures { migration("jpa/notification-platform", "V10__variables_payload_envelope_guard.sql"), migration("jpa/notification-platform", "V11__notification_reconciliation_claim_lease.sql"), migration("jpa/notification-platform", "V12__notification_suppression_target_unique.sql"), - migration("jpa/notification-platform", "V13__notification_request_acceptance.sql")); + migration("jpa/notification-platform", "V13__notification_request_acceptance.sql"), + migration("jpa/notification-platform", "V14__notification_provider_event_open_index.sql")); } /** diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/ServingStateContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/ServingStateContractTest.java index b10deb33..20154219 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/ServingStateContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/ServingStateContractTest.java @@ -8,6 +8,7 @@ import dev.caskeleton.application.notification.platform.observation.Notification import java.io.IOException; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; @@ -15,6 +16,9 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.UUID; import javax.sql.DataSource; import org.junit.jupiter.api.AfterAll; @@ -23,7 +27,12 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.PreparedStatementCreator; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.core.SqlProvider; import org.springframework.jdbc.datasource.SingleConnectionDataSource; /** @@ -100,6 +109,39 @@ class ServingStateContractTest { assertThat(measure().backlogDepth()).isZero(); } + @Test + void aDeliveryWhoseWindowClosedIsNotBacklog() throws SQLException { + // given + // Due, but past its expiry: the claim refuses it and the expiry sweep retires it unsent. + insertDelivery( + "READY_TO_DISPATCH", NOW.minus(Duration.ofMinutes(5)), null, null, NOW.minusSeconds(1)); + + // when + long depth = measure().backlogDepth(); + + // then + assertThat(depth).isZero(); + } + + @ParameterizedTest(name = "[{index}] lease_until = NOW에서 {0}초 → backlog {1}") + @CsvSource({"-1, 1", "0, 0"}) + @DisplayName("lease는 NOW 전에 끝났을 때만 끝난 것이다: claim과 같은 lease_until < now") + void aLeaseHasRunOutOnlyOnceItEndedBeforeNow(long offsetSeconds, long expectedDepth) + throws SQLException { + // given + insertDelivery( + "READY_TO_DISPATCH", + NOW.minus(Duration.ofMinutes(1)), + "worker-1", + NOW.plusSeconds(offsetSeconds)); + + // when + long depth = measure().backlogDepth(); + + // then + assertThat(depth).isEqualTo(expectedDepth); + } + @Test @DisplayName("a delivery held under a live lease is not backlog") void aDeliveryUnderALiveLeaseIsNotBacklog() throws SQLException { @@ -113,22 +155,24 @@ class ServingStateContractTest { } @Test - @DisplayName("a lease whose holder never came back is counted as stuck") void anExpiredLeaseIsStuck() throws SQLException { + // given insertDelivery( "DISPATCHING", NOW.minus(Duration.ofMinutes(20)), "worker-1", NOW.minus(Duration.ofMinutes(10))); + // when NotificationServingState state = measure(); - assertThat(state.stuckLeases()) - .as("a worker died mid-dispatch, so a provider call may have happened with no record") - .isEqualTo(1); - assertThat(state.backlogDepth()) - .as("an expired lease is claimable again, but this row is DISPATCHING, not due") - .isZero(); + // then + assertThat(state) + .as( + "a worker died mid-dispatch, so a provider call may have happened with no record; a" + + " dispatching row is never claimed, so it is not backlog") + .extracting(NotificationServingState::stuckLeases, NotificationServingState::backlogDepth) + .containsExactly(1L, 0L); } @Test @@ -162,6 +206,108 @@ class ServingStateContractTest { assertThat(state.oldestReconciliationAge()).isEqualTo(Duration.ofHours(2)); } + @Test + void theSchedulersBacklogReadIsOneStatement() throws SQLException { + // given + insertDelivery("READY_TO_DISPATCH", NOW.minus(Duration.ofMinutes(30)), null, null); + + // when + List outcome; + try (Connection connection = support.connection()) { + RecordingJdbc jdbc = new RecordingJdbc(new SingleConnectionDataSource(connection, true)); + long depth = new JdbcNotificationServingState(jdbc, CLOCK).backlogDepth(); + outcome = List.of(depth, jdbc.statements.size()); + } + + // then + assertThat(outcome) + .as("the scheduler asks this on every poll; the full snapshot is four statements") + .containsExactly(1L, 1); + } + + @Test + void theProjectionLagReadsOpenEventsThroughTheirIndex() throws SQLException { + // given + insertProviderEvent("PENDING", NOW.minus(Duration.ofMinutes(45)), false); + + // when + String plan; + try (Connection connection = support.connection()) { + RecordingJdbc jdbc = new RecordingJdbc(new SingleConnectionDataSource(connection, true)); + var unused = new JdbcNotificationServingState(jdbc, CLOCK).currentState(); + RecordedStatement lag = + jdbc.statements.stream() + .filter(statement -> statement.sql().contains("notification_provider_event")) + .findFirst() + .orElseThrow(); + plan = explainWithoutSequentialScans(connection, lag); + } + + // then + assertThat(plan) + .as("the event table is never purged; only the index over the open events bounds this read") + .contains("ix_notification_provider_event_open"); + } + + /** The plan PostgreSQL picks for {@code statement} when it may not scan a table sequentially. */ + private static String explainWithoutSequentialScans( + Connection connection, RecordedStatement statement) throws SQLException { + boolean autoCommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + try { + try (Statement off = connection.createStatement()) { + off.execute("SET LOCAL enable_seqscan = off"); + } + StringBuilder plan = new StringBuilder(); + try (PreparedStatement explain = connection.prepareStatement("EXPLAIN " + statement.sql())) { + for (int index = 0; index < statement.args().size(); index++) { + explain.setObject(index + 1, statement.args().get(index)); + } + try (ResultSet rows = explain.executeQuery()) { + while (rows.next()) { + plan.append(rows.getString(1)).append('\n'); + } + } + } + return plan.toString(); + } finally { + connection.rollback(); + connection.setAutoCommit(autoCommit); + } + } + + /** A statement the serving state ran, as it ran it. */ + private record RecordedStatement(String sql, List args) {} + + /** Plain JDBC that keeps the statements it was given, so a case reads the shipped SQL. */ + private static final class RecordingJdbc extends JdbcTemplate { + + private final List statements = new ArrayList<>(); + + private RecordingJdbc(DataSource dataSource) { + super(dataSource); + } + + @Override + public T queryForObject(String sql, RowMapper rowMapper, Object... args) { + statements.add(new RecordedStatement(sql, Arrays.asList(args))); + return super.queryForObject(sql, rowMapper, args); + } + + @Override + public T queryForObject(String sql, Class requiredType, Object... args) { + statements.add(new RecordedStatement(sql, Arrays.asList(args))); + return super.queryForObject(sql, requiredType, args); + } + + /** The named-parameter backlog read arrives here; its parameters stay inside the creator. */ + @Override + public List query(PreparedStatementCreator creator, RowMapper rowMapper) { + statements.add(new RecordedStatement(((SqlProvider) creator).getSql(), List.of())); + return super.query(creator, rowMapper); + } + } + private NotificationServingState measure() throws SQLException { // try-with-resources: the pool holds four connections, and a measurement that keeps one is a // measurement that starves the next test. @@ -174,6 +320,16 @@ class ServingStateContractTest { private void insertDelivery( String state, Instant nextDispatchAt, String leaseOwner, Instant leaseUntil) throws SQLException { + insertDelivery(state, nextDispatchAt, leaseOwner, leaseUntil, null); + } + + private void insertDelivery( + String state, + Instant nextDispatchAt, + String leaseOwner, + Instant leaseUntil, + Instant expiresAt) + throws SQLException { UUID requestId = UUID.randomUUID(); try (Connection connection = support.connection()) { try (PreparedStatement statement = @@ -197,17 +353,18 @@ class ServingStateContractTest { + " id, notification_id, tenant_id, recipient_ref, routing_plan_json," + " route_cursor, delivery_state, submission_outcome, delivery_outcome," + " evidence_level, attempt_count, next_dispatch_at, lease_owner, lease_until," - + " created_at, updated_at)" + + " expires_at, created_at, updated_at)" + " VALUES (?, ?, 'tenant-a', 'user-1', '[]'::jsonb, 0, ?, 'NOT_SUBMITTED'," - + " 'UNKNOWN', 'PLATFORM_QUEUED', 0, ?, ?, ?, ?, ?)")) { + + " 'UNKNOWN', 'PLATFORM_QUEUED', 0, ?, ?, ?, ?, ?, ?)")) { statement.setObject(1, UUID.randomUUID()); statement.setObject(2, requestId); statement.setString(3, state); statement.setTimestamp(4, Timestamp.from(nextDispatchAt)); statement.setString(5, leaseOwner); statement.setTimestamp(6, leaseUntil == null ? null : Timestamp.from(leaseUntil)); - statement.setTimestamp(7, Timestamp.from(NOW)); + statement.setTimestamp(7, expiresAt == null ? null : Timestamp.from(expiresAt)); statement.setTimestamp(8, Timestamp.from(NOW)); + statement.setTimestamp(9, Timestamp.from(NOW)); statement.executeUpdate(); } } diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/HibernateStatelessSessionRunnerIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/HibernateStatelessSessionRunnerIntegrationTest.java index 3ac505ad..282e467c 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/HibernateStatelessSessionRunnerIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/HibernateStatelessSessionRunnerIntegrationTest.java @@ -2,12 +2,16 @@ package dev.caskeleton.adapter.outbound.persistence.platform; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.persistence.hibernate.stateless.HibernateStatelessSessionRunner; import dev.caskeleton.adapter.outbound.persistence.hibernate.stateless.StatelessRowCapExceededException; import dev.caskeleton.adapter.outbound.persistence.hibernate.stateless.StatelessWorkName; import dev.caskeleton.adapter.outbound.persistence.hibernate.stateless.StatelessWorkResult; import dev.caskeleton.adapter.outbound.persistence.testkit.id.SequenceEntity; +import java.time.Duration; +import java.util.List; +import java.util.Locale; import java.util.Set; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -104,6 +108,53 @@ class HibernateStatelessSessionRunnerIntegrationTest { .hasMessageContaining("positive row cap"); } + @Test + void workStillRunningWhenTheTransactionTimeIsUpIsCancelledAndRolledBack() { + // given + var runner = + new HibernateStatelessSessionRunner( + jpa.sessionFactory(), Set.of(WORK), Duration.ofSeconds(1)); + long before = countRows(); + + // when + Throwable failure = + catchThrowable( + () -> + runner.execute( + WORK, + 10, + session -> { + session.insert(new SequenceEntity("timed-out")); + var unused = + session + .createNativeQuery("SELECT 1 FROM pg_sleep(5)", Integer.class) + .getSingleResult(); + return StatelessWorkResult.of(1, 1); + })); + + // then + assertThat(List.of(failure != null && causedByTimeout(failure), countRows())) + .as( + "the transaction's bound cancels the statement; nothing the work wrote is kept: %s", + failure) + .containsExactly(true, before); + } + + /** Whether the failure is the database or Hibernate stopping work that ran out of time. */ + private static boolean causedByTimeout(Throwable failure) { + for (Throwable current = failure; current != null; current = current.getCause()) { + String message = String.valueOf(current.getMessage()).toLowerCase(Locale.ROOT); + if (current instanceof org.hibernate.QueryTimeoutException + || current instanceof jakarta.persistence.QueryTimeoutException + || message.contains("canceling statement due to statement timeout") + || message.contains("canceling statement due to user request") + || message.contains("transaction timeout expired")) { + return true; + } + } + return false; + } + private static long countRows() { return jpa.inTransaction( entityManager -> diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/experimental/TenantPoolCapacityContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/experimental/TenantPoolCapacityContractTest.java index b2ef5043..4b53bf5e 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/experimental/TenantPoolCapacityContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/experimental/TenantPoolCapacityContractTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.platform.experimental; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; @@ -12,6 +13,7 @@ import dev.caskeleton.application.tenant.TenantId; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; import java.util.List; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -51,9 +53,9 @@ class TenantPoolCapacityContractTest { void budgetRefusesTheTenantThatWouldExceedTheCeiling() { var budget = new TenantPoolBudget(4, 8); - budget.requireCapacity(3, 6); - assertThat(budget.hasCapacity(4, 8)).isFalse(); - assertThatThrownBy(() -> budget.requireCapacity(4, 8)) + budget.requireCapacity(3, 6, 2); + assertThat(budget.hasCapacity(3, 7, 2)).isFalse(); + assertThatThrownBy(() -> budget.requireCapacity(3, 7, 2)) .isInstanceOf(IllegalStateException.class); } @@ -77,30 +79,50 @@ class TenantPoolCapacityContractTest { void openingPastTheBudgetIsRefusedBeforeAPoolIsCreated() { var budget = new TenantPoolBudget(2, 4); try (var registry = - new TenantDataSourceRegistry( - TenantPoolCapacityContractTest::poolFor, - TenantPoolCapacityContractTest::poolSizeOf, - budget)) { + new TenantDataSourceRegistry(this::poolFor, tenant -> POOL_SIZE_PER_TENANT, budget)) { registry.openTenants(2); assertThat(registry.openPools()).isEqualTo(2); assertThat(registry.allocatedConnections()).isEqualTo(4); assertThatThrownBy(() -> registry.openTenants(3)).isInstanceOf(IllegalStateException.class); - assertThat(registry.openPools()) - .as("a refused tenant must not leave a half-built pool behind") - .isEqualTo(2); + assertThat(List.of(registry.openPools(), opened.size())) + .as("a refused tenant must not leave a half-built pool behind, or have one built at all") + .containsExactly(2, 2); } } + @Test + void aTenantWhosePoolWouldPassTheConnectionCeilingIsRefusedBeforeItsPoolOpens() { + // given + // Two pools of two use four of five connections; the third pool would make six. + var budget = new TenantPoolBudget(4, 5); + List outcome; + try (var registry = + new TenantDataSourceRegistry(this::poolFor, tenant -> POOL_SIZE_PER_TENANT, budget)) { + registry.openTenants(2); + + // when + Throwable refused = catchThrowable(() -> registry.require(new TenantId("tenant-2"))); + outcome = + List.of( + refused instanceof IllegalStateException, + registry.openPools(), + registry.allocatedConnections(), + opened.size()); + } + + // then + assertThat(outcome) + .as("the third pool is refused on its planned size, so it never opens a connection") + .containsExactly(true, 2, 4, 2); + } + @Test @DisplayName("every opened tenant pool reaches the real server") void everyOpenedTenantPoolReachesTheServer() throws SQLException { var budget = new TenantPoolBudget(3, 6); try (var registry = - new TenantDataSourceRegistry( - TenantPoolCapacityContractTest::poolFor, - TenantPoolCapacityContractTest::poolSizeOf, - budget)) { + new TenantDataSourceRegistry(this::poolFor, tenant -> POOL_SIZE_PER_TENANT, budget)) { registry.openTenants(3); for (TenantId tenant : List.copyOf(registry.openTenantIds())) { @@ -118,10 +140,7 @@ class TenantPoolCapacityContractTest { void evictingATenantReturnsItsConnectionsToTheBudget() { var budget = new TenantPoolBudget(2, 4); try (var registry = - new TenantDataSourceRegistry( - TenantPoolCapacityContractTest::poolFor, - TenantPoolCapacityContractTest::poolSizeOf, - budget)) { + new TenantDataSourceRegistry(this::poolFor, tenant -> POOL_SIZE_PER_TENANT, budget)) { registry.openTenants(2); TenantId evicted = List.copyOf(registry.openTenantIds()).get(0); @@ -133,18 +152,19 @@ class TenantPoolCapacityContractTest { } } - private static javax.sql.DataSource poolFor(TenantId tenant) { + /** Every pool this test's factory built, so a test can see one was never opened. */ + private final List opened = new ArrayList<>(); + + private javax.sql.DataSource poolFor(TenantId tenant) { var config = new HikariConfig(); config.setJdbcUrl(support.jdbcUrl()); config.setUsername(support.username()); config.setPassword(support.password()); config.setMaximumPoolSize(POOL_SIZE_PER_TENANT); config.setPoolName("tenant-" + tenant.value()); - return new HikariDataSource(config); - } - - private static int poolSizeOf(javax.sql.DataSource pool) { - return ((HikariDataSource) pool).getMaximumPoolSize(); + HikariDataSource pool = new HikariDataSource(config); + opened.add(pool); + return pool; } private static int serverMaxConnections() throws SQLException { diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOptionalStreamLifecycle.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOptionalStreamLifecycle.java index f25dfaee..6a47fc8d 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOptionalStreamLifecycle.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOptionalStreamLifecycle.java @@ -104,12 +104,14 @@ final class PostgreSqlOptionalStreamLifecycle { // CONFLICT instead of a find-then-insert two webhooks could both win. // V13 records whether a stored request was accepted or dropped by a deduplication window, // so a retry with the same idempotency key gets the same receipt (review 05-23c). + // V14 indexes the provider events still waiting for something, so the projection lag reads + // them and not the whole history (round9 review 2026-11-12, RR9-01-14). // // This list is the stream's contract, not a note about its length: the lifecycle below // disables, re-enables and interrupts the stream and asserts the applied set is unchanged // each time. So a migration added to the stream belongs here, and the version that landed // without being added is why the lane failed the first time anybody ran it. - List.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13"), + List.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14"), List.of()); } diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/RunbookSql.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/RunbookSql.java index ba1855cd..6e4f3c8c 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/RunbookSql.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/RunbookSql.java @@ -8,9 +8,13 @@ import java.util.Map; /** * The SQL a runbook gives an operator, read out of the runbook itself. Every lane that runs a - * runbook's SQL reads it through this one class: so far the outbox runbooks and the migration - * runbook's recovery of a failed concurrent index migration (re-review 2026-11-03 round 8, - * RR8-03-03: the class kept an outbox name after the migration runbook began to use it). + * runbook's SQL reads it through this one class: so far the outbox runbooks ({@code + * PostgreSqlOutboxDispositionRunbookIntegrationTest}, {@code + * PostgreSqlOutboxOrderKeyRunbookIntegrationTest}), the migration runbook's recovery of a failed + * concurrent index migration ({@code PostgreSqlConcurrentIndexMigrationIntegrationTest}) and the + * lock-timeout runbook's holder lookup ({@code PostgreSqlDistributedLockIntegrationTest}). + * Re-review 2026-11-03 round 8, RR8-03-03: the class kept an outbox name after the migration + * runbook began to use it; round9 review 2026-11-12, RR9-01-10: this list missed the lock lane. * *

A runbook block is marked {@code } and followed by a {@code sql} * fence. The lanes that run these blocks against real PostgreSQL read them from the document rather diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java index 98739e4d..a483b2fe 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java @@ -1,18 +1,34 @@ package dev.caskeleton.adapter.outbound.persistence.autoconfigure; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import dev.caskeleton.adapter.outbound.persistence.liveevent.JpaLiveEventReplay; import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventJpaRepository; +import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReaper; import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.springframework.boot.context.properties.bind.UnboundConfigurationPropertiesException; import org.springframework.boot.convert.ApplicationConversionService; import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.scheduling.config.FixedDelayTask; +import org.springframework.scheduling.config.ScheduledTask; +import org.springframework.scheduling.config.ScheduledTaskHolder; class JpaLiveEventReplayActivationTest { @@ -34,6 +50,110 @@ class JpaLiveEventReplayActivationTest { }); } + @Test + void theReaperIsAssembledWithTheReplayStore() { + runner + .withPropertyValues( + "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H") + .run( + context -> + assertThat(context) + .as("without the reaper nothing removes a row past its retention") + .hasSingleBean(LiveEventReaper.class)); + } + + @Test + void noReaperIsAssembledWithoutTheReplayStore() { + runner.run(context -> assertThat(context).doesNotHaveBean(LiveEventReaper.class)); + } + + @Test + void aZeroReaperIntervalFailsStartup() { + runner + .withPropertyValues( + "app.realtime.replay.enabled=true", + "app.realtime.replay.retention=PT1H", + "app.realtime.replay.reaper-interval=PT0S") + .run( + context -> + assertThat(context.getStartupFailure()) + .rootCause() + .hasMessageContaining("app.realtime.replay.reaper-interval must be positive")); + } + + @Test + void aMisspeltReplayKeyFailsStartup() { + runner + .withPropertyValues( + "app.realtime.replay.enabled=true", + "app.realtime.replay.retention=PT1H", + "app.realtime.replay.reaper-intervall=PT1M") + .run( + context -> + assertThat(context.getStartupFailure()) + .rootCause() + .isInstanceOf(UnboundConfigurationPropertiesException.class) + .hasMessageContaining("reaper-intervall")); + } + + @ParameterizedTest(name = "[{index}] reaper-interval {0} → {1}") + @CsvSource( + value = {"UNSET, PT10M", "PT3M, PT3M"}, + nullValues = "UNSET") + @DisplayName("reaper는 묶인 reaper-interval(기본 PT10M)마다 돌고, 첫 회도 그 간격 뒤다") + void theReaperRunsAtTheBoundInterval(String configured, Duration expected) { + // given + ApplicationContextRunner replay = + runner.withPropertyValues( + "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H"); + if (configured != null) { + replay = replay.withPropertyValues("app.realtime.replay.reaper-interval=" + configured); + } + + // when + List> schedules = new ArrayList<>(); + replay.run(context -> schedules.addAll(fixedDelaySchedules(context))); + + // then + assertThat(schedules) + .as("the schedule is the bound settings' interval, its default included") + .containsExactly(List.of(expected, expected)); + } + + @Test + void theScheduledTaskIsTheReaper() { + // given + ApplicationContextRunner replay = + runner.withPropertyValues( + "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H"); + + // when + List reapedThrough = new ArrayList<>(); + replay.run( + context -> { + fixedDelayTasks(context).forEach(task -> task.getRunnable().run()); + reapedThrough.add(context.getBean(LiveEventJpaRepository.class)); + }); + + // then + verify(reapedThrough.getFirst()).deleteExpiredBatch(any(Instant.class), anyInt()); + } + + /** Interval and initial delay of every fixed-delay task the context scheduled. */ + private static List> fixedDelaySchedules(ApplicationContext context) { + return fixedDelayTasks(context).stream() + .map(task -> List.of(task.getIntervalDuration(), task.getInitialDelayDuration())) + .toList(); + } + + private static List fixedDelayTasks(ApplicationContext context) { + return context.getBean(ScheduledTaskHolder.class).getScheduledTasks().stream() + .map(ScheduledTask::getTask) + .filter(FixedDelayTask.class::isInstance) + .map(FixedDelayTask.class::cast) + .toList(); + } + @Test void enabledReplayCapabilityRequiresExplicitRetention() { runner @@ -42,7 +162,8 @@ class JpaLiveEventReplayActivationTest { context -> { assertThat(context).hasFailed(); assertThat(context.getStartupFailure()) - .hasMessageContaining("app.realtime.replay.retention"); + .rootCause() + .hasMessageContaining("app.realtime.replay.retention must be set"); }); } diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java index a560adf4..74694d1f 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.experimental; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.persistence.experimental.database.TenantPoolBudget; import dev.caskeleton.adapter.outbound.persistence.experimental.replica.ConsistencyAwareDataSourceRouter; @@ -103,14 +104,28 @@ class ExperimentalFailClosedTest { void tenantPoolBudgetIsBounded() { var budget = new TenantPoolBudget(2, 20); - assertThat(budget.hasCapacity(1, 10)).isTrue(); - assertThatThrownBy(() -> budget.requireCapacity(2, 10)) + assertThat(budget.hasCapacity(1, 10, 10)).isTrue(); + assertThatThrownBy(() -> budget.requireCapacity(2, 10, 1)) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("tenant pool budget"); - assertThatThrownBy(() -> budget.requireCapacity(1, 20)) + assertThatThrownBy(() -> budget.requireCapacity(1, 20, 1)) .isInstanceOf(IllegalStateException.class); } + @Test + void aNewPoolLargerThanTheConnectionsLeftIsRefused() { + // given + var budget = new TenantPoolBudget(4, 20); + + // when + Throwable refused = catchThrowable(() -> budget.requireCapacity(1, 15, 10)); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("tenant pool budget exhausted"); + } + /** A lag monitor with a fixed answer, including "I do not know". */ private record FixedLagMonitor(Optional replayedThrough, Optional lag) implements ReplicaLagMonitor {} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java new file mode 100644 index 00000000..1fb7f801 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java @@ -0,0 +1,45 @@ +package dev.caskeleton.adapter.outbound.persistence.hibernate.stateless; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; +import static org.mockito.Mockito.mock; + +import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; +import java.time.Duration; +import java.util.Set; +import org.hibernate.SessionFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * The stateless runner's transaction bound is the write bound every other transaction here has. + * + *

The runner restates the value instead of reading it, because the {@code hibernate} package may + * not depend on {@code transaction} (JpaModuleBoundaryTest). A test may depend on both, so it is + * the place the two copies are held to each other (round9 review 2026-11-12, RR9-01-25). + */ +class StatelessRunnerTimeoutTest { + + @Test + void theRunnersDefaultIsSpringTransactionsWriteDefault() { + assertThat(HibernateStatelessSessionRunner.DEFAULT_TIMEOUT) + .isEqualTo(SpringTransaction.Timeouts.DEFAULT.write()); + } + + @ParameterizedTest + @ValueSource(strings = {"PT0S", "PT-1S", "PT1.5S", "PT2147483648S"}) + void aTimeoutThatIsNotAWholePositiveNumberOfSecondsIsTheAssemblysFault(String timeout) { + // when + Throwable refused = + catchThrowable( + () -> + new HibernateStatelessSessionRunner( + mock(SessionFactory.class), Set.of(), Duration.parse(timeout))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("must be a positive whole number of seconds"); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java new file mode 100644 index 00000000..b43a316e --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java @@ -0,0 +1,53 @@ +package dev.caskeleton.adapter.outbound.persistence.liveevent; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link LiveEventReaper}: rows retained no later than the clock's now go, in + * bounded batches until one comes back short. + * + *

The repository is a Spring Data boundary, so Mockito stands in for it. The statement runs + * against PostgreSQL in {@code JpaLiveEventReplayContractTest}. + */ +class LiveEventReaperTest { + + private static final Instant NOW = Instant.parse("2026-11-13T00:00:00Z"); + private static final int BATCH = LiveEventReaper.BATCH_SIZE; + + private final LiveEventJpaRepository repository = mock(LiveEventJpaRepository.class); + private final LiveEventReaper reaper = + new LiveEventReaper(repository, Clock.fixed(NOW, ZoneOffset.UTC)); + + @Test + void reapKeepsDeletingBatchesUntilOneComesBackShortAndReturnsTheTotal() { + // given + when(repository.deleteExpiredBatch(NOW, BATCH)).thenReturn(BATCH, 4); + + // when + int purged = reaper.reap(); + + // then + assertThat(purged).isEqualTo(BATCH + 4); + } + + @Test + void eachBatchIsItsOwnDeleteStatement() { + // given + when(repository.deleteExpiredBatch(NOW, BATCH)).thenReturn(BATCH, 4); + + // when + var unused = reaper.reap(); + + // then + verify(repository, times(2)).deleteExpiredBatch(NOW, BATCH); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java index 72fe3def..471a36ba 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java @@ -49,6 +49,20 @@ class DistributedLockPersistenceConfigTest { }); } + @Test + @DisplayName("읽는 곳이 없던 app.lock.wait-time은 모르는 키로 기동을 실패시킨다") + void theRemovedWaitTimeKeyFailsStartup() { + // when + contextRunner + .withPropertyValues("app.lock.wait-time=3s") + .run( + context -> { + // then + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()).hasStackTraceContaining("wait-time"); + }); + } + @Test void multiInstanceBuildsTheJdbcProviderAndNotTheInProcessProvider() { contextRunner diff --git a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 index 32edf890..93a51fe4 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 +++ b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 @@ -14,6 +14,7 @@ be51fabd07220f9ef885534ba36ada5f1491baf24cdfb54d25d93a19ddc29926 jpa/notification-platform/V11__notification_reconciliation_claim_lease.sql 43f9429f91c07528154ec01b721649b2c25fa98857ac154c7c08f4655634657f jpa/notification-platform/V12__notification_suppression_target_unique.sql d01aaf59f66834358abf18246faa9a9eb8dfa9f42097ca9d968c44edff564630 jpa/notification-platform/V13__notification_request_acceptance.sql +8a7dc3604d44f1438b39dfec3aa688f792e7363cd3c1b74bf0c799a6b53f9f0d jpa/notification-platform/V14__notification_provider_event_open_index.sql 1ac7cc1c2509868eb0ce202e1917bb36846f98f17ad4922abe3fac0ef128199b jpa/notification-platform/V1__notification_platform_core.sql 0add67aa23580e51712f84fabe9b09e08401ef708cfa1b3292b120fdb5c12fa2 jpa/notification-platform/V2__notification_platform_contact_template_policy.sql f5b98598a59cc4376f5d233b410f8a5ae9f3cd94156291ebdf2d0da5a66f34ef jpa/notification-platform/V3__notification_platform_inbox_admin.sql diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java index 5a81e94b..72a52a8e 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java @@ -1,13 +1,17 @@ package dev.caskeleton.adapter.outbound.persistence.notification.platform; +import dev.caskeleton.adapter.outbound.persistence.notification.platform.repository.RecipientClaimSql; import dev.caskeleton.application.notification.platform.observation.NotificationServingState; import dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort; import java.sql.Timestamp; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Map; import java.util.Objects; import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; /** * Measures the platform's serving state from the durable rows. @@ -17,34 +21,44 @@ import org.springframework.jdbc.core.JdbcOperations; * identical. These are the questions an operator actually asks during an incident: how far behind * are we, how long has the oldest thing waited, and is anything stuck. * - *

One statement per question rather than one joined statement. They are read independently, a - * slow one does not delay the others, and each index the queries rely on is the same index the - * corresponding worker already uses to claim. + *

One statement per question rather than one joined statement. They are read independently and a + * slow one does not delay the others. Every statement is bounded by an index: the backlog and + * stuck-lease counts by the predicates the claim uses, the reconciliation count by the job's due + * time, and the projection lag by {@code ix_notification_provider_event_open} (V14), which holds + * only the events still waiting for something — the provider-event table itself is never purged, so + * the lag query used to read all of it (round9 review 2026-11-12, RR9-01-14). + * + *

The scheduler reads only {@link #backlogDepth()}, on every poll; the full snapshot is for the + * health endpoint. */ public final class JdbcNotificationServingState implements NotificationServingStatePort { /** - * Deliveries that are due and unclaimed. + * Deliveries the claim would take now. * - *

The lease predicate matches the claim's: a row whose lease has expired is available again, - * so it is backlog. Counting it as in-flight is how a stuck worker hides a growing queue. + *

The {@code WHERE} is the claim's own fragment ({@link RecipientClaimSql#CLAIMABLE_AT_NOW}): + * the same states, due, not expired, and a lease that is absent or has run out. A row whose lease + * has expired is available again, so it is backlog; counting it as in-flight is how a stuck + * worker hides a growing queue. An expired delivery is not backlog: the expiry sweep retires it + * unsent. The fragment's parameter is named, so this statement is too. */ private static final String BACKLOG = """ SELECT count(*) AS depth, - COALESCE(EXTRACT(EPOCH FROM (? - min(next_dispatch_at))), 0) AS oldest_seconds + COALESCE(EXTRACT(EPOCH FROM (:now - min(next_dispatch_at))), 0) AS oldest_seconds FROM notification_recipient_delivery - WHERE delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING') - AND next_dispatch_at IS NOT NULL - AND next_dispatch_at <= ? - AND (lease_until IS NULL OR lease_until <= ?) - """; + WHERE\s""" + + RecipientClaimSql.CLAIMABLE_AT_NOW; /** * Leases whose holder never came back. * - *

Distinct from backlog: these rows are claimable again, but the fact that they had to expire - * means a worker died mid-dispatch, and a provider call may have happened without a record. + *

Distinct from backlog: these rows are still dispatching, which the claim never takes; lease + * recovery requeues or reconciles them one lease after they ran out. A row left from before + * notification V4 (an owner but fence 0) is counted here and not recovered; {@code + * docs/upgrade/2026-11-round9-claim-fence.md}, row 5, says how to hand it to recovery. That a + * lease had to run out means a worker died or stalled mid-dispatch, and a provider call may have + * happened without a record. */ private static final String STUCK_LEASES = """ @@ -56,7 +70,21 @@ public final class JdbcNotificationServingState implements NotificationServingSt AND delivery_state = 'DISPATCHING' """; - /** Provider events stored and not yet applied to a delivery. */ + /** + * The provider events still waiting for something: the lag query's {@code WHERE}, and the + * predicate of the partial index {@code V14} builds for it. One string, held to the migration by + * {@code JdbcNotificationServingStateOpenEventsTest}: a query the index does not cover reads the + * whole table again, which is what the index was made to stop. + */ + static final String OPEN_EVENTS = + "projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL"; + + /** + * Provider events stored and not yet applied to a delivery. + * + *

The {@code WHERE} is exactly the predicate of {@code ix_notification_provider_event_open}: + * every row any of the four counts can see, and none of the applied, matched history. + */ private static final String PROJECTION_LAG = """ SELECT @@ -66,7 +94,9 @@ public final class JdbcNotificationServingState implements NotificationServingSt COALESCE(EXTRACT(EPOCH FROM (? - min(received_at) FILTER (WHERE projection_status = 'PENDING'))), 0) AS oldest_seconds FROM notification_provider_event - """; + WHERE + """ + + OPEN_EVENTS; /** Questions to providers that are due and unanswered. */ private static final String RECONCILIATION_LAG = @@ -78,26 +108,36 @@ public final class JdbcNotificationServingState implements NotificationServingSt """; private final JdbcOperations jdbc; + private final NamedParameterJdbcOperations named; private final Clock clock; public JdbcNotificationServingState(JdbcOperations jdbc, Clock clock) { this.jdbc = Objects.requireNonNull(jdbc, "jdbc"); + this.named = new NamedParameterJdbcTemplate(jdbc); this.clock = Objects.requireNonNull(clock, "clock"); } + @Override + public long backlogDepth() { + Measurement backlog = measureBacklog(Timestamp.from(clock.instant())); + return backlog == null ? 0 : backlog.count(); + } + + /** The backlog query alone: the one statement the scheduler's queue-depth gauge needs. */ + private Measurement measureBacklog(Timestamp at) { + return named.queryForObject( + BACKLOG, + Map.of("now", at), + (rows, rowNumber) -> + new Measurement(rows.getLong("depth"), rows.getDouble("oldest_seconds"))); + } + @Override public NotificationServingState currentState() { Instant now = clock.instant(); Timestamp at = Timestamp.from(now); - Measurement backlog = - jdbc.queryForObject( - BACKLOG, - (rows, rowNumber) -> - new Measurement(rows.getLong("depth"), rows.getDouble("oldest_seconds")), - at, - at, - at); + Measurement backlog = measureBacklog(at); Long stuck = jdbc.queryForObject(STUCK_LEASES, Long.class, at); ProjectionMeasurement projection = jdbc.queryForObject( diff --git a/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql b/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql new file mode 100644 index 00000000..2d2416a2 --- /dev/null +++ b/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql @@ -0,0 +1,26 @@ +-- An index over the provider events still waiting for something. +-- +-- The serving-state projection lag counts pending, failed and unmatched events. It had no WHERE and +-- read the whole notification_provider_event table, which nothing ever purges, and the scheduler +-- asked for it on every poll (round9 review 2026-11-12, RR9-01-14). The query now names exactly +-- this predicate, so it reads only these rows; applied, matched events — the history that grows — +-- are outside the index. +-- +-- Built inside this migration's transaction, not CONCURRENTLY, as V21 of the core stream builds its +-- index: a transactional file fails whole and leaves nothing to repair, where a failed concurrent +-- build leaves an INVALID index and a failed history row an operator must clear by hand +-- (docs/runbooks/migration-failed.md). The cost is that the build holds SHARE on +-- notification_provider_event: provider callbacks wait for it, and providers retry a callback that +-- does not get an answer. On a table already large in production, create the same index +-- CONCURRENTLY by hand before deploying; IF NOT EXISTS then makes this migration a no-op. + +-- Bound the wait for the lock (SHARE on notification_provider_event, held until commit). Unbounded, +-- it queues behind any open transaction that touched the table, and every callback insert queues +-- behind the migration. Five seconds, then the migration fails, rolls back whole and the stream +-- stays at V13: deploy again once that transaction has ended (docs/jpa/migration-guide.md "Bound +-- the lock wait"). +SET LOCAL lock_timeout = '5s'; + +CREATE INDEX IF NOT EXISTS ix_notification_provider_event_open + ON notification_provider_event (received_at) + WHERE projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL; diff --git a/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java b/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java new file mode 100644 index 00000000..0805a0d6 --- /dev/null +++ b/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java @@ -0,0 +1,42 @@ +package dev.caskeleton.adapter.outbound.persistence.notification.platform; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; +import org.junit.jupiter.api.Test; + +/** + * The projection-lag query and the partial index built for it name the same events. + * + *

The index is what keeps the lag query, which the scheduler runs on every poll, from reading + * the whole provider-event history. A predicate that drifted on either side would leave the query + * outside the index and scanning the table again, with nothing failing. + */ +class JdbcNotificationServingStateOpenEventsTest { + + private static final String V14 = + "db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql"; + + @Test + void theLagQueryAndItsPartialIndexNameTheSameEvents() throws IOException { + // given + String migration; + try (InputStream sql = getClass().getClassLoader().getResourceAsStream(V14)) { + migration = new String(Objects.requireNonNull(sql, V14).readAllBytes(), UTF_8); + } + int index = migration.indexOf("CREATE INDEX"); + + // when + String indexPredicate = + migration + .substring( + migration.indexOf("WHERE ", index) + "WHERE ".length(), migration.lastIndexOf(';')) + .trim(); + + // then + assertThat(indexPredicate).isEqualTo(JdbcNotificationServingState.OPEN_EVENTS); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java index ce48cf07..7ddece3f 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java @@ -12,7 +12,7 @@ import java.util.Objects; * being applied reported exactly the same numbers as an idle healthy one. Readiness that cannot * distinguish those two states is not readiness. * - * @param backlogDepth deliveries due now and not yet claimed + * @param backlogDepth deliveries a claim would take now: due, not expired, not under a live lease * @param oldestDueAge how long the oldest of them has been due * @param stuckLeases leases whose holder has not finished and whose expiry has passed * @param pendingProjections provider events stored and not yet applied diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java index 79a817aa..f6ae4f15 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java @@ -6,7 +6,25 @@ public interface NotificationServingStatePort { /** * Measure the backlog, the stuck leases and the projection and reconciliation lag. * + *

Several questions, for the health snapshot. A caller that needs only the queue depth asks + * {@link #backlogDepth()} instead: the scheduler reads the depth on every poll, and asking all of + * them that often ran four statements a poll, one of them over the whole provider-event table + * (round9 review 2026-11-12, RR9-01-14). + * * @return the current state */ NotificationServingState currentState(); + + /** + * How many deliveries a claim would take now — due, not expired, and not under a live lease: the + * {@link NotificationServingState#backlogDepth()} of {@link #currentState()}, measured alone. + * + *

The default takes it from the full snapshot, so an implementation that can only measure + * everything at once still answers; a store that can count the backlog by itself overrides it. + * + * @return the backlog depth + */ + default long backlogDepth() { + return currentState().backlogDepth(); + } } From 154cb4b7959916b66fcd99b18380967d59b83924 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 13 Nov 2026 16:21:16 +0900 Subject: [PATCH 3/4] =?UTF-8?q?fix(outbound)!:=20=EB=B0=9C=ED=96=89=20?= =?UTF-8?q?=EB=8C=80=EA=B8=B0=C2=B7SMTP=20=EB=B0=9C=EC=86=A1=C2=B7?= =?UTF-8?q?=EC=BA=90=EC=8B=9C=20=EC=8B=9C=EA=B0=84=20=EC=83=81=ED=95=9C?= =?UTF-8?q?=EC=9D=84=20=EC=A7=80=ED=82=A8=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit outbox 발행 대기가 publish()가 돌아온 뒤부터 재서, 호출을 붙잡은 publisher는 longestAttempt를 넘겨도 relay를 묶는다. SMTP 발송 pool은 대기열이 찰 때까지 thread 하나로 돌고, profile들의 max-concurrency가 pool이 담는 수를 넘어도 기동한다. 넘친 발송은 결과 불명으로 남는다. SMTP profile timeout은 명령 하나만 묶어, 느린 relay에서 발송이 lease를 넘으면 lease 복구가 아직 도는 발송을 운영자 확인으로 넘긴다. Redis 캐시 command-timeout에 lane 대기 2초가 더해지고, 값 크기 상한은 설정할 길이 없다. 고른 방법: 발행 대기 마감은 publish() 전에 잡고 남은 시간만 기다린다. 기다리는 중 interrupt는 결과 불명, 이미 interrupt된 thread와 0 이하 longestAttempt는 보내지 않고 REJECTED_RETRYABLE로 돌려준다. SMTP pool은 dispatch-concurrency개 thread와 그 네 배 대기열로 고정한다. 한 family의 켜진 profile 전체를 보는 ProviderRuntimeAssembler.requireTogether를 두고, SMTP는 max-concurrency 합이 pool보다 크거나 lease가 timeout 두 배 이하면 조립을 실패시킨다. profile timeout은 발송 한 번을 두 번 묶는다: pool 대기가 넘으면 보내지 않고 재시도, 발송이 넘으면 PROVIDER_RESPONSE_LOST 로 남겨 다시 보내지 않는다. 두 시한은 completeOnTimeout이라 취소된다. 기본 timeout은 10초에서 30초로, socket timeout 합보다 길어야 한다. Redis 연산 전체를 command-timeout 하나로 묶고 app.capabilities.cache.maximum-value-bytes를 다시 묶는다(1..16 MiB). 버린 대안: 발송 timeout에 재시도(같은 메일을 두 번 보낼 수 있다), 캐시 상한을 상수로 두기(큰 값을 쓰는 서비스가 바꿀 길이 없다), assembler가 assemble마다 합을 누적하기(조립 순서에 상태가 남는다). 위험·한계: 기동 조건이 늘었다(SMTP max-concurrency 합, lease와 SMTP timeout, 캐시 값 상한 범위). 대기 시한이 돌려주는 stage는 일부러 버린다 (Error Prone 규칙대로 `unused`에 받고, SpotBugs DLS는 config/spotbugs/ exclude.xml에 그 메서드만 제외한다. Jetty abortAtDeadline 선례). send pool을 바꿔 끼운 배포는 부른 thread에서 돌리지 않는 executor여야 한다. outbox 새 거절 코드의 runbook 줄은 runbook 묶음에 넘긴다. claim·fence 커밋 위에 쌓았다. 검증: desktop RED(고치기 전 트리 + 새 시험, pr2-red2) 새 시험 실패. GREEN은 PR 본문의 verify.sh pr 결과. Refs: RR9-01-05, RR9-01-15, RR9-01-16, RR9-01-17, RR9-01-18 BREAKING CHANGE: SMTP profile들의 max-concurrency 합이 send pool(thread와 대기열)을 넘거나 dispatch.lease-duration이 SMTP timeout의 두 배 이하면 기동이 실패한다. SMTP 기본 timeout이 30초이고, 넘은 발송은 결과 불명이 된다. SmtpNotificationProviderAdapter 생성자가 timeout을 받는다. Redis 캐시 값이 maximum-value-bytes(기본 1 MiB)를 넘으면 기록하지 않는다. docs/upgrade/2026-11-round9-outbound.md --- .env.example | 3 +- config/spotbugs/exclude.xml | 9 ++ docs/notification/configuration-reference.md | 33 +++- docs/registries/env-keys.yaml | 11 +- docs/upgrade/2026-11-round9-claim-fence.md | 2 +- docs/upgrade/2026-11-round9-outbound.md | 29 ++++ src/adapter/outbound/cache-redis/README.md | 7 +- .../cache/redis/SemanticRedisCall.java | 24 +++ .../redis/cache/RedisCacheRegionAdapter.java | 73 ++++++++- .../redis/capability/RedisCacheConfig.java | 3 +- .../capability/RedisCapabilitySettings.java | 26 +++ .../config/module-defaults/cache-redis.yml | 3 + .../cache/RedisCacheRegionAdapterTest.java | 127 ++++++++++++++- .../RedisCacheValueCeilingSettingTest.java | 40 +++++ src/adapter/outbound/messaging/AGENTS.md | 4 +- src/adapter/outbound/messaging/README.md | 11 ++ .../outbox/OutboxMessagePublishAdapter.java | 69 +++++++- .../OutboxMessagePublishAdapterTest.java | 150 ++++++++++++++++++ .../NotificationProviderAssembly.java | 28 ++++ .../NotificationSmtpProviderConfig.java | 20 ++- .../ProviderRuntimeAssembler.java | 18 +++ .../SmtpProviderRuntimeAssembler.java | 101 +++++++++++- .../smtp/SmtpNotificationProviderAdapter.java | 140 +++++++++++++++- .../NotificationProviderAssemblyTest.java | 118 ++++++++++++++ .../NotificationSmtpExecutorTest.java | 58 +++++++ .../SmtpProviderRuntimeAssemblerTest.java | 114 ++++++++++++- .../SmtpNotificationProviderAdapterTest.java | 130 ++++++++++++++- .../event/IntegrationEventPublishPort.java | 9 +- .../FakeIntegrationEventPublisher.java | 65 +++++++- .../config/module-defaults/notification.yml | 2 +- 30 files changed, 1384 insertions(+), 43 deletions(-) create mode 100644 docs/upgrade/2026-11-round9-outbound.md create mode 100644 src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisCacheValueCeilingSettingTest.java create mode 100644 src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationSmtpExecutorTest.java diff --git a/.env.example b/.env.example index 1c68b40e..09bb6b74 100644 --- a/.env.example +++ b/.env.example @@ -171,6 +171,7 @@ APP_CACHE_REDIS_COMMAND_TIMEOUT=200ms APP_CACHE_REDIS_POSITIVE_SOFT_TTL=30s APP_CACHE_REDIS_MINIMUM_HARD_TTL=1s APP_CACHE_REDIS_SEMANTIC_REGION=default +APP_CACHE_REDIS_MAXIMUM_VALUE_BYTES=1048576 APP_MESSAGING_BROKER=kafka APP_MESSAGING_PRODUCER_ID= APP_MESSAGING_KAFKA_BROKERS= @@ -293,7 +294,7 @@ APP_NOTIFICATION_PLATFORM_SMTP_ENABLED=false APP_NOTIFICATION_PLATFORM_SMTP_PRIMARY=true APP_NOTIFICATION_PLATFORM_SMTP_ENVIRONMENT=local APP_NOTIFICATION_PLATFORM_SMTP_CREDENTIAL_PROFILE=default -APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT=10s +APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT=30s APP_NOTIFICATION_PLATFORM_SMTP_MAX_CONCURRENCY=4 APP_NOTIFICATION_PLATFORM_SMTP_RATE_PER_SECOND=10 APP_NOTIFICATION_PLATFORM_SMTP_SENDER_IDENTITY=no-reply@example.invalid diff --git a/config/spotbugs/exclude.xml b/config/spotbugs/exclude.xml index 2ee07473..156d6eae 100644 --- a/config/spotbugs/exclude.xml +++ b/config/spotbugs/exclude.xml @@ -359,6 +359,15 @@ + + + + + + +