From afdedabf2a396caa1446926af2e6ad20bc01a205 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 13 Nov 2026 13:17:33 +0900 Subject: [PATCH 1/8] =?UTF-8?q?fix(persistence)!:=20=EB=84=98=EA=B2=A8?= =?UTF-8?q?=EB=B0=9B=EC=9D=80=20claim=EB=A7=8C=20=EC=93=B0=EA=B2=8C=20fenc?= =?UTF-8?q?e=EC=99=80=20=EC=A1=B0=EA=B1=B4=EB=B6=80=20=EC=93=B0=EA=B8=B0?= =?UTF-8?q?=EB=A5=BC=20=EB=A7=9E=EC=B6=98=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lease를 넘겨받는 저장소 다섯 곳이 앞 실행의 쓰기를 막지 못한다. durable operation은 owner만 보고 써서 같은 worker id가 다시 claim하면 앞 실행의 결과가 남는다. 알림 lease 회복은 상태를 다시 읽은 뒤 조건 없이 되돌려 끝난 delivery를 다시 보낸다. 만료 sweep·전체 읽음은 기다린 행의 상태를 다시 보지 않는다. 멱등성 reaper는 앱 시계로 비교하고 한 번에 다 지우며, 같은 키 요청이 되살린 기록까지 지운다. fileserver recovery enqueue는 동시 삽입에서 25P02로 실패하고, cleanup은 claim lease가 끝난 뒤에도 항목을 시작한다. 저장소가 행에서 만드는 claim·lease record는 잘못된 값을 IllegalArgumentException으로 거절해 web이 400으로 바꾼다. 고른 방법: 모든 쓰기를 owner·fence(·상태·만료) 조건을 단 한 문장으로 바꾼다. durable operation은 V23 lease_fence를 더하고 DurableOperationClaim 을 돌려준다. 알림 회복은 requeueAbandoned 한 문장이다. 만료 sweep은 subquery가 SKIP LOCKED로 쥔 행을 건너뛰고 바깥 UPDATE가 상태·만료를 다시 본다. 전체 읽음은 바깥 UPDATE가 상태를 다시 본다. reaper는 DB now()로 1000행씩, 트랜잭션마다 30초(SpringTransaction.DEFAULT_TIMEOUT_SECONDS)이고 바깥 DELETE에도 만료를 둔다. recovery는 INSERT … ON CONFLICT, cleanup은 lease 마지막 2분에 시작하지 않고 시도를 세지 않는 release로 돌려준다. 알림 provider timeout은 lease보다 짧아야 기동한다(호출이 lease를 넘으면 lease 복구가 아직 도는 호출을 맞추기로 넘긴다). 저장소가 만드는 record 일곱은 IllegalStateException으로 거절한다. 버린 대안: 읽고 비교한 뒤 쓰기(경합이 그대로 남음), 앱 쪽 락(인스턴스 사이에서 무의미). 위험·한계: 포트 시그니처가 바뀐다(DurableOperationStorePort, RecipientLeaseStorePort.expiredDispatching, RecipientDeliveryStorePort .requeueAbandoned, CleanupQueuePort.release). owner가 없거나 fence가 0인 V4 전 DISPATCHING 행은 회복이 고르지 않는다(안내 4행에 고치는 SQL). provider timeout 검사는 필요조건일 뿐이고, 호출 전체를 묶는 것은 SES다. 검증: desktop RED(고치기 전 트리 + 새 시험, pr1-red3) 새 시험 13건 실패. GREEN은 PR 본문의 verify.sh pr·PostgreSQL lane 결과. Refs: RR9-01-02, RR9-01-04, RR9-01-06, RR9-01-12, RR9-01-13, RR9-01-23, RR9-01-24, RR9-01-29, R1 넘김 DurableOperationStoreContractTest BREAKING CHANGE: DurableOperationStorePort의 heartbeat·reportProgress·succeed·fail이 DurableOperationClaim을 받고 claimNext가 그것을 돌려준다. RecipientLeaseStorePort.expiredDispatching이 List를 돌려주고 RecipientDeliveryStorePort.requeueAbandoned, CleanupQueuePort.release가 새로 생긴다. IdempotencyReaper 생성자에서 Clock이 빠진다. 알림 provider timeout이 lease 이상이면 기동이 실패한다. 저장소가 만드는 claim·lease record가 IllegalStateException을 던진다. docs/upgrade/2026-11-round9-claim-fence.md --- docs/architecture/jpa-api-surface.txt | 3 +- docs/ledger/estimates.jsonl | 1 + docs/manual/04-persistence.md | 11 + docs/notification/configuration-reference.md | 9 +- docs/registries/env-keys.yaml | 3 +- docs/upgrade/2026-11-round9-claim-fence.md | 38 ++ .../NotificationPlatformSettings.java | 36 ++ .../NotificationDispatchProperties.java | 5 +- .../NotificationPlatformSettingsTest.java | 33 ++ .../fileserver/JpaCleanupQueue.java | 6 + .../fileserver/JpaRecoveryQueue.java | 19 +- .../fileserver/entity/RecoveryItemEntity.java | 11 - .../FileserverCleanupRepository.java | 26 + .../FileserverRecoveryRepository.java | 31 +- .../outbound/persistence-jpa/AGENTS.md | 3 + .../outbound/persistence-jpa/README.md | 29 +- .../autoconfigure/JpaAdapterBeansConfig.java | 4 +- .../idempotency/jpa/IdempotencyReaper.java | 23 +- .../jpa/IdempotencyRecordJpaRepository.java | 35 +- .../DurableOperationJpaRepository.java | 24 +- .../operation/JpaDurableOperationStore.java | 74 +-- .../outbox/jpa/JpaOutboxStore.java | 2 +- .../outbox/jpa/OutboxEventJpaRepository.java | 6 +- .../persistence/outbox/jpa/OutboxReaper.java | 10 +- .../transaction/BatchedDeletes.java | 36 ++ .../transaction/SpringTransaction.java | 13 +- .../V23__durable_operation_lease_fence.sql | 24 + .../IdempotencyReaperIntegrationTest.java | 205 ++++++++ ...ostgreSqlIdempotencyStoreContractTest.java | 3 +- .../notification/NotificationJpaStack.java | 30 ++ ...greSqlRecipientLeaseStoreContractTest.java | 72 +++ .../InboxBulkReadVersionContractTest.java | 51 ++ .../DurableOperationSchemaConstraintTest.java | 214 ++++++++ .../DurableOperationStoreContractTest.java | 474 ------------------ .../JpaDurableOperationStoreContractTest.java | 99 +++- .../readiness/FileserverPostgreSqlStack.java | 30 ++ .../PostgreSqlMigrationIntegrationTest.java | 2 +- ...ficationBackgroundPassIntegrationTest.java | 27 +- .../PostgreSqlRecoveryQueueContractTest.java | 69 +++ .../jpa/IdempotencyReaperTest.java | 56 ++- .../resources/db/applied-migrations.sha256 | 1 + .../persistence-notification/AGENTS.md | 3 +- .../platform/JpaRecipientDeliveryStore.java | 17 + .../platform/JpaRecipientLeaseStore.java | 31 +- .../repository/InboxItemJpaRepository.java | 21 +- .../platform/repository/InboxItemSql.java | 13 + .../repository/RecipientClaimSql.java | 51 +- .../RecipientDeliveryJpaRepository.java | 41 +- .../operation/DurableOperationClaim.java | 45 ++ .../operation/DurableOperationStorePort.java | 42 +- .../application/operation/OperationLease.java | 8 +- .../outbox/ClaimedOutboxEvent.java | 8 +- .../application/outbox/OutboxLease.java | 8 +- .../CancelDurableOperationUseCaseTest.java | 5 +- .../operation/StoreBuiltRecordsTest.java | 80 +++ .../ClaimedOutboxEventContractTest.java | 3 +- .../DurableOperationStoreContract.java | 180 ++++++- .../InMemoryDurableOperationStore.java | 71 +-- .../fileserver/cleanup/CleanupQueuePort.java | 10 + .../cleanup/DefaultCleanupService.java | 32 +- .../cleanup/CleanupServiceTest.java | 89 +++- .../testkit/CleanupQueuePortContract.java | 34 ++ .../testkit/FakeCleanupContentGateway.java | 13 + .../testkit/InMemoryCleanupQueue.java | 22 + .../platform/admin/AdminOperationClaim.java | 6 +- .../dispatch/LeaseRecoveryService.java | 37 +- .../dispatch/RecipientDeliveryStorePort.java | 24 + .../platform/dispatch/RecipientLease.java | 8 +- .../dispatch/RecipientLeaseStorePort.java | 24 +- .../platform/dispatch/ReconciliationJob.java | 8 +- .../dispatch/LeaseRecoveryServiceTest.java | 34 ++ .../dispatch/StoreBuiltRecordsTest.java | 76 +++ .../testkit/InMemoryDeliveryAttemptStore.java | 23 +- .../testkit/InMemoryRecipientQueue.java | 42 +- .../RecipientDeliveryStoreContract.java | 78 +++ .../testkit/RecipientLeaseStoreContract.java | 62 ++- .../config/module-defaults/notification.yml | 5 +- 77 files changed, 2345 insertions(+), 757 deletions(-) create mode 100644 docs/upgrade/2026-11-round9-claim-fence.md create mode 100644 src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/BatchedDeletes.java create mode 100644 src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V23__durable_operation_lease_fence.sql create mode 100644 src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/IdempotencyReaperIntegrationTest.java create mode 100644 src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSchemaConstraintTest.java delete mode 100644 src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationStoreContractTest.java create mode 100644 src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemSql.java create mode 100644 src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java create mode 100644 src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java create mode 100644 src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java diff --git a/docs/architecture/jpa-api-surface.txt b/docs/architecture/jpa-api-surface.txt index e2d3f24a..0063f347 100644 --- a/docs/architecture/jpa-api-surface.txt +++ b/docs/architecture/jpa-api-surface.txt @@ -5,7 +5,7 @@ # root yet. # Update only after review with: # ./gradlew :adapter:outbound:persistence-jpa:updateJpaApiSurface -PapproveJpaApiSurfaceChange -# types: 165 +# types: 166 dev.caskeleton.adapter.outbound.persistence.api.PersistenceOperationName dev.caskeleton.adapter.outbound.persistence.api.capability.CapabilitySupport dev.caskeleton.adapter.outbound.persistence.api.capability.JpaCapability @@ -169,5 +169,6 @@ dev.caskeleton.adapter.outbound.persistence.springdata.SafeSortField dev.caskeleton.adapter.outbound.persistence.springdata.SafeSortMapper dev.caskeleton.adapter.outbound.persistence.springdata.SafeSortRegistry dev.caskeleton.adapter.outbound.persistence.springdata.ScrollPolicy +dev.caskeleton.adapter.outbound.persistence.transaction.BatchedDeletes dev.caskeleton.adapter.outbound.persistence.transaction.CallerTransaction dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction diff --git a/docs/ledger/estimates.jsonl b/docs/ledger/estimates.jsonl index a506c749..8efac6e1 100644 --- a/docs/ledger/estimates.jsonl +++ b/docs/ledger/estimates.jsonl @@ -15,3 +15,4 @@ {"schema": 1, "kind": "actual", "task": "w2-be-ci-perf", "transcript": "a6b6b25f-a4f7-46c2-a746-5ab20a588c78", "minutes": 663, "active_minutes": 417, "calls": 464, "output_tokens": 438927, "cache_read_tokens": 256417797, "max_context": 911512, "within": "over", "result": "partial", "commits": ["ecc14004", "6325b518", "86488256", "c7aff0f4", "7054d179", "575aa1f7", "deeb1126", "96caf997", "cfc80ae8", "000a59ed", "5a95db45", "d237bacc", "7284b581", "14053060", "7dbf11ca", "2c180b90", "24dde586", "aeb60339", "083c6910"], "why": "시간이 p80을 넘었다: desktop 공유 잠금 대기자 10여 개(flock이 순서 없이 깨움)로 측정·검증이 2시간 넘게 밀렸고, runner 1대 대기열에서 늦게 풀린 job이 맨 뒤로 가 수동 실행 결과가 몇 시간씩 늦었다. 예측에 없던 결정: Gitea needs+if Blocked 재현(일회용 workflow 셋), Trivy가 처음 읽은 lockfile 라이선스와 이미지가 jar 라이선스를 못 읽는 것에 맞춘 판정 재설계. 결과는 partial: runner 초록은 NFS(소유자 sudo)·compose(수정 뒤 runner 미확인) 빼고 달성, 속도 목표 넷은 미달(캐시 효과는 병합 뒤, PG 컨테이너 159개 공유는 후속).", "recorded_at": "2026-11-12T08:34+09:00"} {"schema": 1, "kind": "actual", "task": "w2-be-ci-perf", "correction": true, "transcript": "a6b6b25f-a4f7-46c2-a746-5ab20a588c78", "minutes": 1266, "active_minutes": 555, "calls": 556, "output_tokens": 534063, "cache_read_tokens": 301335415, "max_context": 963079, "within": "over", "result": "partial", "commits": ["ecc14004", "6325b518", "86488256", "c7aff0f4", "7054d179", "575aa1f7", "deeb1126", "96caf997", "cfc80ae8", "000a59ed", "5a95db45", "d237bacc", "7284b581", "14053060", "7dbf11ca", "2c180b90", "24dde586", "aeb60339", "083c6910", "335fc967", "9682c52c", "3e06a809", "d17f8bcf", "7db40c9b"], "why": "앞 actual 줄을 고친다. 그 뒤 총괄 지시 둘(새 main 위로, main.yml concurrency와 취소 증거)과 main 두 번 병합으로 작업 중 138분·호출 92가 더 들었다. 대부분 runner 대기열(PR run이 한 번에 2~4시간 뒤로 밀림)과 desktop 공유 잠금 대기였다. 결과는 여전히 partial: runner PR 130 초록·release gate 125 초록, NFS는 소유자 sudo, 속도 목표 넷은 미달(PR 13.5→12.0분/목표 8, release gate 45.6→32.5분/목표 35는 이미지 job 포함), 캐시 효과는 병합 뒤 측정.", "recorded_at": "2026-11-12T18:36+09:00"} {"schema": 1, "kind": "estimate", "task": "w2-be-ci-perf-2", "transcript": "a6b6b25f-a4f7-46c2-a746-5ab20a588c78", "started_at": "2026-11-12T18:45+09:00", "base": "e7be517e", "task_kind": "improve", "size": "large", "minutes": {"p30": 150, "p50": 240, "p80": 400}, "calls": {"p30": 120, "p50": 200, "p80": 320}, "output_tokens": {"p30": 100000, "p50": 170000, "p80": 280000}, "basis": "직전 라운드 w2-be-ci-perf 실측 555분(p50 144의 3.9배, 대부분 runner·desktop 잠금 대기). 이번 판단 5개(B2·NFS·C2·job 합치기·main 줄 순서), desktop main 측정 1회(30분)+pr 1회, runner nightly 1회(2시간)+release 1회+PR 2회. 대기가 시간을 정한다", "recorded_at": "2026-11-12T18:58+09:00"} +{"schema": 1, "kind": "estimate", "task": "r9-group-b", "transcript": "0ef672ab-db3e-427a-9f53-b2e918462534", "started_at": "2026-11-13T07:00:00+09:00", "base": "d3a90479", "task_kind": "improve", "size": "large", "minutes": {"p30": 190, "p50": 286, "p80": 424}, "calls": {"p30": 378, "p50": 567, "p80": 819}, "output_tokens": {"p30": 348000, "p50": 522000, "p80": 754000}, "basis": "발견 42건·PR 3·결정 약 15·desktop 실행 약 20회(Testcontainers lane)·코드 약 3000줄; 원값 360/540/800분·600/900/1300호출·600k/900k/1300k에 보정 0.53/0.63/0.58 곱함; 선례 w2-be-ci-perf 555분", "recorded_at": "2026-11-13T06:52+09:00"} diff --git a/docs/manual/04-persistence.md b/docs/manual/04-persistence.md index 040425ef..88720690 100644 --- a/docs/manual/04-persistence.md +++ b/docs/manual/04-persistence.md @@ -91,6 +91,17 @@ store는 자기 트랜잭션을 열지 않는다. 유스케이스가 연 트랜 예: `DurableOperationStoreContract`(`InMemoryDurableOperationStore`, `JpaDurableOperationStoreContractTest`). - 여러 인스턴스가 집는 작업 → `FOR UPDATE SKIP LOCKED`는 **트랜잭션 안에서**, 그리고 lease(만료 시각)와 fence 값을 함께 저장한다. 예: `ReconciliationJobStorePort.claimDue`(application-notification). + - 연장·완료·되돌리기는 그 claim의 owner와 fence를 `WHERE`에 넣은 조건부 UPDATE 하나다. fence는 claim마다 + 오르므로 같은 worker id가 다시 claim해도 앞 실행의 쓰기는 0행이다(예: `DurableOperationClaim`). + - `WHERE id IN (SELECT … LIMIT n)`으로 고른 행을 바꿀 때는 subquery를 `FOR UPDATE SKIP LOCKED`로 잠가 남이 쥔 행을 + 고르지 않는다. 상태 조건도 바깥 `UPDATE … WHERE`에 한 번 더 둔다. 잠금 절이 빠지면 READ COMMITTED는 잠금을 기다린 + 행에 바깥 조건만 다시 보고 subquery 안 조건은 보지 않기 때문이다(예: `RecipientClaimSql.EXPIRE_OVERDUE`). 잠그지 않는 + subquery라면 바깥 조건이 유일한 방어다(예: inbox `markAllRead`). + - 만료는 그 시각을 쓴 시계로 본다. row lease(outbox, 알림 수신자, durable operation, fileserver cleanup, + Mongo migration lock)는 앱 `Clock`으로 쓰고 앱 `Clock`으로 비교한다. 시험이 시계를 옮길 수 있게 하려는 + 것이고, 대신 인스턴스 사이 시계 차이가 가장 짧은 lease보다 훨씬 작아야 한다(NTP 동기화가 배포 조건이다). + DB가 쓴 시각은 DB 시계로(멱등성 `expires_at`은 `clock_timestamp()`로 쓰고 `now()`로 지운다), Redis가 쓴 + 시각은 Redis `TIME`으로 본다. 한 저장소 안에서 두 시계를 섞지 않는다. - 수정 충돌 → 버전 열(낙관적 잠금). 예: `DurableOperationEntity`(persistence-jpa)의 `@Version` 열 `row_version`. - **예외**: DB가 볼 수 없는 규칙(외부 시스템 상태)은 분산 락(`DistributedLockPort`)이나 멱등성 키(05장). - 동시성 테스트는 실 DB lane에서 동시 요청 N개 → 행 1개를 단언한다. diff --git a/docs/notification/configuration-reference.md b/docs/notification/configuration-reference.md index 1b298505..ca79ccd6 100644 --- a/docs/notification/configuration-reference.md +++ b/docs/notification/configuration-reference.md @@ -26,7 +26,7 @@ support leaf의 `ModuleDefaultsEnvironmentPostProcessor`가 가장 낮은 우선 | 속성 | 환경 변수 | 기본값 | 범위 | |---|---|---|---| | `dispatch.claim-batch-size` | `APP_NOTIFICATION_PLATFORM_CLAIM_BATCH_SIZE` | `50` | 1..1000 | -| `dispatch.lease-duration` | `APP_NOTIFICATION_PLATFORM_LEASE_DURATION` | `2m` | 양수, 유한 | +| `dispatch.lease-duration` | `APP_NOTIFICATION_PLATFORM_LEASE_DURATION` | `2m` | 양수, 유한. `poll-interval`과 켜진 provider profile의 `timeout`보다 길어야 기동한다 | | `dispatch.poll-interval` | `APP_NOTIFICATION_PLATFORM_POLL_INTERVAL` | `1s` | 양수, 유한 | | `dispatch.max-global-concurrency` | `APP_NOTIFICATION_PLATFORM_MAX_CONCURRENCY` | `64` | 양수 | | `dispatch.max-additional-attempts` | `APP_NOTIFICATION_PLATFORM_MAX_ADDITIONAL_ATTEMPTS` | `4` | 0 이상 | @@ -35,8 +35,11 @@ support leaf의 `ModuleDefaultsEnvironmentPostProcessor`가 가장 낮은 우선 모든 값에 상한이 있다. 「무제한」은 받는 설정이 아니다. -lease는 provider 호출과 그 시간 상한보다 길어야 한다. 그보다 짧으면, 살아 있는 worker가 아직 기다리는 전달을 두 번째 -worker가 claim하고, 수신자는 알림을 두 번 받는다. +lease는 provider 호출 한 번이 delivery를 쥐는 시간보다 길어야 한다. 그보다 짧으면 아직 기다리는 호출이 멈춘 lease로 +보인다. lease가 끝난 뒤 lease만큼 더 지나도 호출이 끝나지 않았으면, lease 복구가 그 delivery를 버려진 것으로 보고 맞추기로 +넘긴다. 기본 제공 provider(SES·SMTP)는 결과 조회가 없어 운영자 확인(`RECONCILIATION_REQUIRED`)으로 넘어간다. 그 뒤에 +끝난 호출도 자기 결과를 기록한다(owner와 fence가 그대로다). 두 번째 worker가 다시 보내지는 않는다. claim은 +`DISPATCHING`인 delivery를 가져가지 않는다. ## 콜백 diff --git a/docs/registries/env-keys.yaml b/docs/registries/env-keys.yaml index 3df26fc2..99a9b670 100644 --- a/docs/registries/env-keys.yaml +++ b/docs/registries/env-keys.yaml @@ -5063,7 +5063,8 @@ env_keys: required_test: adapter-contract:notification-dispatch-bounds - name: APP_NOTIFICATION_PLATFORM_LEASE_DURATION - # source: NTF-025 — provider 호출과 그 timeout보다 길어야 한다. 아니면 살아 있는 worker의 전달을 다른 worker가 claim한다 + # source: NTF-025 — poll-interval과 켜진 provider profile의 timeout보다 길어야 기동한다(2026-11-13부터 검사). + # 짧으면 살아 있는 worker의 전달을 다른 worker가 claim한다. 넉넉히 길게 둔다: lease는 claim 때 시작한다 type: duration default: 2m allowed_values: null diff --git a/docs/upgrade/2026-11-round9-claim-fence.md b/docs/upgrade/2026-11-round9-claim-fence.md new file mode 100644 index 00000000..99ae53f9 --- /dev/null +++ b/docs/upgrade/2026-11-round9-claim-fence.md @@ -0,0 +1,38 @@ +# 업그레이드 안내 — 2026-11 claim·fence·트랜잭션 시한·정리 + +- 범위: durable operation·알림 수신자·fileserver cleanup·멱등성 기록의 lease, fence, 조건부 쓰기, 정리를 고친 + 변경. 포트 시그니처와 기동 조건이 바뀐 곳이 있다. 아래 표가 그 목록이다. +- 근거: 2026-11-12 전체 리뷰(`docs/reviews/2026-11-12-round9-review/`)의 claim·fence(B8) 발견 + +## 1. 누가 읽나 + +이 템플릿을 `upstream` remote로 두고 merge해 받는 파생 서비스가 읽는다. 특히 durable operation worker를 직접 쓴 서비스, +알림 recipient store를 직접 구현한 서비스, 알림 provider timeout을 설정한 서비스다. + +## 2. 할 일 + +| # | 할 일 | 안 하면 | 해당 서비스 | +| --- | --- | --- | --- | +| 1 | durable operation worker는 `claimNext`가 돌려주는 `DurableOperationClaim`을 들고 있다가 `heartbeat(claim, …)`·`reportProgress(claim, …)`·`succeed(claim, …)`·`fail(claim, …)`에 넘긴다. 예전에는 `(operationId, workerId, …)`였다 | 컴파일이 실패한다 | durable operation worker를 쓴 서비스 | +| 2 | `DurableOperationStorePort`를 직접 구현했으면 claim마다 오르는 fence를 저장하고, 네 쓰기가 owner와 fence를 함께 확인하게 한다. `DurableOperationStoreContract`의 새 사례(같은 worker id가 다시 claim한 뒤 앞 실행의 쓰기는 거절)로 확인한다 | 재시작한 worker나 한 프로세스의 두 실행이 같은 id를 쓰면 앞 실행의 결과가 넘겨받은 실행 위에 기록된다 | 자기 durable operation store를 둔 서비스 | +| 3 | persistence-jpa의 `V23__durable_operation_lease_fence.sql`(`durable_operation.lease_fence BIGINT NOT NULL DEFAULT 0`)이 기본 stream에 더해진다. 같은 번호의 서비스 마이그레이션이 있으면 번호를 옮긴다. `durable_operation`을 건드린 트랜잭션이 열려 있으면 5초 뒤 실패하고 V22에 남는다(다시 배포) | Flyway가 번호 충돌로 기동하지 않는다 | persistence-jpa를 쓰는 모든 서비스 | +| 4 | `RecipientLeaseStorePort.expiredDispatching`이 `List`(버려질 때의 owner·fence·만료 시각)를 돌려주고, `RecipientDeliveryStorePort`에 `requeueAbandoned(lease, nextDispatchAt)`가 생겼다. 직접 구현했으면 상태가 `DISPATCHING`이고 owner·fence가 같고 lease가 늘지 않았을 때만 `READY_TO_DISPATCH`로 바꾼다(`RecipientDeliveryStoreContract` 새 사례). owner가 없거나 fence가 0인 `DISPATCHING` 행(notification V4 전에 남은 행)은 복구가 더는 고르지 않는다. 그런 행은 `SELECT id FROM notification_recipient_delivery WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 찾는다. 배포 전에 `UPDATE notification_recipient_delivery SET lease_owner = COALESCE(lease_owner, 'pre-v4'), lease_fence = lease_fence + 1, lease_until = now() - interval '1 day', version = version + 1, updated_at = now() WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 owner를 채우고 fence를 올려 복구 패스에 넘긴다. 복구는 attempt를 보고 정한다: attempt가 없으면 다시 보낼 차례로 돌리고, 끝나지 않은 attempt가 있으면 맞추기로 넘긴다(그 뒤는 `docs/notification/configuration-reference.md` 「발송」의 lease 문단). 손으로 `READY_TO_DISPATCH`로 바꾸지 않는다. 발송 중에 멈춘 행이면 같은 알림이 한 번 더 나간다. owner가 없는 행은 stuck lease 지표에도 잡히지 않는다 | 컴파일이 실패한다. 조건 없이 바꾸면 끝났거나 다시 claim된 delivery가 다시 발송된다 | 알림 store를 직접 구현한 서비스 | +| 5 | 알림 provider profile의 `timeout`이 `app.notification.platform.dispatch.lease-duration`보다 짧은지 확인한다. 짧은 것은 필요조건일 뿐이다: lease는 claim 때 시작해서 attempt 기록·렌더링·provider 허가 대기도 lease를 쓴다. lease는 timeout보다 넉넉히 길게 둔다. 이 timeout이 provider 호출 전체를 묶는 것은 SES(API 호출 시간 상한)다. SMTP는 `spring.mail` socket timeout이 명령마다 묶을 뿐이라 이 검사만으로는 호출 전체가 lease 안에 끝난다고 말할 수 없다 | 짧지 않으면 기동이 실패한다(`notification provider profile '…': timeout (…) must be shorter than …lease-duration`). 전에는 provider 호출 하나가 lease보다 길어도 기동했다. 그때 생기는 일은 `docs/notification/configuration-reference.md` 「발송」의 lease 문단이다 | 알림 capability를 켠 서비스 | +| 6 | `IdempotencyReaper`를 직접 만들던 코드는 `new IdempotencyReaper(repository)`로 바꾼다(`Clock` 인자 삭제). 만료 비교는 DB 시계(`now()`)이고 1000행씩 지운다 | 컴파일이 실패한다 | reaper를 직접 조립한 서비스 | +| 7 | fileserver cleanup 묶음의 `max-items`를 정할 때, 한 묶음이 claim lease(10분)의 첫 8분 안에 시작할 수 있는 만큼만 처리된다는 것을 안다. 나머지는 시도하지 않은 채 queue에 돌아가 다음 묶음이 다시 claim한다. 시도 횟수는 그대로다(바이트 예산이 바닥나 돌려준 항목도 이제 같다. 전에는 시도 한 번으로 세어 8번째에 보류됐다). 직접 구현한 `CleanupQueuePort`는 새 연산 `release`를 구현한다(`CleanupQueuePortContract` 새 사례) | 느린 저장소에서 큰 `max-items`는 그 사유로 도는 항목이 늘어난다. 전에는 lease가 끝난 뒤에도 남은 항목을 처리해 다른 worker와 겹쳤다 | fileserver capability를 켠 서비스 | + +## 3. 바뀐 동작 (따라 할 일 없음) + +- reaper 둘(outbox·멱등성)의 트랜잭션은 `SpringTransaction.DEFAULT_TIMEOUT_SECONDS`(30초)로 묶인다. + `app.persistence-jpa.transaction.*`를 바꿔도 reaper 시한은 그대로다. 한 트랜잭션이 1000행 한 묶음만 지운다. +- 저장소가 행에서 만드는 `DurableOperationClaim`·`OperationLease`·`RecipientLease`·`OutboxLease`·`ClaimedOutboxEvent`· + `ReconciliationJob`·`AdminOperationClaim`이 값을 거절할 때 `IllegalStateException`을 던진다 + (전에는 `IllegalArgumentException`). 요청에서 오는 값이 아니라 서버 결함이다. 이 IAE를 잡던 코드는 ISE를 잡는다. +- 알림 lease 회복은 버려질 때의 lease 그대로일 때만 delivery를 다시 넣는다. 다른 인스턴스가 이미 다시 넣었거나 holder가 + 끝냈으면 아무것도 쓰지 않고 처리 수에 세지 않는다. +- 알림 만료 sweep(`EXPIRE_OVERDUE`)은 다른 트랜잭션이 쥔 행을 건너뛰고, 바깥 `UPDATE`에서도 상태·만료를 다시 본다. + inbox 전체 읽음(`markAllRead`)도 바깥 `UPDATE`에서 상태를 다시 본다. 기다리는 사이 claim된 delivery나 보관된 inbox 항목을 + 바꾸지 않는다. +- 멱등성 reaper는 다른 트랜잭션이 쥔 기록을 건너뛰고, 바깥 `DELETE`에서도 만료를 다시 본다. 같은 키 요청이 만료된 기록을 + 되살리는 동안 돈 reap이 그 claim을 지우지 않는다. +- fileserver recovery 대기열 enqueue는 `INSERT … ON CONFLICT` 한 문장이다. 같은 파일을 동시에 넣어도 25P02로 실패하지 않는다. diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettings.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettings.java index 15e9eb5d..19a86e65 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettings.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettings.java @@ -37,6 +37,8 @@ public record NotificationPlatformSettings( providers = providers == null ? Map.of() : Map.copyOf(providers); boolean callbacksEnabled = callbacks.enabled(); providers.forEach((id, provider) -> provider.validate(id, callbacksEnabled)); + Duration leaseDuration = dispatch.leaseDuration(); + providers.forEach((id, provider) -> provider.requireCallFitsInLease(id, leaseDuration)); } /** Source-compatible constructor for callers that rely on the original settings shape. */ @@ -264,6 +266,40 @@ public record NotificationPlatformSettings( } } + /** + * Fail the boot when one call of this profile alone would outlast the dispatch lease. + * + *

A worker holds a delivery's lease while it waits on the provider, and the profile's + * timeout is how long that wait can be. A lease that ends first shows a live call as a stuck + * lease; if the call is still running one lease later, lease recovery takes the delivery for + * abandoned and hands it to reconciliation, and the shipped families (SES, SMTP) have no status + * query, so it is parked for an operator. A call that finishes after that still records its + * result, since it holds the same owner and fence. No second worker claims it: the claim never + * takes a dispatching delivery. Each value is fine on its own, so only the pair can be checked + * (round9 review 2026-11-12, RR9-01-24). + * + *

A necessary condition, not a sufficient one: the lease starts at the claim, so writing the + * attempt, rendering and waiting for a provider permit spend it before the call begins. A + * deployment keeps the lease well above the timeout rather than just above it. + * + * @param profileId the profile id + * @param leaseDuration {@code app.notification.platform.dispatch.lease-duration} + */ + void requireCallFitsInLease(String profileId, Duration leaseDuration) { + if (!enabled) { + return; + } + require( + timeout.compareTo(leaseDuration) < 0, + profileId, + "timeout (" + + timeout + + ") must be shorter than app.notification.platform.dispatch.lease-duration (" + + leaseDuration + + "): a provider call that outlasts its lease shows as a stuck lease, and one lease" + + " later lease recovery hands its delivery to reconciliation while it may still run"); + } + private static void require(boolean condition, String profileId, String message) { if (!condition) { throw new IllegalArgumentException( diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationDispatchProperties.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationDispatchProperties.java index 7cafab9b..cf00636b 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationDispatchProperties.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationDispatchProperties.java @@ -42,8 +42,9 @@ public record NotificationDispatchProperties( requirePositive(estimatedDispatchDuration, "estimatedDispatchDuration"); requirePositive(maxQueueAge, "maxQueueAge"); if (leaseDuration.compareTo(pollInterval) <= 0) { - // A lease shorter than the poll interval expires before the worker can renew it, so two - // workers would routinely claim the same job. + // A sanity bound, not a renewal rule (see RecipientLeaseStorePort.renew). The lease is how + // long one dispatch may hold a delivery and the poll interval only how long an idle worker + // sleeps, so a lease no longer than that nap is too short for any real send. throw new IllegalArgumentException("leaseDuration must exceed pollInterval"); } } diff --git a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettingsTest.java b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettingsTest.java index 05182ad6..91a446d6 100644 --- a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettingsTest.java +++ b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettingsTest.java @@ -31,6 +31,39 @@ class NotificationPlatformSettingsTest { .hasMessageContaining("unknown provider type '" + removedType + "'"); } + @Test + void aProfileWhoseTimeoutIsNotShorterThanTheLeaseFailsStartup() { + // given + Duration lease = NotificationPlatformSettings.Dispatch.defaults().leaseDuration(); + NotificationPlatformSettings.Provider slow = + new NotificationPlatformSettings.Provider( + "SMTP", true, false, "PRODUCTION", "smtp-key", lease, 8, 20, null); + + // when + Throwable failure = catchThrowable(() -> properties(slow)); + + // then + assertThat(failure) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("notification provider profile 'profile'") + .hasMessageContaining("must be shorter than app.notification.platform.dispatch.lease"); + } + + @Test + void aDisabledProfileIsNotHeldToTheLease() { + // given + Duration lease = NotificationPlatformSettings.Dispatch.defaults().leaseDuration(); + NotificationPlatformSettings.Provider off = + new NotificationPlatformSettings.Provider( + "SMTP", false, false, "PRODUCTION", "smtp-key", lease.multipliedBy(2), 8, 20, null); + + // when + NotificationPlatformSettings settings = properties(off); + + // then + assertThat(settings.providers()).containsOnlyKeys("profile"); + } + @Test void enabledSesRequiresItsTypedTransportIdentity() { assertThatThrownBy( diff --git a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaCleanupQueue.java b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaCleanupQueue.java index 8912277d..56eed895 100644 --- a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaCleanupQueue.java +++ b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaCleanupQueue.java @@ -118,6 +118,12 @@ public class JpaCleanupQueue implements CleanupQueuePort { settle(item, STATUS_FAILED, nextAttemptAt, reasonCode, clock.instant()); } + @Override + public void release(CleanupItem item, Instant nextAttemptAt) { + CallerTransaction.require("cleanup queue release"); + items.releaseClaim(item.cleanupId(), item.claimToken(), nextAttemptAt, clock.instant()); + } + @Override public void markAbandoned(CleanupItem item, String reasonCode) { CallerTransaction.require("cleanup queue markAbandoned"); diff --git a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaRecoveryQueue.java b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaRecoveryQueue.java index e56faffe..caa3ce2b 100644 --- a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaRecoveryQueue.java +++ b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaRecoveryQueue.java @@ -7,7 +7,6 @@ import dev.caskeleton.application.fileserver.api.FileId; import dev.caskeleton.application.fileserver.recovery.ReconciliationStatus; import dev.caskeleton.application.fileserver.recovery.RecoveryQueuePort; import java.time.Clock; -import java.time.Instant; import java.util.List; import java.util.UUID; import org.springframework.data.domain.Limit; @@ -15,10 +14,11 @@ import org.springframework.data.domain.Limit; /** * Durable recovery queue over {@code fs_recovery_item}. * - *

An enqueue is an upsert: the same file reported twice updates the open item rather than adding - * a second one. Reconciliation runs on a schedule and re-raises whatever it still cannot settle, so - * an append-only queue would grow one row per sweep per unresolved file and bury the distinct - * problems under repetitions of the same one. + *

An enqueue is an upsert, one {@code INSERT … ON CONFLICT} statement: the same file reported + * twice — in sequence or in two concurrent transactions — updates the open item rather than adding + * a second one or failing on the open-item index. Reconciliation runs on a schedule and re-raises + * whatever it still cannot settle, so an append-only queue would grow one row per sweep per + * unresolved file and bury the distinct problems under repetitions of the same one. * *

Resolution keeps the outcome rather than deleting the row. {@code UNRESOLVED} and {@code * QUARANTINE_REQUIRED} are the two answers a human has to act on, and both are worthless if the @@ -26,8 +26,6 @@ import org.springframework.data.domain.Limit; */ public class JpaRecoveryQueue implements RecoveryQueuePort { - private static final String STATUS_PENDING = "PENDING"; - private final FileserverRecoveryRepository items; private final Clock clock; @@ -39,12 +37,7 @@ public class JpaRecoveryQueue implements RecoveryQueuePort { @Override public void enqueue(FileId fileId, String reasonCode) { CallerTransaction.require("recovery queue enqueue"); - Instant now = clock.instant(); - if (items.refreshPending(fileId.value(), reasonCode, now) > 0) { - return; - } - items.save( - new RecoveryItemEntity(UUID.randomUUID(), fileId.value(), reasonCode, STATUS_PENDING, now)); + items.upsertPending(UUID.randomUUID(), fileId.value(), reasonCode, clock.instant()); } @Override diff --git a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/entity/RecoveryItemEntity.java b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/entity/RecoveryItemEntity.java index 020901fe..5d4cd9e8 100644 --- a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/entity/RecoveryItemEntity.java +++ b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/entity/RecoveryItemEntity.java @@ -52,17 +52,6 @@ public class RecoveryItemEntity extends AssignedIdEntity { return recoveryId; } - public RecoveryItemEntity( - UUID recoveryId, UUID fileId, String reasonCode, String status, Instant createdAt) { - this.recoveryId = recoveryId; - this.fileId = fileId; - this.reasonCode = reasonCode; - this.status = status; - this.attempt = 0; - this.createdAt = createdAt; - this.updatedAt = createdAt; - } - public UUID getRecoveryId() { return recoveryId; } diff --git a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverCleanupRepository.java b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverCleanupRepository.java index 0e302f72..80105229 100644 --- a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverCleanupRepository.java +++ b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverCleanupRepository.java @@ -63,6 +63,32 @@ public interface FileserverCleanupRepository extends JpaRepositoryThe item goes back to {@code PENDING} when it was never tried and to {@code FAILED} when an + * earlier attempt was — the two statuses a claim takes from — and keeps its counter and its last + * error. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query( + """ + update CleanupItemEntity c + set c.status = case when c.attempt = 0 then 'PENDING' else 'FAILED' end, + c.nextAttemptAt = :nextAttemptAt, + c.claimOwner = null, + c.claimToken = null, + c.leaseUntil = null, + c.updatedAt = :now + where c.cleanupId = :cleanupId + and c.claimToken = :token + """) + int releaseClaim( + @Param("cleanupId") UUID cleanupId, + @Param("token") UUID token, + @Param("nextAttemptAt") Instant nextAttemptAt, + @Param("now") Instant now); + /** * Returns items whose claim has expired, for a reaper to take over. * diff --git a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverRecoveryRepository.java b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverRecoveryRepository.java index 88f8f885..49792a40 100644 --- a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverRecoveryRepository.java +++ b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/FileserverRecoveryRepository.java @@ -22,23 +22,34 @@ public interface FileserverRecoveryRepository extends JpaRepository findPending(Limit limit); /** - * Re-raises an open item instead of adding a second one. + * Opens an item for the file, or re-raises the open one instead of adding a second, in one + * statement. * *

Reconciliation is retried on a schedule, so the same file reaches the queue repeatedly. One * open item per file keeps the queue a worklist rather than a failure log; the newest reason wins * because it describes the most recent evidence. + * + *

One {@code INSERT … ON CONFLICT} on the partial unique index {@code uq_fs_recovery_open} + * rather than an update followed by an insert when it matched nothing. Two enqueues of the same + * file in concurrent transactions both saw no open row, both inserted, and the second hit the + * index — a unique violation that aborts its transaction ({@code 25P02}) and fails the sweep that + * reported it. Here the second waits for the first to commit and then updates the row it wrote. */ @Modifying(clearAutomatically = true, flushAutomatically = true) @Query( - """ - update RecoveryItemEntity r - set r.reasonCode = :reasonCode, - r.attempt = r.attempt + 1, - r.updatedAt = :now - where r.fileId = :fileId - and r.status = 'PENDING' - """) - int refreshPending( + value = + """ + INSERT INTO fs_recovery_item + (recovery_id, file_id, reason_code, status, attempt, created_at, updated_at) + VALUES (:recoveryId, :fileId, :reasonCode, 'PENDING', 0, :now, :now) + ON CONFLICT (file_id) WHERE status = 'PENDING' + DO UPDATE SET reason_code = EXCLUDED.reason_code, + attempt = fs_recovery_item.attempt + 1, + updated_at = EXCLUDED.updated_at + """, + nativeQuery = true) + int upsertPending( + @Param("recoveryId") UUID recoveryId, @Param("fileId") UUID fileId, @Param("reasonCode") String reasonCode, @Param("now") Instant now); diff --git a/src/adapter/outbound/persistence-jpa/AGENTS.md b/src/adapter/outbound/persistence-jpa/AGENTS.md index e34653d4..26c31492 100644 --- a/src/adapter/outbound/persistence-jpa/AGENTS.md +++ b/src/adapter/outbound/persistence-jpa/AGENTS.md @@ -88,6 +88,9 @@ lock TTL)과 계약 표(트랜잭션 모드, SQLState → 오류 코드, vendor, `lease_token`을 올린다(`V17`). `markPublished`/`markFailed`/`markDead`는 `event_id` + `IN_FLIGHT` + owner + token에 대한 조건부 UPDATE 하나이고 `APPLIED`/`STALE`을 돌려준다. 완료는 행을 읽어서 덮어쓰지 않는다. V2 outbox 세대는 2026-09-25에 지웠다(`docs/capability-status.md`). +- durable operation fencing: `claimNext`가 claim의 행 lock 아래에서 `lease_fence`를 올리고(`V23`) `DurableOperationClaim` + 으로 돌려준다. heartbeat·progress·succeed·fail은 operation id + `RUNNING` + owner + fence에 대한 조건부 UPDATE 하나다. + 같은 worker id가 다시 claim하면 앞 실행의 쓰기는 0행이다. - `TransactionPort` 구현(`SpringTransaction`) — 애플리케이션의 트랜잭션 의도와 Spring `PlatformTransactionManager` 사이의 다리. - 감사 metadata 기반 + actor 이음매(`audit/AuditableEntity`, `audit/AuditContextPort`, `audit/DomainContextAuditContext`) — diff --git a/src/adapter/outbound/persistence-jpa/README.md b/src/adapter/outbound/persistence-jpa/README.md index 403e60e6..9269264e 100644 --- a/src/adapter/outbound/persistence-jpa/README.md +++ b/src/adapter/outbound/persistence-jpa/README.md @@ -389,6 +389,23 @@ PUBLISHED backlog 를 한 transaction 으로 지우면 그 사이 모든 row loc cadence 측정 필요). retention 한 값은 reaper-local 이라 `@Value` 로 받지만, canonical 6-property 문서는 app-bootstrap `OutboxSettings` / `application.yml` 에 있다. +## operation — `JpaDurableOperationStore` + +### claim과 fence (`V23`) + +lease가 지난 실행은 아직 살아서 일을 끝내는 중일 수 있다. 예전 heartbeat·progress·succeed·fail은 +`lease_owner = :workerId`만 확인해서, 같은 worker id가 다시 claim하면(재시작한 worker, 한 프로세스의 두 실행) +앞 실행의 결과가 넘겨받은 실행 위에 기록됐다(round9 리뷰 2026-11-12, RR9-01-02). outbox(`lease_token`), +알림 수신자(`lease_fence`), reconciliation(`claim_fence`), fileserver cleanup(`claim_token`)과 같은 모양으로 맞췄다. + +- **claim**: `claimNext(workerId, …)`가 row lock(`FOR UPDATE SKIP LOCKED`)을 쥔 상태에서 `lease_owner`, + `lease_fence = lease_fence + 1`을 기록하고 `DurableOperationClaim(submission, workerId, fence)`을 돌려준다. +- **쓰기**: heartbeat·progress·succeed·fail은 한 문장의 조건부 UPDATE다 — + `WHERE operation_id = ? AND state = 'RUNNING' AND lease_owner = ? AND lease_fence = ?`(heartbeat는 + `lease_expires_at > now`도). 1행이면 적용, 0행이면 그 claim은 더 이상 이 operation을 쥐지 않는다. +- fence는 줄지 않는다. reclaim·cancel·expire는 owner를 비우고 fence는 그대로 둔다. `V23` 전에 쓴 행은 0이고, + claim은 1 이상을 나눠 주므로 fence 0을 쥔 worker는 없다. + ## JPA 운영 capability 후보 `src/adapter/outbound/persistence-jpa/release/readiness-cards.yaml`이 capability와 독립 schema stream의 @@ -533,7 +550,7 @@ provider 선택(flag → bean → registry)의 정본은 설정 코드 `lock/Dis 반환 핸들은 `lock::unlock` 이다. lease TTL 이 `close()` 전에 만료된 `JdbcLockRegistry` 의 경우, 내부 `JdbcLock.unlock()` 이 `ConcurrentModificationException` 을 던진다(row 가 이미 회수됨). 이 어댑터는 여기서 일부러 잡지 않는다 — metered `distributedLockProvider` 데코레이터 -(컴포지션 루트)가 SI-LOCK-C5 계약을 소유한다: 로그 + `lock.lease.expired` metric 후 `close()` 에서 +(이 모듈의 `DistributedLockPersistenceConfig`가 감싼 `MeteredDistributedLock`)가 SI-LOCK-C5 계약을 소유한다: 로그 + `lock.lease.expired` metric 후 `close()` 에서 정상 return 해, 만료가 호출자의 보호작업 예외를 가리지 않게 한다. in-process `DefaultLockRegistry` 경로는 만료가 없어 그 `close()` 가 이 예외를 던질 수 없다. @@ -554,11 +571,11 @@ provider 선택(flag → bean → registry)의 정본은 설정 코드 `lock/Dis `DistributedLock.tryLock(waitTime, leaseTtl)` 로 전달한다. ### `LockSettings` -`app.lock.*` 에서 바인딩되는 yaml-only 기본값이다(`config/module-defaults/persistence-jpa.yml`). -새 `APP_*` env 키가 아니므로 `env-keys.yaml` 엔트리가 필요 없다. `DistributedLockPersistenceConfig`의 -`@EnableConfigurationProperties`가 바인딩한다. cross-field invariant: `leaseTtl >= waitTime` -이어야 한다 — TTL 이 waitTime 보다 먼저 만료되면 첫 holder 의 보호작업이 끝나기 전에 두 번째 -holder 가 락을 잡을 수 있다. +`app.lock.lease-ttl` 하나를 바인딩하는 yaml-only 기본값이다(`config/module-defaults/persistence-jpa.yml`). +새 `APP_*` env 키가 아니므로 `env-keys.yaml` 엔트리가 필요 없다. `DistributedLockPersistenceConfig#lockSettings`가 +`StrictSettingsBinder`로 바인딩해, `app.lock` 아래 모르는 키는 기동을 실패시킨다. 기다릴 시간은 호출자가 +`tryAcquire(key, waitTime, leaseTtl)`마다 넘긴다. 예전 `app.lock.wait-time`은 lease와 비교만 하고 읽는 곳이 +없었다(round9 리뷰 2026-11-12, RR9-01-07). 지금은 모르는 키라 설정해 두면 기동이 실패한다. ### `MeteredDistributedLock` (app-bootstrap에서 이동) - **`DistributedLockPort`를 감싸는 얇은 Micrometer 데코레이터다.** `tryAcquire` 호출마다 diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java index 18964a00..193187e5 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java @@ -154,8 +154,8 @@ class JpaAdapterBeansConfig { static class IdempotencyExpiry { @Bean - IdempotencyReaper idempotencyReaper(IdempotencyRecordJpaRepository repository, Clock clock) { - return new IdempotencyReaper(repository, clock); + IdempotencyReaper idempotencyReaper(IdempotencyRecordJpaRepository repository) { + return new IdempotencyReaper(repository); } } } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaper.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaper.java index 3e754839..053baef9 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaper.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaper.java @@ -1,32 +1,41 @@ package dev.caskeleton.adapter.outbound.persistence.idempotency.jpa; -import java.time.Clock; +import dev.caskeleton.adapter.outbound.persistence.transaction.BatchedDeletes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.transaction.annotation.Transactional; /** * Periodically deletes idempotency rows whose {@code expires_at} has passed. The PostgreSQL store * moves {@code expires_at} forward on every completion and failure, and a claim is reset in place * rather than re-inserted, so this is the only thing that bounds table growth. + * + *

Deletes in batches of {@link #BATCH_SIZE}, each committed on its own, until a batch comes back + * short — the outbox reaper's shape. Expiry is the database's clock, not this process's: the store + * wrote {@code expires_at} from {@code clock_timestamp()} (see {@link + * IdempotencyRecordJpaRepository#deleteExpiredBatch}). */ public class IdempotencyReaper { + /** Rows per delete statement and transaction. */ + static final int BATCH_SIZE = BatchedDeletes.BATCH_SIZE; + private static final Logger log = LoggerFactory.getLogger(IdempotencyReaper.class); private final IdempotencyRecordJpaRepository repository; - private final Clock clock; - public IdempotencyReaper(IdempotencyRecordJpaRepository repository, Clock clock) { + public IdempotencyReaper(IdempotencyRecordJpaRepository repository) { this.repository = repository; - this.clock = clock; } + /** + * Deletes every record whose expiry has passed by the database clock; returns the count. + * + *

Not transactional itself: each batch is its own transaction. + */ @Scheduled(fixedDelayString = "${app.capabilities.idempotency.reaper-interval:PT10M}") - @Transactional public int reap() { - int purged = repository.deleteExpired(clock.instant()); + int purged = BatchedDeletes.untilShort(repository::deleteExpiredBatch); if (purged > 0) { log.debug("idempotency reaper purged {} expired record(s)", purged); } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyRecordJpaRepository.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyRecordJpaRepository.java index 22a07ca7..0a8183d7 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyRecordJpaRepository.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyRecordJpaRepository.java @@ -1,12 +1,13 @@ package dev.caskeleton.adapter.outbound.persistence.idempotency.jpa; import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.entity.IdempotencyRecordEntity; -import java.time.Instant; +import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; import java.util.UUID; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.Repository; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; /** * Spring Data repository for {@link IdempotencyRecordEntity}: the reaper's expiry purge, and @@ -18,8 +19,34 @@ import org.springframework.data.repository.query.Param; */ public interface IdempotencyRecordJpaRepository extends Repository { - /** Reaper / TTL boundary (§E): purge every record whose expiry is at/before {@code now}. */ + /** + * Reaper / TTL boundary (§E): deletes at most {@code limit} records whose expiry has passed, in a + * transaction of its own unless the caller already has one; returns the count. + * + *

Expiry is judged by the database clock ({@code now()}) because that is the clock that wrote + * it: the PostgreSQL store sets {@code expires_at} from {@code clock_timestamp()}. Compared with + * the application's clock instead, a host whose clock runs ahead purged a record whose lease or + * replay window was still open. + * + *

Bounded for the reason the outbox purge is: a backlog of expired rows is what a reaper + * outage leaves behind, and one unbounded {@code DELETE} over it is a long transaction holding + * every deleted row's lock. + * + *

A same-key request reclaims an expired record in place (the store's reset). The delete must + * not take that claim: the next retry would run the work again. Two things stop it. The subquery + * locks the rows it picks and skips rows another transaction holds, so a record being reclaimed + * is not picked. The expiry is also a condition of the {@code DELETE} itself, so if the lock + * clause is ever lost — and a delete waits on the row and, under READ COMMITTED, re-checks only + * its own predicate — the record it waited for is still kept once it is no longer expired. + */ + @Transactional(timeout = SpringTransaction.DEFAULT_TIMEOUT_SECONDS) @Modifying - @Query("delete from IdempotencyRecordEntity e where e.expiresAt <= :now") - int deleteExpired(@Param("now") Instant now); + @Query( + value = + "DELETE FROM idempotency_record WHERE id IN (" + + "SELECT id FROM idempotency_record WHERE expires_at <= now()" + + " LIMIT :limit FOR UPDATE SKIP LOCKED)" + + " AND expires_at <= now()", + nativeQuery = true) + int deleteExpiredBatch(@Param("limit") int limit); } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationJpaRepository.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationJpaRepository.java index 3d17b6ab..38924d6b 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationJpaRepository.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationJpaRepository.java @@ -17,10 +17,11 @@ import org.springframework.data.repository.query.Param; * claim-then-check has every worker fetch the same head row and all but one retry, and a plain * {@code FOR UPDATE} makes them queue behind each other so the pool serialises. * - *

Every state-changing statement carries the owning worker in its {@code WHERE} clause and - * returns the affected count. A worker whose lease lapsed while it was still working must not be - * able to record a result over the worker that took over; the update simply matches nothing, and - * the caller learns it lost the lease from the count. + *

Every state-changing statement carries the owning worker and the fence of its claim in its + * {@code WHERE} clause and returns the affected count. A run whose lease lapsed while it was still + * working must not be able to record a result over the run that took over — even when both runs use + * the same worker id, which only the fence tells apart; the update simply matches nothing, and the + * caller learns it lost the lease from the count. */ public interface DurableOperationJpaRepository extends Repository { @@ -79,6 +80,8 @@ public interface DurableOperationJpaRepository extends RepositoryRunnable is PENDING, or RUNNING whose lease has lapsed. The second half is the crash * recovery path: without it an operation claimed by a worker that then died is never picked up * again, and the queue stops draining while every row still reads RUNNING. + * + * @return one row of {@code [operation_id, lease_fence]} when a row was claimed, none otherwise */ @Modifying(clearAutomatically = true, flushAutomatically = true) @Query( @@ -90,6 +93,7 @@ public interface DurableOperationJpaRepository extends Repository claimNext( + List claimNext( @Param("workerId") String workerId, @Param("leaseExpiresAt") Instant leaseExpiresAt, @Param("now") Instant now); @@ -119,12 +123,14 @@ public interface DurableOperationJpaRepository extends Repository :now """, nativeQuery = true) int heartbeat( @Param("operationId") String operationId, @Param("workerId") String workerId, + @Param("fence") long fence, @Param("leaseExpiresAt") Instant leaseExpiresAt, @Param("now") Instant now); @@ -141,11 +147,13 @@ public interface DurableOperationJpaRepository extends Repository claimNext( + public Optional claimNext( String workerId, Duration leaseDuration, Instant now) { requireCallerTransaction("claimNext"); - List claimed = repository.claimNext(workerId, now.plus(leaseDuration), now); + List claimed = repository.claimNext(workerId, now.plus(leaseDuration), now); if (claimed.isEmpty()) { return Optional.empty(); } + String operationId = (String) claimed.get(0)[0]; + long fence = ((Number) claimed.get(0)[1]).longValue(); return repository - .findById(claimed.get(0)) + .findById(operationId) .map( entity -> - new DurableOperationSubmission( - new DurableOperationId(entity.getOperationId()), - entity.getOperationName(), - entity.getPrincipal(), - entity.getTenant().isEmpty() ? null : entity.getTenant(), - new RequestFingerprint(entity.getRequestHash()), - entity.getPayload(), - entity.getSubmittedAt(), - Duration.between(entity.getSubmittedAt(), entity.getExpiresAt()))); + new DurableOperationClaim( + new DurableOperationSubmission( + new DurableOperationId(entity.getOperationId()), + entity.getOperationName(), + entity.getPrincipal(), + entity.getTenant().isEmpty() ? null : entity.getTenant(), + new RequestFingerprint(entity.getRequestHash()), + entity.getPayload(), + entity.getSubmittedAt(), + Duration.between(entity.getSubmittedAt(), entity.getExpiresAt())), + workerId, + fence)); } @Override - public boolean heartbeat( - DurableOperationId operationId, String workerId, Duration leaseDuration, Instant now) { + public boolean heartbeat(DurableOperationClaim claim, Duration leaseDuration, Instant now) { requireCallerTransaction("heartbeat"); - return repository.heartbeat(operationId.value(), workerId, now.plus(leaseDuration), now) == 1; + return repository.heartbeat( + claim.operationId().value(), + claim.workerId(), + claim.fence(), + now.plus(leaseDuration), + now) + == 1; } @Override - public boolean reportProgress( - DurableOperationId operationId, String workerId, OperationProgressSnapshot progress) { + public boolean reportProgress(DurableOperationClaim claim, OperationProgressSnapshot progress) { requireCallerTransaction("reportProgress"); return repository.reportProgress( - operationId.value(), - workerId, + claim.operationId().value(), + claim.workerId(), + claim.fence(), progress.completedUnits(), progress.totalUnits().orElse(null), progress.phase().orElse(null)) @@ -124,25 +135,24 @@ public class JpaDurableOperationStore implements DurableOperationStorePort { } @Override - public boolean succeed( - DurableOperationId operationId, - String workerId, - String resultReference, - Instant completedAt) { + public boolean succeed(DurableOperationClaim claim, String resultReference, Instant completedAt) { requireCallerTransaction("succeed"); - return repository.succeed(operationId.value(), workerId, resultReference, completedAt) == 1; + return repository.succeed( + claim.operationId().value(), + claim.workerId(), + claim.fence(), + resultReference, + completedAt) + == 1; } @Override - public boolean fail( - DurableOperationId operationId, - String workerId, - OperationFailure failure, - Instant completedAt) { + public boolean fail(DurableOperationClaim claim, OperationFailure failure, Instant completedAt) { requireCallerTransaction("fail"); return repository.fail( - operationId.value(), - workerId, + claim.operationId().value(), + claim.workerId(), + claim.fence(), failure.code(), failure.detail(), failure.retryable(), diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStore.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStore.java index e23e34ca..9d8158b1 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStore.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStore.java @@ -378,7 +378,7 @@ public class JpaOutboxStore OutboxEventStatus.valueOf(entity.getStatus()), entity.getAttemptCount(), entity.getDeferredAttemptCount()); - } catch (IllegalArgumentException exception) { + } catch (IllegalArgumentException | IllegalStateException exception) { throw new IllegalStateException( "invalid canonical outbox metadata for eventId=" + entity.getEventId(), exception); } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxEventJpaRepository.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxEventJpaRepository.java index 9f4d6d0c..65de4485 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxEventJpaRepository.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxEventJpaRepository.java @@ -1,6 +1,7 @@ package dev.caskeleton.adapter.outbound.persistence.outbox.jpa; import dev.caskeleton.adapter.outbound.persistence.outbox.jpa.entity.OutboxEventEntity; +import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; import java.time.Instant; import java.util.List; import java.util.Optional; @@ -34,9 +35,10 @@ public interface OutboxEventJpaRepository extends RepositoryBounded because a backlog of published rows is exactly what a relay outage leaves behind, * and one unbounded {@code DELETE} over it is a long transaction holding every deleted row's - * lock. The key subquery is plain SQL that PostgreSQL and H2 both run. + * lock. The key subquery is plain SQL that PostgreSQL and H2 both run. The transaction has the + * default bound, as the other reapers' do. */ - @Transactional + @Transactional(timeout = SpringTransaction.DEFAULT_TIMEOUT_SECONDS) @Modifying @Query( value = diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxReaper.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxReaper.java index e9042775..a90f66cf 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxReaper.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxReaper.java @@ -1,5 +1,6 @@ package dev.caskeleton.adapter.outbound.persistence.outbox.jpa; +import dev.caskeleton.adapter.outbound.persistence.transaction.BatchedDeletes; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -18,7 +19,7 @@ public class OutboxReaper { /** Rows per delete statement and transaction. */ // ponytail: fixed batch size; make it a setting if a deployment's backlog needs another value. - static final int BATCH_SIZE = 1_000; + static final int BATCH_SIZE = BatchedDeletes.BATCH_SIZE; private static final Logger log = LoggerFactory.getLogger(OutboxReaper.class); @@ -41,12 +42,7 @@ public class OutboxReaper { @Scheduled(fixedDelayString = "${app.outbox.reaper-interval:PT10M}") public int reap() { Instant cutoff = clock.instant().minus(retention); - int purged = 0; - int deleted; - do { - deleted = repository.deletePublishedBatch(cutoff, BATCH_SIZE); - purged += deleted; - } while (deleted == BATCH_SIZE); + int purged = BatchedDeletes.untilShort(limit -> repository.deletePublishedBatch(cutoff, limit)); if (purged > 0) { log.debug("outbox reaper purged {} published row(s)", purged); } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/BatchedDeletes.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/BatchedDeletes.java new file mode 100644 index 00000000..4889a740 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/BatchedDeletes.java @@ -0,0 +1,36 @@ +package dev.caskeleton.adapter.outbound.persistence.transaction; + +import java.util.function.IntUnaryOperator; + +/** + * Deletes a backlog in bounded batches, one transaction each, until a batch comes back short. + * + *

Every reaper here has the same reason to: a backlog of expired rows is what a reaper outage + * leaves behind, and one unbounded {@code DELETE} over it is a long transaction holding every + * deleted row's lock. The outbox, idempotency and live-event reapers each held their own copy of + * this loop; one copy is the one that cannot drift. + */ +public final class BatchedDeletes { + + /** The most rows one batch deletes. */ + public static final int BATCH_SIZE = 1_000; + + private BatchedDeletes() {} + + /** + * Runs {@code batch} until it deletes fewer than {@link #BATCH_SIZE} rows. + * + * @param batch deletes at most its argument's number of rows, in a transaction of its own, and + * returns how many it deleted + * @return how many rows all the batches deleted + */ + public static int untilShort(IntUnaryOperator batch) { + int purged = 0; + int deleted; + do { + deleted = batch.applyAsInt(BATCH_SIZE); + purged += deleted; + } while (deleted == BATCH_SIZE); + return purged; + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransaction.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransaction.java index aad85fd9..10475d66 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransaction.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransaction.java @@ -115,6 +115,15 @@ public final class SpringTransaction implements TransactionPort { return execute(requiresNewTemplate, action); } + /** + * The default bound, in whole seconds, of every transaction this leaf opens: {@link + * Timeouts#DEFAULT}, and the reapers' own transactions, whose {@code @Transactional(timeout)} + * needs a compile-time constant. A deployment's {@code app.persistence-jpa.transaction.*} moves + * the first and not the second; a reaper deletes one bounded batch per transaction, so the + * default is ample for it whatever the application's units of work need. + */ + public static final int DEFAULT_TIMEOUT_SECONDS = 30; + /** * How long one {@link SpringTransaction} write or read transaction may run (review 04-10). * @@ -140,7 +149,9 @@ public final class SpringTransaction implements TransactionPort { /** 30 seconds each; see the class comment for why. */ public static final Timeouts DEFAULT = - new Timeouts(Duration.ofSeconds(30), Duration.ofSeconds(30)); + new Timeouts( + Duration.ofSeconds(DEFAULT_TIMEOUT_SECONDS), + Duration.ofSeconds(DEFAULT_TIMEOUT_SECONDS)); public Timeouts { requireWholePositiveSeconds(write, "write"); diff --git a/src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V23__durable_operation_lease_fence.sql b/src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V23__durable_operation_lease_fence.sql new file mode 100644 index 00000000..1aef212e --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V23__durable_operation_lease_fence.sql @@ -0,0 +1,24 @@ +-- A fence on the durable operation lease. +-- +-- Until now a heartbeat, a progress report and a completion checked only lease_owner. A worker that +-- restarts under the same id, or one process running two executions under one id, claims the +-- operation again, and the earlier run — still alive and still finishing — then matches the same +-- lease_owner and records its result over the run that took over. The outbox (lease_token), the +-- notification recipient lease (lease_fence), the reconciliation claim (claim_fence) and the +-- fileserver claim (claim_token) all fence; this was the one store that did not (round9 review +-- 2026-11-12, RR9-01-02). +-- +-- Every claim increments lease_fence and returns it; every later write names the fence it holds in +-- its WHERE clause. NOT NULL DEFAULT 0 is a constant default, so PostgreSQL 11+ records it in the +-- catalog and rewrites no row; rows written before this keep 0 until their next claim, and no +-- worker holds fence 0 because a claim hands out 1 or more. + +-- Bound the wait for the lock this migration takes (ACCESS EXCLUSIVE on durable_operation, held +-- until it commits). Unbounded, it queues behind any open transaction that has touched the table, +-- and every submit, poll and claim queues behind the migration. Five seconds, then the migration +-- fails, rolls back whole and the database stays at V22: deploy again once that transaction has +-- ended (docs/jpa/migration-guide.md "Bound the lock wait"). +SET LOCAL lock_timeout = '5s'; + +ALTER TABLE durable_operation + ADD COLUMN lease_fence BIGINT NOT NULL DEFAULT 0; diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/IdempotencyReaperIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/IdempotencyReaperIntegrationTest.java new file mode 100644 index 00000000..ec92e02d --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/IdempotencyReaperIntegrationTest.java @@ -0,0 +1,205 @@ +package dev.caskeleton.adapter.outbound.persistence.idempotency.postgresql; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.IdempotencyReaper; +import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.IdempotencyRecordJpaRepository; +import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; +import jakarta.persistence.EntityManagerFactory; +import java.sql.Connection; +import java.sql.Statement; +import java.time.Duration; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.data.jpa.repository.support.JpaRepositoryFactory; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.SharedEntityManagerCreator; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import org.springframework.transaction.TransactionManager; +import org.springframework.transaction.annotation.AnnotationTransactionAttributeSource; +import org.springframework.transaction.interceptor.TransactionInterceptor; + +/** + * The idempotency reaper's delete against a real PostgreSQL: expiry is the database clock's, and a + * backlog larger than one batch is drained. + * + *

The store writes {@code expires_at} from {@code clock_timestamp()}, so the rows here are + * written the same way, relative to the database's {@code now()}. A reaper that compared them with + * the application's clock purged a live record whenever the host clock ran ahead of the database's + * (round9 review 2026-11-12, RR9-01-04); the reaper now has no clock of its own to compare with. + * + *

The repository is the production shape: its own {@code @Transactional} opens each batch's + * transaction, through the same interceptor Spring Data installs, and no test transaction wraps the + * reap. + */ +@Tag("jpa-contract") +class IdempotencyReaperIntegrationTest { + + private static JpaPlatformContractSupport support; + private static JdbcTemplate jdbc; + private static LocalContainerEntityManagerFactoryBean persistence; + private static IdempotencyReaper reaper; + + @BeforeAll + static void start() { + support = JpaPlatformContractSupport.start(); + Flyway.configure() + .dataSource(support.dataSource()) + .locations("classpath:db/migration/postgresql") + .table("flyway_schema_history") + .baselineOnMigrate(false) + .outOfOrder(false) + .load() + .migrate(); + jdbc = new JdbcTemplate(support.dataSource()); + persistence = new LocalContainerEntityManagerFactoryBean(); + persistence.setDataSource(support.dataSource()); + persistence.setJpaVendorAdapter(new HibernateJpaVendorAdapter()); + persistence.setPackagesToScan( + "dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.entity"); + Properties properties = new Properties(); + properties.setProperty("hibernate.jdbc.time_zone", "UTC"); + persistence.setJpaProperties(properties); + persistence.afterPropertiesSet(); + EntityManagerFactory entityManagers = persistence.getObject(); + JpaTransactionManager transactionManager = new JpaTransactionManager(entityManagers); + JpaRepositoryFactory repositories = + new JpaRepositoryFactory( + SharedEntityManagerCreator.createSharedEntityManager(entityManagers)); + repositories.addRepositoryProxyPostProcessor( + (proxy, information) -> + proxy.addAdvice( + new TransactionInterceptor( + (TransactionManager) transactionManager, + new AnnotationTransactionAttributeSource()))); + reaper = + new IdempotencyReaper(repositories.getRepository(IdempotencyRecordJpaRepository.class)); + } + + @AfterAll + static void stop() { + if (persistence != null) { + persistence.destroy(); + } + if (support != null) { + support.close(); + } + } + + @BeforeEach + void empty() { + jdbc.update("delete from idempotency_record"); + } + + @Test + void aRecordTheDatabaseClockHasNotExpiredSurvivesTheReap() { + // given + insert("live", 1, "now() + interval '1 hour'"); + insert("expired", 1, "now() - interval '1 second'"); + + // when + reap(); + + // then + assertThat(jdbc.queryForList("select idempotency_key from idempotency_record", String.class)) + .containsExactly("live-0"); + } + + @Test + void backlogLargerThanOneBatchIsDrainedInOneReap() { + // given + insert("expired", 2_501, "now() - interval '1 minute'"); + + // when + int purged = reap(); + + // then + assertThat(List.of(purged, count())).containsExactly(2_501, 0); + } + + @Test + void aRecordReclaimedWhileTheReapRunsSurvivesIt() throws Exception { + // given + insert("reset", 1, "now() - interval '1 second'"); + try (Connection reclaim = support.dataSource().getConnection()) { + reclaim.setAutoCommit(false); + try (Statement statement = reclaim.createStatement()) { + // What the store's reset does to an expired record a same-key request finds: it holds the + // row and claims it again in place. + statement.executeUpdate( + "update idempotency_record set status = 'CLAIMED'," + + " expires_at = now() + interval '1 hour' where idempotency_key = 'reset-0'"); + } + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + Future reaping = executor.submit(() -> reap()); + awaitDoneOrBlocked(reaping, Duration.ofSeconds(5)); + reclaim.commit(); + + // when + reaping.get(30, TimeUnit.SECONDS); + } + } + + // then + assertThat(jdbc.queryForList("select status from idempotency_record", String.class)) + .as("the reap deleted the claim a same-key request had just taken") + .containsExactly("CLAIMED"); + } + + private static int reap() { + return reaper.reap(); + } + + /** + * Returns once the reap has finished or is waiting on a row lock, whichever comes first, and + * fails the test when neither happened in time: committing the reclaim before the reap had run + * would pass without the race this test exists for. + */ + private static void awaitDoneOrBlocked(Future reaping, Duration within) + throws InterruptedException { + long deadline = System.nanoTime() + within.toNanos(); + while (!reaping.isDone()) { + Integer waiting = + jdbc.queryForObject( + "select count(*) from pg_locks where not granted and locktype = 'transactionid'", + Integer.class); + if (waiting != null && waiting > 0) { + return; + } + if (System.nanoTime() >= deadline) { + throw new AssertionError( + "the reap neither finished nor waited on the reclaimed row within " + within); + } + Thread.sleep(20); + } + } + + private static int count() { + Integer rows = jdbc.queryForObject("select count(*) from idempotency_record", Integer.class); + return rows == null ? -1 : rows; + } + + private static void insert(String keyPrefix, int rows, String expiresAtSql) { + jdbc.update( + "insert into idempotency_record (id, principal, idempotency_key, use_case_name," + + " request_hash, status, created_at, expires_at)" + + " select gen_random_uuid(), 'alice', ? || '-' || n, 'orders.create'," + + " repeat('a', 64), 'COMPLETED', now(), " + + expiresAtSql + + " from generate_series(0, ? - 1) as n", + keyPrefix, + rows); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/PostgreSqlIdempotencyStoreContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/PostgreSqlIdempotencyStoreContractTest.java index cac6ff47..8825edf8 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/PostgreSqlIdempotencyStoreContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/idempotency/postgresql/PostgreSqlIdempotencyStoreContractTest.java @@ -7,7 +7,6 @@ import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction import dev.caskeleton.application.idempotency.IdempotencyStorePort; import dev.caskeleton.application.idempotency.testkit.IdempotencyStoreContract; import jakarta.persistence.EntityManagerFactory; -import java.time.Clock; import java.time.Duration; import java.util.Properties; import java.util.function.Supplier; @@ -85,7 +84,7 @@ class PostgreSqlIdempotencyStoreContractTest extends IdempotencyStoreContract { new JpaRepositoryFactory( SharedEntityManagerCreator.createSharedEntityManager(entityManagers)) .getRepository(IdempotencyRecordJpaRepository.class); - reaper = new IdempotencyReaper(repository, Clock.systemUTC()); + reaper = new IdempotencyReaper(repository); reaperTransactions = new TransactionTemplate(new JpaTransactionManager(entityManagers)); } diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationJpaStack.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationJpaStack.java index 8408e920..72b52075 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationJpaStack.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationJpaStack.java @@ -3,7 +3,10 @@ package dev.caskeleton.adapter.outbound.persistence.notification; import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; import jakarta.persistence.EntityManager; import jakarta.persistence.EntityManagerFactory; +import java.time.Duration; +import java.time.Instant; import java.util.Map; +import java.util.concurrent.Future; import java.util.function.Supplier; import org.flywaydb.core.Flyway; import org.springframework.data.jpa.repository.support.JpaRepositoryFactory; @@ -69,6 +72,33 @@ public final class NotificationJpaStack implements AutoCloseable { return new JdbcTemplate(database.dataSource()); } + /** + * Waits until {@code work} has finished or some session of this database is blocked on a lock. + * + *

A race case holds one transaction open and starts a second statement that may or may not + * need the first's row lock; this is how it knows the second has either finished or queued behind + * the first before it lets the first commit, without sleeping a guessed interval. + */ + public void awaitDoneOrBlocked(Future work, Duration deadline) { + Instant giveUp = Instant.now().plus(deadline); + JdbcTemplate jdbc = jdbc(); + while (Instant.now().isBefore(giveUp)) { + if (work.isDone()) { + return; + } + Integer waiting = + jdbc.queryForObject( + "SELECT count(*) FROM pg_stat_activity" + + " WHERE datname = current_database() AND wait_event_type = 'Lock'", + Integer.class); + if (waiting != null && waiting > 0) { + return; + } + Thread.onSpinWait(); + } + throw new IllegalStateException("work neither finished nor waited on a lock in " + deadline); + } + /** * Deletes every row of the delivery tables, children first, so a contract case starts empty. * diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/PostgreSqlRecipientLeaseStoreContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/PostgreSqlRecipientLeaseStoreContractTest.java index fe4134f7..a4bec546 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/PostgreSqlRecipientLeaseStoreContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/PostgreSqlRecipientLeaseStoreContractTest.java @@ -15,11 +15,17 @@ import dev.caskeleton.application.notification.platform.dispatch.NotificationReq import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryRecord; import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort; import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort; +import dev.caskeleton.application.notification.platform.dispatch.testkit.ContractClock; import dev.caskeleton.application.notification.platform.dispatch.testkit.NotificationRows; import dev.caskeleton.application.notification.platform.dispatch.testkit.RecipientLeaseStoreContract; import java.time.Duration; import java.util.List; import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -114,6 +120,72 @@ class PostgreSqlRecipientLeaseStoreContractTest extends RecipientLeaseStoreContr .isGreaterThan(before); } + @Test + void anExpirySweepRacingAClaimLeavesTheClaimedJobDispatching() throws Exception { + // given + NotificationRequestRecord request = + NotificationRows.request(new TenantId("tenant-a"), "idem-race", NOW); + RecipientDeliveryRecord job = + NotificationRows.job( + request, + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(NOW), + Optional.of(NOW.plusSeconds(10)), + NOW); + inTransaction(() -> requests().insert(request, List.of(job))); + CountDownLatch claimed = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + ExecutorService pool = Executors.newFixedThreadPool(2); + + // when + List outcome; + try { + // The claim holds its row lock, uncommitted, while the sweep runs at an instant past the + // job's expiry. The sweep used to pick the id from the committed READY row, wait for the + // lock, and then retire the row the claim had just made DISPATCHING. + CompletableFuture claim = + CompletableFuture.runAsync( + () -> + inTransaction( + () -> { + var unused = leases().claim("worker-a", 10, Duration.ofMinutes(5)); + claimed.countDown(); + awaitOrFail(release); + return null; + }), + pool); + awaitOrFail(claimed); + ((ContractClock) clock()).moveTo(NOW.plusSeconds(11)); + CompletableFuture sweep = + CompletableFuture.supplyAsync( + () -> inTransaction(() -> leases().expireOverdue(10)), pool); + stack.awaitDoneOrBlocked(sweep, Duration.ofSeconds(30)); + release.countDown(); + claim.get(30, TimeUnit.SECONDS); + outcome = List.of(sweep.get(30, TimeUnit.SECONDS), stateOf(job.id())); + } finally { + pool.shutdownNow(); + } + + // then + assertThat(outcome).containsExactly(0, RecipientDeliveryState.DISPATCHING); + } + + private RecipientDeliveryState stateOf(RecipientDeliveryId id) { + return deliveries().find(id).orElseThrow().state(); + } + + private static void awaitOrFail(CountDownLatch latch) { + try { + if (!latch.await(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("the other transaction did not arrive within 30 seconds"); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while waiting", interrupted); + } + } + private static long versionOf(RecipientDeliveryId id) { return stack .jdbc() diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxBulkReadVersionContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxBulkReadVersionContractTest.java index 76863346..4851dadc 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxBulkReadVersionContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxBulkReadVersionContractTest.java @@ -5,9 +5,17 @@ import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.persistence.notification.NotificationJpaStack; import dev.caskeleton.adapter.outbound.persistence.notification.platform.entity.InboxItemEntity; +import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.Timestamp; +import java.time.Duration; import java.time.Instant; +import java.util.List; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; @@ -74,6 +82,49 @@ class InboxBulkReadVersionContractTest { .isEqualTo("READ"); } + @Test + void anItemArchivedWhileTheBulkReadWaitsForItStaysArchived() throws Exception { + // given + UUID id = insertUnreadItem(); + ExecutorService pool = Executors.newSingleThreadExecutor(); + + // when + List outcome; + try (Connection archiving = stack.jdbc().getDataSource().getConnection()) { + // The archive holds the row uncommitted while mark-all-read runs. The bulk read picks the id + // from the committed unread row, waits for the lock, and used to mark READ the item the + // user had just archived — putting it back in the inbox. + archiving.setAutoCommit(false); + try (PreparedStatement archive = + archiving.prepareStatement( + "UPDATE notification_inbox_item SET state = 'ARCHIVED', version = version + 1" + + " WHERE id = ?")) { + archive.setObject(1, id); + archive.executeUpdate(); + } + CompletableFuture bulkRead = + CompletableFuture.supplyAsync( + () -> + stack.inTransaction( + () -> items.markAllRead("tenant-a", "user-1", null, null, 100, NOW)), + pool); + stack.awaitDoneOrBlocked(bulkRead, Duration.ofSeconds(30)); + archiving.commit(); + outcome = List.of(bulkRead.get(30, TimeUnit.SECONDS), stateOf(id)); + } finally { + pool.shutdownNow(); + } + + // then + assertThat(outcome).containsExactly(0, "ARCHIVED"); + } + + private static String stateOf(UUID id) { + return stack + .jdbc() + .queryForObject("SELECT state FROM notification_inbox_item WHERE id = ?", String.class, id); + } + private static UUID insertUnreadItem() { UUID id = UUID.randomUUID(); stack diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSchemaConstraintTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSchemaConstraintTest.java new file mode 100644 index 00000000..501e79c7 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSchemaConstraintTest.java @@ -0,0 +1,214 @@ +package dev.caskeleton.adapter.outbound.persistence.operation; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +/** + * The {@code durable_operation} table's own constraints, against a real PostgreSQL. + * + *

What the store does is {@link JpaDurableOperationStoreContractTest}: the shipped adapter + * through the shared {@code DurableOperationStoreContract}. This class holds only what that + * contract cannot reach — the constraints a second writer meets when it bypasses the adapter. A + * {@code CHECK} constraint a fake does not enforce is a comment, so each one is shown refusing a + * row here, by writing the row directly. + * + *

It replaces a class that re-typed the adapter's SQL into the test and ran that copy (round9 + * review 2026-11-12, handed over from RR9 area 01): the copy passed whatever the shipped statements + * did, and seven of its cases repeated the contract. + */ +@Tag("jpa-contract") +class DurableOperationSchemaConstraintTest { + + private static final Instant NOW = Instant.parse("2026-08-25T09:00:00Z"); + + private static JpaPlatformContractSupport support; + + @BeforeAll + static void startServer() { + support = JpaPlatformContractSupport.start(); + } + + @AfterAll + static void stopServer() { + if (support != null) { + support.close(); + } + } + + @BeforeEach + void migrate() throws SQLException, IOException { + try (Connection connection = support.connection(); + Statement statement = connection.createStatement()) { + statement.execute("DROP SCHEMA public CASCADE"); + statement.execute("CREATE SCHEMA public"); + statement.execute(migration("V11__durable_operation.sql")); + statement.execute(migration("V23__durable_operation_lease_fence.sql")); + } + } + + @Test + void identicalScopeIsRefusedByTheUniqueConstraint() throws SQLException { + // given + insertPending("op-1", "hash-1"); + + // when + // The constraint is the arbiter the adapter's ON CONFLICT names, not a prior read: two + // identical submissions can arrive at the same instant, and a read-then-insert lets both in. + Throwable refused = catchThrowable(() -> insertPending("op-2", "hash-1")); + + // then + assertThat(refused) + .isInstanceOf(SQLException.class) + .hasMessageContaining("uq_durable_operation_scope"); + } + + @Test + void succeededRowWithoutAResultIsRefused() throws SQLException { + // given + insertPending("op-1", "hash-1"); + + // when + Throwable refused = + catchThrowable( + () -> + execute( + "UPDATE durable_operation SET state = 'SUCCEEDED', completed_at = now()" + + " WHERE operation_id = 'op-1'")); + + // then + assertThat(refused) + .isInstanceOf(SQLException.class) + .hasMessageContaining("ck_durable_operation_succeeded_has_result"); + } + + @Test + void failedRowWithoutAReasonIsRefused() throws SQLException { + // given + insertPending("op-1", "hash-1"); + + // when + Throwable refused = + catchThrowable( + () -> + execute( + "UPDATE durable_operation SET state = 'FAILED', completed_at = now()" + + " WHERE operation_id = 'op-1'")); + + // then + assertThat(refused) + .isInstanceOf(SQLException.class) + .hasMessageContaining("ck_durable_operation_failed_has_reason"); + } + + @Test + void runningRowWithoutALeaseIsRefused() throws SQLException { + // given + insertPending("op-1", "hash-1"); + + // when + Throwable refused = + catchThrowable( + () -> + execute( + "UPDATE durable_operation SET state = 'RUNNING' WHERE operation_id = 'op-1'")); + + // then + assertThat(refused) + .isInstanceOf(SQLException.class) + .hasMessageContaining("ck_durable_operation_running_has_lease"); + } + + @Test + void stateOutsideThePublishedVocabularyIsRefused() throws SQLException { + // given + insertPending("op-1", "hash-1"); + + // when + // A worker that could write its own state would extend the published contract by writing a + // row, and a client branching on status would meet a value that is in no version of the API. + Throwable refused = + catchThrowable( + () -> + execute( + "UPDATE durable_operation SET state = 'ALMOST_DONE'" + + " WHERE operation_id = 'op-1'")); + + // then + assertThat(refused) + .isInstanceOf(SQLException.class) + .hasMessageContaining("ck_durable_operation_state"); + } + + @Test + void expiryNotAfterSubmissionIsRefused() throws SQLException { + // given + insertPending("op-1", "hash-1"); + + // when + Throwable refused = + catchThrowable( + () -> + execute( + "UPDATE durable_operation SET expires_at = submitted_at" + + " WHERE operation_id = 'op-1'")); + + // then + assertThat(refused) + .isInstanceOf(SQLException.class) + .hasMessageContaining("ck_durable_operation_expiry_after_submission"); + } + + private void insertPending(String operationId, String hash) throws SQLException { + try (Connection connection = support.connection(); + PreparedStatement statement = + connection.prepareStatement( + """ + INSERT INTO durable_operation + (operation_id, tenant, principal, operation_name, request_hash, payload, + state, submitted_at, expires_at) + VALUES (?, '', 'alice', 'transfers.create', ?, '{}', 'PENDING', ?, ?) + """)) { + statement.setString(1, operationId); + statement.setString(2, hash); + statement.setTimestamp(3, Timestamp.from(NOW)); + statement.setTimestamp(4, Timestamp.from(NOW.plus(Duration.ofDays(1)))); + statement.executeUpdate(); + } + } + + private void execute(String sql) throws SQLException { + try (Connection connection = support.connection(); + Statement statement = connection.createStatement()) { + statement.execute(sql); + } + } + + private static String migration(String file) throws IOException { + try (InputStream resource = + DurableOperationSchemaConstraintTest.class + .getClassLoader() + .getResourceAsStream("db/migration/postgresql/" + file)) { + if (resource == null) { + throw new IOException("the durable operation migration " + file + " is missing"); + } + return new String(resource.readAllBytes(), StandardCharsets.UTF_8); + } + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationStoreContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationStoreContractTest.java deleted file mode 100644 index 0e3b7a3d..00000000 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationStoreContractTest.java +++ /dev/null @@ -1,474 +0,0 @@ -package dev.caskeleton.adapter.outbound.persistence.operation; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Timestamp; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -/** - * The durable operation store against a real PostgreSQL. - * - *

Every claim in this store is about concurrency or about a constraint, and neither survives - * being tested against a fake. {@code FOR UPDATE SKIP LOCKED} does not exist in H2's PostgreSQL - * mode with the semantics that matter, a {@code CHECK} constraint a fake does not enforce is a - * comment, and a lease race has no meaning where only one connection exists. - * - *

A dead worker is written the only way that is faithful here: the row a crash leaves behind — a - * lease nobody will renew — and then recovery runs against it. That is precisely what a killed - * process is from the database's side. - */ -@Tag("jpa-contract") -class DurableOperationStoreContractTest { - - private static final Instant NOW = Instant.parse("2026-08-25T09:00:00Z"); - private static final Duration LEASE = Duration.ofMinutes(2); - - private static JpaPlatformContractSupport support; - - @BeforeAll - static void startServer() { - support = JpaPlatformContractSupport.start(); - } - - @AfterAll - static void stopServer() { - if (support != null) { - support.close(); - } - } - - @BeforeEach - void migrate() throws SQLException, IOException { - try (Connection connection = support.connection(); - Statement statement = connection.createStatement()) { - statement.execute("DROP SCHEMA public CASCADE"); - statement.execute("CREATE SCHEMA public"); - statement.execute(migration()); - } - } - - @Test - @DisplayName("a lease left behind by a dead worker is reclaimed by another") - void expiredLeaseCanBeReclaimedAfterWorkerCrash() throws SQLException { - submit("op-1", "hash-1"); - claim("worker-a", NOW); - - // The worker is gone. Nothing renews the lease, and nothing else may touch the operation until - // it lapses — that exclusivity is the point of the lease. - assertThat(claim("worker-b", NOW.plusSeconds(30))).isEmpty(); - - assertThat(claim("worker-b", NOW.plus(LEASE).plusSeconds(1))).contains("op-1"); - assertThat(state("op-1")).isEqualTo("RUNNING"); - assertThat(leaseOwner("op-1")).isEqualTo("worker-b"); - } - - @Test - @DisplayName("a reclaimed operation keeps the moment it first started") - void reclaimKeepsTheOriginalStartTime() throws SQLException { - submit("op-1", "hash-1"); - claim("worker-a", NOW); - - claim("worker-b", NOW.plus(LEASE).plusSeconds(1)); - - // COALESCE, not an unconditional assignment. Overwriting it would make every retry look like a - // fresh start and hide how long the operation has really been running. - assertThat(startedAt("op-1")).isEqualTo(NOW); - } - - @Test - @DisplayName("two workers claiming at once take different operations") - void concurrentWorkersDoNotTakeTheSameOperation() throws Exception { - submit("op-1", "hash-1"); - submit("op-2", "hash-2"); - - List claimed = new ArrayList<>(); - try (Connection first = support.connection(); - Connection second = support.connection()) { - first.setAutoCommit(false); - second.setAutoCommit(false); - // Interleaved on purpose and inside open transactions: with plain FOR UPDATE the second - // claim blocks here until the first commits, and with no locking at all both take op-1. - claimed.add(claimOn(first, "worker-a", NOW).orElseThrow()); - claimed.add(claimOn(second, "worker-b", NOW).orElseThrow()); - first.commit(); - second.commit(); - } - - assertThat(claimed).containsExactlyInAnyOrder("op-1", "op-2"); - } - - @Test - @DisplayName("a worker whose lease lapsed cannot record a result over its successor") - void aStaleWorkerCannotOverwriteTheNewOwner() throws SQLException { - submit("op-1", "hash-1"); - claim("worker-a", NOW); - claim("worker-b", NOW.plus(LEASE).plusSeconds(1)); - - // worker-a is alive and still finishing. Letting it report here is how one operation ends up - // with two answers, and which one survives would come down to timing. - assertThat(succeed("op-1", "worker-a", "/results/a")).isZero(); - assertThat(succeed("op-1", "worker-b", "/results/b")).isOne(); - assertThat(resultReference("op-1")).isEqualTo("/results/b"); - } - - @Test - @DisplayName("a heartbeat from a worker that lost the lease fails rather than extending it") - void heartbeatFromAStaleWorkerFails() throws SQLException { - submit("op-1", "hash-1"); - claim("worker-a", NOW); - claim("worker-b", NOW.plus(LEASE).plusSeconds(1)); - - assertThat(heartbeat("op-1", "worker-a", NOW.plus(LEASE).plusSeconds(2))).isZero(); - assertThat(heartbeat("op-1", "worker-b", NOW.plus(LEASE).plusSeconds(2))).isOne(); - } - - @Test - @DisplayName("cancel works once and never reverses a finished operation") - void cancelIsIdempotentAndNeverReversesTerminal() throws SQLException { - submit("op-1", "hash-1"); - claim("worker-a", NOW); - succeed("op-1", "worker-a", "/results/a"); - - // The work happened. Recording it as CANCELED would replace the only record of a real outcome - // with a claim that it never occurred. - assertThat(cancel("op-1", "alice")).isZero(); - assertThat(state("op-1")).isEqualTo("SUCCEEDED"); - - submit("op-2", "hash-2"); - assertThat(cancel("op-2", "alice")).isOne(); - assertThat(cancel("op-2", "alice")).isZero(); - assertThat(state("op-2")).isEqualTo("CANCELED"); - } - - @Test - @DisplayName("another principal cannot cancel an operation they did not submit") - void cancelIsScopedToThePrincipal() throws SQLException { - submit("op-1", "hash-1"); - - assertThat(cancel("op-1", "mallory")).isZero(); - assertThat(state("op-1")).isEqualTo("PENDING"); - } - - @Test - @DisplayName("a cancelled operation is not handed to a worker") - void cancelledOperationsAreNotClaimable() throws SQLException { - submit("op-1", "hash-1"); - cancel("op-1", "alice"); - - assertThat(claim("worker-a", NOW)).isEmpty(); - } - - @Test - @DisplayName("an identical resubmission is refused by the unique constraint") - void resubmissionCollidesOnScope() throws SQLException { - submit("op-1", "hash-1"); - - // The constraint is the arbiter rather than a prior read: two identical submissions can arrive - // at the same instant, and a read-then-insert lets both through. - assertThatThrownBy(() -> submit("op-2", "hash-1")).isInstanceOf(SQLException.class); - } - - @Test - @DisplayName("a succeeded row with no result is refused by the database") - void succeededRowRequiresAResult() throws SQLException { - submit("op-1", "hash-1"); - - assertThatThrownBy( - () -> - execute( - "UPDATE durable_operation SET state = 'SUCCEEDED', completed_at = now()," - + " lease_owner = NULL WHERE operation_id = 'op-1'")) - .isInstanceOf(SQLException.class) - .hasMessageContaining("ck_durable_operation_succeeded_has_result"); - } - - @Test - @DisplayName("a running row with no lease is refused by the database") - void runningRowRequiresALease() { - assertThatThrownBy( - () -> { - submit("op-1", "hash-1"); - execute("UPDATE durable_operation SET state = 'RUNNING' WHERE operation_id = 'op-1'"); - }) - .isInstanceOf(SQLException.class) - .hasMessageContaining("ck_durable_operation_running_has_lease"); - } - - @Test - @DisplayName("a state outside the published vocabulary is refused") - void stateVocabularyIsClosed() { - // A worker that could write its own state would extend the published contract by writing a - // row, and a client branching on status would meet a value that is in no version of the API. - assertThatThrownBy( - () -> { - submit("op-1", "hash-1"); - execute( - "UPDATE durable_operation SET state = 'ALMOST_DONE' WHERE operation_id = 'op-1'"); - }) - .isInstanceOf(SQLException.class) - .hasMessageContaining("ck_durable_operation_state"); - } - - @Test - @DisplayName("an aged-out operation is expired, and a finished one is left alone") - void expirySweepLeavesRealOutcomesAlone() throws SQLException { - submit("op-1", "hash-1"); - submit("op-2", "hash-2"); - claim("worker-a", NOW); - - List expired = expireStale(NOW.plus(Duration.ofDays(2))); - - assertThat(expired).containsExactlyInAnyOrder("op-1", "op-2"); - submit("op-3", "hash-3"); - claim("worker-b", NOW); - succeed("op-3", "worker-b", "/results/c"); - assertThat(expireStale(NOW.plus(Duration.ofDays(2)))).isEmpty(); - assertThat(state("op-3")).isEqualTo("SUCCEEDED"); - } - - @Test - @DisplayName("reclaim returns lapsed leases to the queue") - void reclaimReturnsLapsedLeases() throws SQLException { - submit("op-1", "hash-1"); - claim("worker-a", NOW); - - assertThat(reclaim(NOW.plusSeconds(30))).isEmpty(); - assertThat(reclaim(NOW.plus(LEASE).plusSeconds(1))).containsExactly("op-1"); - assertThat(state("op-1")).isEqualTo("PENDING"); - assertThat(leaseOwner("op-1")).isNull(); - } - - private void submit(String operationId, String hash) throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = - connection.prepareStatement( - """ - INSERT INTO durable_operation - (operation_id, tenant, principal, operation_name, request_hash, payload, - state, submitted_at, expires_at) - VALUES (?, '', 'alice', 'transfers.create', ?, '{}', 'PENDING', ?, ?) - """)) { - statement.setString(1, operationId); - statement.setString(2, hash); - statement.setTimestamp(3, Timestamp.from(NOW)); - statement.setTimestamp(4, Timestamp.from(NOW.plus(Duration.ofDays(1)))); - statement.executeUpdate(); - } - } - - private Optional claim(String workerId, Instant now) throws SQLException { - try (Connection connection = support.connection()) { - return claimOn(connection, workerId, now); - } - } - - private Optional claimOn(Connection connection, String workerId, Instant now) - throws SQLException { - try (PreparedStatement statement = - connection.prepareStatement( - """ - UPDATE durable_operation - SET state = 'RUNNING', - lease_owner = ?, - lease_acquired_at = ?, - lease_expires_at = ?, - started_at = COALESCE(started_at, ?), - row_version = row_version + 1 - WHERE operation_id = ( - SELECT operation_id - FROM durable_operation - WHERE (state = 'PENDING' - OR (state = 'RUNNING' AND lease_expires_at <= ?)) - AND expires_at > ? - ORDER BY submitted_at - FOR UPDATE SKIP LOCKED - LIMIT 1) - RETURNING operation_id - """)) { - statement.setString(1, workerId); - statement.setTimestamp(2, Timestamp.from(now)); - statement.setTimestamp(3, Timestamp.from(now.plus(LEASE))); - statement.setTimestamp(4, Timestamp.from(now)); - statement.setTimestamp(5, Timestamp.from(now)); - statement.setTimestamp(6, Timestamp.from(now)); - try (ResultSet rows = statement.executeQuery()) { - return rows.next() ? Optional.of(rows.getString(1)) : Optional.empty(); - } - } - } - - private int succeed(String operationId, String workerId, String resultReference) - throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = - connection.prepareStatement( - """ - UPDATE durable_operation - SET state = 'SUCCEEDED', result_reference = ?, completed_at = ?, - lease_owner = NULL, lease_acquired_at = NULL, lease_expires_at = NULL, - row_version = row_version + 1 - WHERE operation_id = ? AND state = 'RUNNING' AND lease_owner = ? - """)) { - statement.setString(1, resultReference); - statement.setTimestamp(2, Timestamp.from(NOW.plusSeconds(10))); - statement.setString(3, operationId); - statement.setString(4, workerId); - return statement.executeUpdate(); - } - } - - private int heartbeat(String operationId, String workerId, Instant now) throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = - connection.prepareStatement( - """ - UPDATE durable_operation - SET lease_expires_at = ?, row_version = row_version + 1 - WHERE operation_id = ? AND state = 'RUNNING' AND lease_owner = ? - AND lease_expires_at > ? - """)) { - statement.setTimestamp(1, Timestamp.from(now.plus(LEASE))); - statement.setString(2, operationId); - statement.setString(3, workerId); - statement.setTimestamp(4, Timestamp.from(now)); - return statement.executeUpdate(); - } - } - - private int cancel(String operationId, String principal) throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = - connection.prepareStatement( - """ - UPDATE durable_operation - SET state = 'CANCELED', completed_at = ?, lease_owner = NULL, - lease_acquired_at = NULL, lease_expires_at = NULL, - row_version = row_version + 1 - WHERE operation_id = ? AND principal = ? AND state IN ('PENDING', 'RUNNING') - """)) { - statement.setTimestamp(1, Timestamp.from(NOW.plusSeconds(5))); - statement.setString(2, operationId); - statement.setString(3, principal); - return statement.executeUpdate(); - } - } - - private List reclaim(Instant now) throws SQLException { - return idsFrom( - """ - UPDATE durable_operation - SET state = 'PENDING', lease_owner = NULL, lease_acquired_at = NULL, - lease_expires_at = NULL, row_version = row_version + 1 - WHERE state = 'RUNNING' AND lease_expires_at <= ? - RETURNING operation_id - """, - now); - } - - private List expireStale(Instant now) throws SQLException { - return idsFrom( - """ - UPDATE durable_operation - SET state = 'EXPIRED', completed_at = ?, lease_owner = NULL, - lease_acquired_at = NULL, lease_expires_at = NULL, - row_version = row_version + 1 - WHERE state IN ('PENDING', 'RUNNING') AND expires_at <= ? - RETURNING operation_id - """, - now, - now); - } - - private List idsFrom(String sql, Instant... arguments) throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = connection.prepareStatement(sql)) { - for (int index = 0; index < arguments.length; index++) { - statement.setTimestamp(index + 1, Timestamp.from(arguments[index])); - } - List ids = new ArrayList<>(); - try (ResultSet rows = statement.executeQuery()) { - while (rows.next()) { - ids.add(rows.getString(1)); - } - } - return ids; - } - } - - private String state(String operationId) throws SQLException { - return column(operationId, "state"); - } - - private String leaseOwner(String operationId) throws SQLException { - return column(operationId, "lease_owner"); - } - - private String resultReference(String operationId) throws SQLException { - return column(operationId, "result_reference"); - } - - private Instant startedAt(String operationId) throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = - connection.prepareStatement( - "SELECT started_at FROM durable_operation WHERE operation_id = ?")) { - statement.setString(1, operationId); - try (ResultSet rows = statement.executeQuery()) { - rows.next(); - return rows.getTimestamp(1).toInstant(); - } - } - } - - private String column(String operationId, String name) throws SQLException { - try (Connection connection = support.connection(); - PreparedStatement statement = - connection.prepareStatement( - "SELECT " + name + " FROM durable_operation WHERE operation_id = ?")) { - statement.setString(1, operationId); - try (ResultSet rows = statement.executeQuery()) { - rows.next(); - return rows.getString(1); - } - } - } - - private void execute(String sql) throws SQLException { - try (Connection connection = support.connection(); - Statement statement = connection.createStatement()) { - statement.execute(sql); - } - } - - private static String migration() throws IOException { - try (InputStream resource = - DurableOperationStoreContractTest.class - .getClassLoader() - .getResourceAsStream("db/migration/postgresql/V11__durable_operation.sql")) { - if (resource == null) { - throw new IOException("the durable operation migration is missing from the classpath"); - } - return new String(resource.readAllBytes(), StandardCharsets.UTF_8); - } - } -} diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/JpaDurableOperationStoreContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/JpaDurableOperationStoreContractTest.java index 9cd7d16a..22fa4f0f 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/JpaDurableOperationStoreContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/JpaDurableOperationStoreContractTest.java @@ -1,9 +1,15 @@ package dev.caskeleton.adapter.outbound.persistence.operation; +import static org.assertj.core.api.Assertions.assertThat; + import dev.caskeleton.adapter.outbound.persistence.operation.entity.DurableOperationEntity; import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; +import dev.caskeleton.application.idempotency.RequestFingerprint; +import dev.caskeleton.application.operation.DurableOperationClaim; +import dev.caskeleton.application.operation.DurableOperationId; import dev.caskeleton.application.operation.DurableOperationStorePort; +import dev.caskeleton.application.operation.DurableOperationSubmission; import dev.caskeleton.application.operation.testkit.DurableOperationStoreContract; import dev.caskeleton.application.transaction.TransactionPort; import jakarta.persistence.EntityManagerFactory; @@ -13,13 +19,22 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.time.Duration; +import java.time.Instant; +import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import javax.sql.DataSource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.springframework.context.annotation.AnnotationConfigApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -32,9 +47,10 @@ import org.springframework.transaction.PlatformTransactionManager; /** * The shipped adapter, through Spring Data and a Spring transaction, against real PostgreSQL. * - *

The raw-SQL {@link DurableOperationStoreContractTest} proves the schema; this proves the code - * that talks to it. A resubmission only breaks where PostgreSQL aborts the transaction after a - * constraint violation, so no fake can stand in here. + *

{@link DurableOperationSchemaConstraintTest} proves the table's constraints; this proves the + * code that talks to it. A resubmission only breaks where PostgreSQL aborts the transaction after a + * constraint violation, and two claims only race where two transactions hold row locks at once, so + * no fake can stand in here. */ @Tag("jpa-contract") class JpaDurableOperationStoreContractTest extends DurableOperationStoreContract { @@ -45,7 +61,8 @@ class JpaDurableOperationStoreContractTest extends DurableOperationStoreContract @BeforeAll static void start() throws SQLException, IOException { support = JpaPlatformContractSupport.start(); - execute(migration()); + execute(migration("V11__durable_operation.sql")); + execute(migration("V23__durable_operation_lease_fence.sql")); context = new AnnotationConfigApplicationContext(); context.getBeanFactory().registerSingleton("dataSource", support.dataSource()); context.register(AdapterConfiguration.class); @@ -67,6 +84,74 @@ class JpaDurableOperationStoreContractTest extends DurableOperationStoreContract execute("TRUNCATE durable_operation"); } + @Test + void twoWorkersClaimingAtOnceTakeDifferentOperations() throws Exception { + // given + Instant now = Instant.parse("2026-08-25T09:00:00Z"); + inTransaction(() -> store().submit(submission("op-1", "a", now))); + inTransaction(() -> store().submit(submission("op-2", "b", now.plusSeconds(1)))); + CountDownLatch firstClaimed = new CountDownLatch(1); + CountDownLatch secondDone = new CountDownLatch(1); + ExecutorService pool = Executors.newSingleThreadExecutor(); + + // when + List claimed; + try { + // The first claim holds its row lock until the second has claimed. With plain FOR UPDATE the + // second would block behind it; with no lock at all both would take op-1. + CompletableFuture first = + CompletableFuture.supplyAsync( + () -> + inTransaction( + () -> { + DurableOperationClaim claim = claimNow("worker-a", now); + firstClaimed.countDown(); + awaitOrFail(secondDone); + return claim; + }), + pool); + awaitOrFail(firstClaimed); + DurableOperationClaim second = inTransaction(() -> claimNow("worker-b", now)); + secondDone.countDown(); + claimed = + List.of( + first.get(30, TimeUnit.SECONDS).operationId().value(), second.operationId().value()); + } finally { + pool.shutdownNow(); + } + + // then + assertThat(claimed).containsExactlyInAnyOrder("op-1", "op-2"); + } + + private DurableOperationClaim claimNow(String workerId, Instant now) { + return store().claimNext(workerId, Duration.ofMinutes(2), now).orElseThrow(); + } + + private static void awaitOrFail(CountDownLatch latch) { + try { + if (!latch.await(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("the other claim did not arrive within 30 seconds"); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while waiting for the other claim", interrupted); + } + } + + private static DurableOperationSubmission submission( + String operationId, String hashLetter, Instant submittedAt) { + return new DurableOperationSubmission( + new DurableOperationId(operationId), + "reports.generate", + "alice", + null, + new RequestFingerprint(hashLetter.repeat(64)), + "{}", + submittedAt, + Duration.ofDays(1)); + } + @Override protected DurableOperationStorePort store() { return context.getBean(JpaDurableOperationStore.class); @@ -84,13 +169,13 @@ class JpaDurableOperationStoreContractTest extends DurableOperationStoreContract } } - private static String migration() throws IOException { + private static String migration(String file) throws IOException { try (InputStream resource = JpaDurableOperationStoreContractTest.class .getClassLoader() - .getResourceAsStream("db/migration/postgresql/V11__durable_operation.sql")) { + .getResourceAsStream("db/migration/postgresql/" + file)) { if (resource == null) { - throw new IOException("the durable operation migration is missing from the classpath"); + throw new IOException("the durable operation migration " + file + " is missing"); } return new String(resource.readAllBytes(), StandardCharsets.UTF_8); } diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/FileserverPostgreSqlStack.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/FileserverPostgreSqlStack.java index a6e77b62..068f1dc0 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/FileserverPostgreSqlStack.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/FileserverPostgreSqlStack.java @@ -96,6 +96,36 @@ final class FileserverPostgreSqlStack implements AutoCloseable { return fileId; } + /** + * Waits until some session of this database is blocked on a lock, so a case can hold one + * transaction open until a second one is queued behind it without sleeping a guessed interval. + */ + void awaitASessionWaitingOnALock(Duration deadline) { + Instant giveUp = Instant.now().plus(deadline); + while (Instant.now().isBefore(giveUp)) { + Integer waiting = + jdbc.queryForObject( + "select count(*) from pg_stat_activity" + + " where datname = current_database() and wait_event_type = 'Lock'", + Integer.class); + if (waiting != null && waiting > 0) { + return; + } + Thread.onSpinWait(); + } + throw new IllegalStateException("no session waited on a lock within " + deadline); + } + + /** Counts the rows of {@code fs_recovery_item} for one file, open and resolved. */ + int recoveryItemsOf(FileId fileId) { + Integer rows = + jdbc.queryForObject( + "select count(*) from fs_recovery_item where file_id = ?", + Integer.class, + fileId.value()); + return rows == null ? -1 : rows; + } + /** Deletes every Fileserver row, children first. */ void empty() { jdbc.execute( diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java index bf5117a8..0439e9d5 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java @@ -50,7 +50,7 @@ class PostgreSqlMigrationIntegrationTest { assertThat(appliedVersions(postgres, "flyway_schema_history")) .containsExactly( "1", "3", "4", "5", "6", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", - "19", "20", "21", "22"); + "19", "20", "21", "22", "23"); assertThat( singleValue( postgres, diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlNotificationBackgroundPassIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlNotificationBackgroundPassIntegrationTest.java index 74d09587..09606e87 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlNotificationBackgroundPassIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlNotificationBackgroundPassIntegrationTest.java @@ -70,6 +70,7 @@ import org.flywaydb.core.Flyway; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.springframework.context.annotation.AnnotationConfigApplicationContext; @@ -189,7 +190,25 @@ class PostgreSqlNotificationBackgroundPassIntegrationTest { } @Test - @org.junit.jupiter.api.DisplayName("요청 저장은 collapse 와 dedup DROP 판정을 함께 남겨 같은 키 재조회가 같은 답을 준다") + @DisplayName("fence 없이 DISPATCHING으로 남은 옛 행이 있어도 복구 패스는 다른 행을 되돌린다") + void aDispatchingRowNoClaimWroteDoesNotStopTheRecoveryPass() { + // given + UUID legacy = insertDelivery("DISPATCHING", Instant.now().minus(Duration.ofMinutes(20)), null); + jdbc.update("update notification_recipient_delivery set lease_fence = 0 where id = ?", legacy); + UUID abandoned = + insertDelivery("DISPATCHING", Instant.now().minus(Duration.ofMinutes(10)), null); + + // when + recovery().recoverOnce(); + + // then + assertThat(List.of(deliveryState(legacy), deliveryState(abandoned))) + .as("a row before notification V4's fence is no lease the requeue can match on") + .containsExactly("DISPATCHING", "READY_TO_DISPATCH"); + } + + @Test + @DisplayName("요청 저장은 collapse 와 dedup DROP 판정을 함께 남겨 같은 키 재조회가 같은 답을 준다") void aStoredRequestKeepsItsCollapseAndItsDropVerdict() { // given JpaNotificationRequestStore requests = @@ -414,13 +433,15 @@ class PostgreSqlNotificationBackgroundPassIntegrationTest { "insert into notification_recipient_delivery (id, notification_id, tenant_id," + " recipient_ref, routing_plan_json, route_cursor, delivery_state, submission_outcome," + " delivery_outcome, evidence_level, attempt_count, next_dispatch_at, lease_owner," - + " lease_until, expires_at, created_at, updated_at)" + + " lease_fence, lease_until, expires_at, created_at, updated_at)" + " values (?, ?, 'tenant-a', 'user-1', '[]'::jsonb, 0, ?, 'NOT_SUBMITTED', 'UNKNOWN'," - + " 'PLATFORM_QUEUED', 0, now(), ?, ?, ?, now(), now())", + + " 'PLATFORM_QUEUED', 0, now(), ?, ?, ?, ?, now(), now())", delivery, request, state, leaseUntil == null ? null : "dead-worker", + // A claim writes the owner and raises the fence together, so a leased row has both. + leaseUntil == null ? 0L : 1L, leaseUntil == null ? null : Timestamp.from(leaseUntil), expiresAt == null ? null : Timestamp.from(expiresAt)); return delivery; diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlRecoveryQueueContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlRecoveryQueueContractTest.java index 9ba78957..a4c34b3c 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlRecoveryQueueContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlRecoveryQueueContractTest.java @@ -1,16 +1,25 @@ package dev.caskeleton.adapter.outbound.persistence.readiness; +import static org.assertj.core.api.Assertions.assertThat; + import dev.caskeleton.adapter.outbound.persistence.fileserver.JpaRecoveryQueue; import dev.caskeleton.application.fileserver.api.FileId; import dev.caskeleton.application.fileserver.api.StorageNamespace; import dev.caskeleton.application.fileserver.recovery.RecoveryQueuePort; import dev.caskeleton.application.fileserver.testkit.RecoveryQueuePortContract; import java.time.Duration; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * {@link JpaRecoveryQueue} held to the contract the in-memory queue passes, on real PostgreSQL @@ -38,6 +47,66 @@ class PostgreSqlRecoveryQueueContractTest extends RecoveryQueuePortContract { stack.empty(); } + @Test + void theSameFileEnqueuedInTwoConcurrentTransactionsLeavesOneOpenItem() throws Exception { + // given + FileId file = recordedFile(); + CountDownLatch firstEnqueued = new CountDownLatch(1); + CountDownLatch secondQueued = new CountDownLatch(1); + ExecutorService pool = Executors.newFixedThreadPool(2); + + // when + List outcome; + try { + // The first transaction holds its new row uncommitted until the second has queued behind the + // open-item index. Update-then-insert made the second insert too and fail on that index. + CompletableFuture first = + CompletableFuture.runAsync( + () -> + inTransaction( + () -> { + queue().enqueue(file, "FIRST_REPORT"); + firstEnqueued.countDown(); + awaitOrFail(secondQueued); + return null; + }), + pool); + awaitOrFail(firstEnqueued); + CompletableFuture second = + CompletableFuture.runAsync( + () -> + inTransaction( + () -> { + queue().enqueue(file, "SECOND_REPORT"); + return null; + }), + pool); + stack.awaitASessionWaitingOnALock(Duration.ofSeconds(30)); + secondQueued.countDown(); + first.get(30, TimeUnit.SECONDS); + second.get(30, TimeUnit.SECONDS); + outcome = + List.of(stack.recoveryItemsOf(file), inTransaction(() -> queue().pending(10)).size()); + } finally { + pool.shutdownNow(); + } + + // then + assertThat(outcome).containsExactly(1, 1); + } + + private static void awaitOrFail(CountDownLatch latch) { + try { + if (!latch.await(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("the other enqueue did not arrive within 30 seconds"); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "interrupted while waiting for the other enqueue", interrupted); + } + } + @Override protected RecoveryQueuePort queue() { return stack.bean(JpaRecoveryQueue.class); diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java index 72235d67..e80b313d 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java @@ -2,23 +2,61 @@ package dev.caskeleton.adapter.outbound.persistence.idempotency.jpa; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.time.Clock; -import java.time.Instant; -import java.time.ZoneOffset; import org.junit.jupiter.api.Test; +/** + * Unit tests for {@link IdempotencyReaper}: the delete runs in bounded batches until one comes back + * short, as the outbox reaper's does. + * + *

The repository is a Spring Data boundary, so Mockito stands in for it. The statement itself — + * expiry judged by the database clock — runs against PostgreSQL in {@code + * IdempotencyReaperIntegrationTest}. + */ class IdempotencyReaperTest { - private static final Instant NOW = Instant.parse("2026-06-09T12:00:00Z"); + private static final int BATCH = IdempotencyReaper.BATCH_SIZE; + + private final IdempotencyRecordJpaRepository repository = + mock(IdempotencyRecordJpaRepository.class); + private final IdempotencyReaper reaper = new IdempotencyReaper(repository); @Test - void reapDeletesRecordsExpiredAsOfNow() { - IdempotencyRecordJpaRepository repo = mock(IdempotencyRecordJpaRepository.class); - when(repo.deleteExpired(NOW)).thenReturn(3); - IdempotencyReaper reaper = new IdempotencyReaper(repo, Clock.fixed(NOW, ZoneOffset.UTC)); + void reapKeepsDeletingBatchesUntilOneComesBackShortAndReturnsTheTotal() { + // given + when(repository.deleteExpiredBatch(BATCH)).thenReturn(BATCH, BATCH, 3); - assertThat(reaper.reap()).isEqualTo(3); + // when + int purged = reaper.reap(); + + // then + assertThat(purged).isEqualTo(2 * BATCH + 3); + } + + @Test + void eachBatchIsItsOwnDeleteStatement() { + // given + when(repository.deleteExpiredBatch(BATCH)).thenReturn(BATCH, BATCH, 3); + + // when + var unused = reaper.reap(); + + // then + verify(repository, times(3)).deleteExpiredBatch(BATCH); + } + + @Test + void reapReturnsZeroWhenNothingHasExpired() { + // given + when(repository.deleteExpiredBatch(BATCH)).thenReturn(0); + + // when + int purged = reaper.reap(); + + // then + assertThat(purged).isZero(); } } diff --git a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 index 1eb57d14..32edf890 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 +++ b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 @@ -42,6 +42,7 @@ a0306e5b0fbcaf43984d256362dc4244aec57a0ce5031e4f5263be3762fe1caf postgresql/V19 25db296e55444636c8a017d9b736eecf4a839a9caf1733c0c4a5f37bae5a3161 postgresql/V20__outbox_event_deferred_attempt_count.sql 21744ea1aab2b29fbb3417b9a5c4389522199413048cc7035aa649ffcdd852cf postgresql/V21__outbox_aggregate_sequence.sql 20da5649a29bf2305f0e04ab14003b21b7f120d867439fb01f1658c98179dba0 postgresql/V22__outbox_event_last_failure_code.sql +9492ace50ab828144792b77c72d92fce50bec29a028ebc917f7e55facf58d85a postgresql/V23__durable_operation_lease_fence.sql 8ab106731d4d04d3719da0692069947736dc490ca46aa5a3ea9bdf65a5d956dc postgresql/V1__idempotency_record.sql 6b62a01a4eeb792b81015b4810ecc6f3f021e0a17dea50c871493f7694d4f0b9 postgresql/V3__outbox_event.sql c57ab8a35a10079190ad8a563be58fae4147c9406c5bb752debb6998324ccdf6 postgresql/V4__int_lock.sql diff --git a/src/adapter/outbound/persistence-notification/AGENTS.md b/src/adapter/outbound/persistence-notification/AGENTS.md index 94fa03ad..cb9ae8b1 100644 --- a/src/adapter/outbound/persistence-notification/AGENTS.md +++ b/src/adapter/outbound/persistence-notification/AGENTS.md @@ -8,7 +8,8 @@ - 패키지 루트: `dev.caskeleton.adapter.outbound.persistence.notification`. 패키지 규칙은 `persistence-fileserver`와 같아서 두 capability leaf를 같은 방식으로 읽는다: - `platform.entity` — JPA 행만. - - `platform.repository` — Spring Data 인터페이스와 그것이 돌리는 SQL 글(`RecipientClaimSql`). + - `platform.repository` — Spring Data 인터페이스와 그것이 돌리는 SQL 글(`RecipientClaimSql`). `platform`의 store도 + 이 SQL 조각을 쓴다. - `platform` — store 어댑터(`Jpa*`, `Jdbc*`, inbox의 `JpaNotificationInbox`와 그 outbox 도우미), `NotificationRecordMapper`, payload codec. - `configuration` — store를 bean으로 등록하는 자동 구성 창구. leaf 루트에는 scan 설정과 schema stream이 있다. diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java index 23fdb2dc..a2da23c2 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java @@ -99,6 +99,23 @@ public final class JpaRecipientDeliveryStore implements RecipientDeliveryStorePo return reread(written, record.id()); } + @Override + public Optional requeueAbandoned( + RecipientLease abandoned, Instant nextDispatchAt) { + CallerTransaction.require("recipient delivery requeueAbandoned"); + Objects.requireNonNull(abandoned, "abandoned"); + Objects.requireNonNull(nextDispatchAt, "nextDispatchAt"); + int written = + recipients.requeueAbandoned( + abandoned.recipientDeliveryId().value(), + abandoned.owner(), + abandoned.fence(), + abandoned.leaseUntil(), + nextDispatchAt, + clock.instant()); + return reread(written, abandoned.recipientDeliveryId()); + } + @Override public Optional transitionHeldBy( RecipientDeliveryId id, diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java index 65784bac..98cd4bef 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java @@ -5,9 +5,11 @@ import dev.caskeleton.adapter.outbound.persistence.transaction.CallerTransaction import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; import dev.caskeleton.application.notification.platform.dispatch.RecipientLease; import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort; +import java.sql.Timestamp; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.time.OffsetDateTime; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -109,13 +111,38 @@ public final class JpaRecipientLeaseStore implements RecipientLeaseStorePort { } @Override - public List expiredDispatching(int limit, Duration olderThan) { + public List expiredDispatching(int limit, Duration olderThan) { Objects.requireNonNull(olderThan, "olderThan"); if (limit < 1) { throw new IllegalArgumentException("limit"); } return recipients.selectExpiredDispatching(clock.instant().minus(olderThan), limit).stream() - .map(RecipientDeliveryId::new) + .map( + row -> + new RecipientLease( + new RecipientDeliveryId((UUID) row[0]), + (String) row[1], + ((Number) row[2]).longValue(), + instantOf(row[3]))) .toList(); } + + /** + * A {@code timestamptz} column of a native row. Hibernate hands it back as an {@link + * OffsetDateTime} or a {@link Timestamp} depending on its JDBC type mapping, never null here + * because the select only returns rows that have a lease. + */ + private static Instant instantOf(Object column) { + if (column instanceof Instant instant) { + return instant; + } + if (column instanceof OffsetDateTime offset) { + return offset.toInstant(); + } + if (column instanceof Timestamp timestamp) { + return timestamp.toInstant(); + } + throw new IllegalStateException( + "lease_until came back as " + (column == null ? "null" : column.getClass().getName())); + } } diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java index 15428027..e7ccfb32 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java @@ -60,18 +60,16 @@ public interface InboxItemJpaRepository extends Repository{@code read_at IS NULL} in the predicate is what makes the update idempotent under * concurrency: a second caller updates zero rows instead of overwriting the first caller's * timestamp. - */ - /** - * Marks everything up to and including a cursor position as read. * *

The cutoff is a tuple, matching the list's own {@code (created_at DESC, id DESC)} ordering. * It used to be a timestamp alone, and the list's order is not a total order on {@code @@ -95,14 +93,23 @@ public interface InboxItemJpaRepository extends RepositoryThe lease store contract ({@code RecipientLeaseStoreContract}, run on PostgreSQL by {@code * PostgreSqlRecipientLeaseStoreContractTest}) now drives this statement through the repository and * the store, so no test retypes it or runs it beside the adapter (review 2026-11-01 04-14). The - * named parameters are Spring Data's. + * named parameters are bound by Spring Data for the repository statements, and by a {@code + * NamedParameterJdbcTemplate} where the serving state reads {@link #CLAIMABLE_AT_NOW}. */ public final class RecipientClaimSql { + /** + * The states a delivery waits in to be sent, which the claim takes from, the expiry retires from + * and the serving state counts as backlog. One list so the statements cannot disagree about what + * is waiting. + */ + public static final String CLAIMABLE_STATES = "('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')"; + + /** + * Which deliveries the claim takes at {@code :now}: due, in a waiting state, not expired, and + * with no lease or one that has run out. The serving state counts its backlog with this same + * fragment, so the backlog is exactly what a claim would take. + */ + public static final String CLAIMABLE_AT_NOW = + "next_dispatch_at <= :now" + + " AND delivery_state IN " + + CLAIMABLE_STATES + + " AND (expires_at IS NULL OR expires_at > :now)" + + " AND (lease_until IS NULL OR lease_until < :now)"; + /** * Claims due deliveries and returns their fences. * @@ -28,10 +48,8 @@ public final class RecipientClaimSql { public static final String CLAIM_BATCH = "WITH claimable AS (" + " SELECT id FROM notification_recipient_delivery" - + " WHERE next_dispatch_at <= :now" - + " AND delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')" - + " AND (expires_at IS NULL OR expires_at > :now)" - + " AND (lease_until IS NULL OR lease_until < :now)" + + " WHERE " + + CLAIMABLE_AT_NOW + " ORDER BY next_dispatch_at, id" + " FOR UPDATE SKIP LOCKED" + " LIMIT :batchSize" @@ -51,7 +69,18 @@ public final class RecipientClaimSql { + " WHERE d.id = claimable.id" + " RETURNING d.id, d.lease_fence"; - /** Retires deliveries whose window closed before anyone claimed them. */ + /** + * Retires deliveries whose window closed before anyone claimed them. + * + *

A delivery a worker claimed while this statement ran must not be retired to {@code EXPIRED} + * under the worker dispatching it, which is what the unlocked subquery did (round9 review + * 2026-11-12, RR9-01-13). Two things stop it. The subquery locks the rows it picks and skips rows + * another transaction holds, so a delivery being claimed is not picked. The state and expiry + * conditions are also on the {@code UPDATE} itself: under READ COMMITTED PostgreSQL re-checks + * only the outer {@code WHERE} against a row it had to wait for, so if the lock clause is ever + * lost the claimed row is still left alone. A delivery being claimed right now is not expiring + * either: the claim refuses expired rows in the same statement. + */ public static final String EXPIRE_OVERDUE = "UPDATE notification_recipient_delivery SET" + " delivery_state = 'EXPIRED'," @@ -61,11 +90,17 @@ public final class RecipientClaimSql { + " updated_at = :now" + " WHERE id IN (" + " SELECT id FROM notification_recipient_delivery" - + " WHERE delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')" + + " WHERE delivery_state IN " + + CLAIMABLE_STATES + " AND expires_at IS NOT NULL" + " AND expires_at <= :now" + " ORDER BY expires_at" - + " LIMIT :batchSize)"; + + " FOR UPDATE SKIP LOCKED" + + " LIMIT :batchSize)" + + " AND delivery_state IN " + + CLAIMABLE_STATES + + " AND expires_at IS NOT NULL" + + " AND expires_at <= :now"; private RecipientClaimSql() {} } diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java index b5f1ee18..1240cf0f 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java @@ -183,13 +183,48 @@ public interface RecipientDeliveryJpaRepository extends RepositoryOnly rows a claim wrote: an owner and a fence of at least one. The requeue matches on both, + * and a row without them — one left dispatching before notification V4 added the fence — is no + * lease anyone holds. Read here it would be a lease the application refuses to build, and the one + * row would fail every recovery pass after it. + * + * @return one row per delivery: id, lease_owner, lease_fence, lease_until + */ @Query( value = - "SELECT id FROM notification_recipient_delivery " + "SELECT id, lease_owner, lease_fence, lease_until FROM notification_recipient_delivery " + "WHERE delivery_state = 'DISPATCHING' AND lease_until < :threshold " + + "AND lease_owner IS NOT NULL AND lease_fence > 0 " + "ORDER BY lease_until LIMIT :batchSize", nativeQuery = true) - List selectExpiredDispatching( + List selectExpiredDispatching( @Param("threshold") Instant threshold, @Param("batchSize") int batchSize); + + /** + * Returns an abandoned dispatch to the queue while it is still the one recovery found. + * + *

Owner, fence, state and expiry are all conditions. A new claim raises the fence, a finished + * holder moves the state, another instance's recovery moves the state, and a holder that renewed + * in time moves the expiry past what recovery saw; in each case the job is no longer abandoned + * and this matches nothing. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query( + value = + "UPDATE notification_recipient_delivery " + + "SET delivery_state = 'READY_TO_DISPATCH', next_dispatch_at = :nextDispatchAt, " + + "version = version + 1, updated_at = :now " + + "WHERE id = :id AND lease_owner = :owner AND lease_fence = :fence " + + "AND delivery_state = 'DISPATCHING' AND lease_until <= :leaseUntil", + nativeQuery = true) + int requeueAbandoned( + @Param("id") UUID id, + @Param("owner") String owner, + @Param("fence") long fence, + @Param("leaseUntil") Instant leaseUntil, + @Param("nextDispatchAt") Instant nextDispatchAt, + @Param("now") Instant now); } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java new file mode 100644 index 00000000..80bdbe0c --- /dev/null +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java @@ -0,0 +1,45 @@ +package dev.caskeleton.application.operation; + +import java.util.Objects; + +/** + * One worker's claim on one durable operation, as returned by {@link + * DurableOperationStorePort#claimNext}. + * + *

The {@code fence} is what tells one run from the next. Every claim increments it, and a + * heartbeat, progress report or completion names the fence it holds, so a run that stalled past its + * lease finds nothing left to change instead of overwriting the run that took over. The worker id + * alone cannot do this: a worker that restarts with the same id, or one process running two + * executions under one id, would otherwise have its earlier run accepted as the current one. + * + *

Kept apart from {@link DurableOperationSubmission} on purpose, as the outbox keeps its lease + * apart from the claimed event: the work to execute is the submission, and the claim is store + * bookkeeping the work has no use for. + * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * + * @param submission the claimed work + * @param workerId the worker that holds the claim + * @param fence the fencing token this claim was granted; at least 1 + */ +public record DurableOperationClaim( + DurableOperationSubmission submission, String workerId, long fence) { + + public DurableOperationClaim { + Objects.requireNonNull(submission, "submission"); + if (workerId == null || workerId.isBlank()) { + throw new IllegalStateException( + "a claim needs an owner; an anonymous one cannot be verified when the work completes"); + } + if (fence < 1) { + throw new IllegalStateException("fence must be >= 1, was " + fence); + } + } + + /** The claimed operation. */ + public DurableOperationId operationId() { + return submission.operationId(); + } +} diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java index 440d3fb2..66c718b2 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java @@ -11,9 +11,12 @@ import java.util.Optional; *

A port because the application owns what an operation is and nothing about how it is stored. * The JPA adapter behind it is one implementation; an in-memory one serves tests. * - *

Every method that changes a running operation takes the {@code workerId} and verifies it. A - * worker whose lease lapsed may still be alive and still finishing its work, and letting it report - * a result after another worker took over is how an operation ends up with two answers. + *

Every method that changes a running operation takes the {@link DurableOperationClaim} that + * {@link #claimNext} returned and verifies both its worker and its fence. A worker whose lease + * lapsed may still be alive and still finishing its work, and letting it report a result after + * another run took over is how an operation ends up with two answers. The worker id alone does not + * tell the runs apart: the same id may claim the operation again after a restart, so the fence — + * incremented by every claim — is what the store compares. */ public interface DurableOperationStorePort { @@ -43,48 +46,43 @@ public interface DurableOperationStorePort { * *

Runnable means PENDING, or RUNNING with a lapsed lease. The second case is the recovery * path: without it, an operation claimed by a worker that then died is never picked up again. + * Every claim gets a fence higher than any earlier claim of the same operation, whoever made it. * * @param workerId who is claiming * @param leaseDuration how long the claim holds without a heartbeat * @param now the current instant + * @return the claim the worker names on every later write, or empty when nothing is runnable */ - Optional claimNext( - String workerId, Duration leaseDuration, Instant now); + Optional claimNext(String workerId, Duration leaseDuration, Instant now); /** - * Extends a lease the worker still holds. + * Extends a lease the claim still holds. * - * @return false when the lease is no longer this worker's, which means it must stop + * @return false when the lease lapsed or another claim (by any worker, this one included) took + * the operation over, which means this run must stop */ - boolean heartbeat( - DurableOperationId operationId, String workerId, Duration leaseDuration, Instant now); + boolean heartbeat(DurableOperationClaim claim, Duration leaseDuration, Instant now); /** - * Publishes progress for an operation this worker holds. + * Publishes progress for an operation the claim holds. * - * @return false when the lease is no longer this worker's + * @return false when another claim took the operation over or it is no longer running */ - boolean reportProgress( - DurableOperationId operationId, String workerId, OperationProgressSnapshot progress); + boolean reportProgress(DurableOperationClaim claim, OperationProgressSnapshot progress); /** * Records success. * - * @return false when the lease is no longer this worker's, so the result is not recorded + * @return false when another claim took the operation over, so the result is not recorded */ - boolean succeed( - DurableOperationId operationId, String workerId, String resultReference, Instant completedAt); + boolean succeed(DurableOperationClaim claim, String resultReference, Instant completedAt); /** * Records failure. * - * @return false when the lease is no longer this worker's + * @return false when another claim took the operation over, so the failure is not recorded */ - boolean fail( - DurableOperationId operationId, - String workerId, - OperationFailure failure, - Instant completedAt); + boolean fail(DurableOperationClaim claim, OperationFailure failure, Instant completedAt); /** * Requests cancellation. diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java index 86328d84..618f23c1 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java @@ -11,6 +11,10 @@ import java.util.Objects; * ever with no worker behind it — the failure that makes a queue stop draining while every * dashboard reports it busy. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param workerId who holds it * @param acquiredAt when it was taken * @param expiresAt when another worker may take it @@ -22,11 +26,11 @@ public record OperationLease(String workerId, Instant acquiredAt, Instant expire Objects.requireNonNull(acquiredAt, "acquiredAt"); Objects.requireNonNull(expiresAt, "expiresAt"); if (workerId.isBlank()) { - throw new IllegalArgumentException( + throw new IllegalStateException( "a lease needs an owner; an anonymous one cannot be verified when the work completes"); } if (!expiresAt.isAfter(acquiredAt)) { - throw new IllegalArgumentException("a lease that expires when taken is not a lease"); + throw new IllegalStateException("a lease that expires when taken is not a lease"); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java b/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java index bafa1b5e..05c0062e 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java @@ -12,6 +12,10 @@ import java.util.Objects; * publishes the bytes that were validated rather than a re-rendering of them. The accessors below * name the few identities the relay and its diagnostics read without reaching into the envelope. * + *

A store builds it from a row, never from a request, so a count it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param event the appended event * @param status the row's lifecycle status; {@link OutboxEventStatus#IN_FLIGHT} right after a claim * @param attemptCount claims made, counting the claim that returned this row; the retry delay grows @@ -30,10 +34,10 @@ public record ClaimedOutboxEvent( Objects.requireNonNull(event, "event must not be null"); Objects.requireNonNull(status, "status must not be null"); if (attemptCount < 0) { - throw new IllegalArgumentException("attemptCount must be >= 0, was " + attemptCount); + throw new IllegalStateException("attemptCount must be >= 0, was " + attemptCount); } if (deferredAttemptCount < 0 || deferredAttemptCount > attemptCount) { - throw new IllegalArgumentException( + throw new IllegalStateException( "deferredAttemptCount must be between 0 and attemptCount (" + attemptCount + "), was " diff --git a/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java b/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java index eed397ab..ae783bc1 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java @@ -13,6 +13,10 @@ import java.util.Objects; *

Kept apart from {@link ClaimedOutboxEvent} on purpose: the broker adapter receives the event, * and the claim is store bookkeeping it has no use for. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param event the claimed row * @param owner the relay instance that holds the claim * @param token the fencing token this claim was granted @@ -22,10 +26,10 @@ public record OutboxLease(ClaimedOutboxEvent event, String owner, long token) { public OutboxLease { Objects.requireNonNull(event, "event must not be null"); if (owner == null || owner.isBlank()) { - throw new IllegalArgumentException("owner must not be blank"); + throw new IllegalStateException("owner must not be blank"); } if (token < 1) { - throw new IllegalArgumentException("token must be >= 1, was " + token); + throw new IllegalStateException("token must be >= 1, was " + token); } } diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java index 628ee3ef..8c845cac 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java @@ -39,8 +39,9 @@ class CancelDurableOperationUseCaseTest { void cancelOfAFinishedOperationReportsNoChange() { // given operations.submit(submission(null)); - operations.claimNext("worker-a", Duration.ofMinutes(1), NOW); - operations.succeed(ID, "worker-a", "/results/1", NOW); + DurableOperationClaim claim = + operations.claimNext("worker-a", Duration.ofMinutes(1), NOW).orElseThrow(); + operations.succeed(claim, "/results/1", NOW); // when Optional canceled = diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java new file mode 100644 index 00000000..13f6efaf --- /dev/null +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java @@ -0,0 +1,80 @@ +package dev.caskeleton.application.operation; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.idempotency.RequestFingerprint; +import dev.caskeleton.application.outbox.ClaimedOutboxEvent; +import dev.caskeleton.application.outbox.OutboxEventStatus; +import dev.caskeleton.application.outbox.OutboxLease; +import dev.caskeleton.application.outbox.testkit.OutboxEvents; +import java.time.Duration; +import java.time.Instant; +import java.util.stream.Stream; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** The claims and leases a store builds from its rows refuse a bad value as each record says. */ +class StoreBuiltRecordsTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + + static Stream refusedValues() { + DurableOperationSubmission submission = + new DurableOperationSubmission( + new DurableOperationId("op-1"), + "reports.generate", + "alice", + "tenant-a", + new RequestFingerprint("a".repeat(64)), + "{}", + NOW, + Duration.ofHours(1)); + return Stream.of( + Arguments.of( + "owner 없는 claim", + (ThrowingCallable) () -> new DurableOperationClaim(submission, " ", 1), + "a claim needs an owner"), + Arguments.of( + "fence 0인 claim", + (ThrowingCallable) () -> new DurableOperationClaim(submission, "worker-1", 0), + "fence must be >= 1, was 0"), + Arguments.of( + "owner 없는 lease", + (ThrowingCallable) () -> new OperationLease(" ", NOW, NOW.plusSeconds(30)), + "a lease needs an owner"), + Arguments.of( + "잡는 순간 끝나는 lease", + (ThrowingCallable) () -> new OperationLease("worker-1", NOW, NOW), + "a lease that expires when taken is not a lease"), + Arguments.of( + "owner 없는 outbox lease", + (ThrowingCallable) () -> new OutboxLease(OutboxEvents.claimed("evt-1", NOW, 1), " ", 1), + "owner must not be blank"), + Arguments.of( + "token 0인 outbox lease", + (ThrowingCallable) + () -> new OutboxLease(OutboxEvents.claimed("evt-1", NOW, 1), "relay-1", 0), + "token must be >= 1, was 0"), + Arguments.of( + "미룬 claim이 claim보다 많은 outbox 행", + (ThrowingCallable) + () -> + new ClaimedOutboxEvent( + OutboxEvents.event("evt-2", NOW), OutboxEventStatus.IN_FLIGHT, 1, 2), + "deferredAttemptCount must be between 0 and attemptCount (1), was 2")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("refusedValues") + void aValueAStoreCannotHaveBuiltIsAServerFault( + String name, ThrowingCallable build, String message) { + // when + Throwable refused = catchThrowable(build); + + // then + assertThat(refused).isInstanceOf(IllegalStateException.class).hasMessageContaining(message); + } +} diff --git a/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java b/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java index a5a6a3fa..b22ab546 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java @@ -49,7 +49,8 @@ class ClaimedOutboxEventContractTest { // then assertThat(thrown) - .isInstanceOf(IllegalArgumentException.class) + .as("a store builds the claim from its row, so a bad count is the store's fault") + .isInstanceOf(IllegalStateException.class) .hasMessage("attemptCount must be >= 0, was -1"); } } diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java index 4be958d2..38f896aa 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.application.idempotency.RequestFingerprint; import dev.caskeleton.application.operation.DurableOperation; +import dev.caskeleton.application.operation.DurableOperationClaim; import dev.caskeleton.application.operation.DurableOperationId; import dev.caskeleton.application.operation.DurableOperationState; import dev.caskeleton.application.operation.DurableOperationStorePort; @@ -25,6 +26,10 @@ import org.junit.jupiter.api.Test; * *

Every call goes through {@link #inTransaction}, because the real store refuses writes without * a caller transaction and because a resubmission must leave that transaction usable. + * + *

A lease write names the {@link DurableOperationClaim} it holds. The fence cases are why: the + * same worker id claiming an operation again must fence out its own earlier run, which a check on + * the worker id alone lets through. */ public abstract class DurableOperationStoreContract { @@ -33,6 +38,9 @@ public abstract class DurableOperationStoreContract { private static final Duration TTL = Duration.ofDays(1); private static final String HASH_A = "a".repeat(64); private static final String HASH_B = "b".repeat(64); + private static final String HASH_C = "c".repeat(64); + private static final OperationFailure TIMED_OUT = + new OperationFailure("DEPENDENCY_TIMEOUT", "timed out", true); /** The store under test, empty at the start of each test. */ protected abstract DurableOperationStorePort store(); @@ -182,10 +190,36 @@ public abstract class DurableOperationStoreContract { submit(submission); // when - Optional claimed = claim("worker-a", NOW); + Optional claimed = claim("worker-a", NOW); // then - assertThat(claimed).contains(submission); + assertThat(claimed).contains(new DurableOperationClaim(submission, "worker-a", 1)); + } + + @Test + void everyClaimOfAnOperationGetsAHigherFence() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim first = claim("worker-a", NOW).orElseThrow(); + + // when + DurableOperationClaim second = claim("worker-a", NOW.plus(LEASE).plusSeconds(1)).orElseThrow(); + + // then + assertThat(second.fence()).isGreaterThan(first.fence()); + } + + @Test + void canceledOperationIsNotHandedToAWorker() { + // given + submit(submission("op-1", null, HASH_A)); + cancel("op-1", "alice", NOW.plusSeconds(5)); + + // when + Optional claimed = claim("worker-a", NOW.plusSeconds(6)); + + // then + assertThat(claimed).isEmpty(); } @Test @@ -195,7 +229,7 @@ public abstract class DurableOperationStoreContract { claim("worker-a", NOW); // when - Optional claimed = claim("worker-b", NOW.plusSeconds(30)); + Optional claimed = claim("worker-b", NOW.plusSeconds(30)); // then assertThat(claimed).isEmpty(); @@ -224,11 +258,11 @@ public abstract class DurableOperationStoreContract { void heartbeatFromTheLeaseHolderExtendsTheLease() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); Instant beat = NOW.plusSeconds(60); // when - inTransaction(() -> store().heartbeat(id("op-1"), "worker-a", LEASE, beat)); + inTransaction(() -> store().heartbeat(held, LEASE, beat)); // then assertThat(find("op-1", "alice", beat).orElseThrow().lease()) @@ -239,13 +273,12 @@ public abstract class DurableOperationStoreContract { void heartbeatFromAWorkerThatLostTheLeaseIsRefused() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim stale = claim("worker-a", NOW).orElseThrow(); Instant later = NOW.plus(LEASE).plusSeconds(1); claim("worker-b", later); // when - boolean extended = - inTransaction(() -> store().heartbeat(id("op-1"), "worker-a", LEASE, later.plusSeconds(1))); + boolean extended = inTransaction(() -> store().heartbeat(stale, LEASE, later.plusSeconds(1))); // then assertThat(extended).isFalse(); @@ -255,12 +288,12 @@ public abstract class DurableOperationStoreContract { void progressFromTheLeaseHolderIsPublished() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); OperationProgressSnapshot progress = new OperationProgressSnapshot(3, Optional.of(10L), Optional.of("extracting")); // when - inTransaction(() -> store().reportProgress(id("op-1"), "worker-a", progress)); + inTransaction(() -> store().reportProgress(held, progress)); // then assertThat(find("op-1", "alice", NOW).orElseThrow().progress()).contains(progress); @@ -270,12 +303,13 @@ public abstract class DurableOperationStoreContract { void progressFromAnotherWorkerIsRefused() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); + DurableOperationClaim impostor = + new DurableOperationClaim(held.submission(), "worker-b", held.fence()); // when boolean published = - inTransaction( - () -> store().reportProgress(id("op-1"), "worker-b", OperationProgressSnapshot.of(1))); + inTransaction(() -> store().reportProgress(impostor, OperationProgressSnapshot.of(1))); // then assertThat(published).isFalse(); @@ -285,11 +319,11 @@ public abstract class DurableOperationStoreContract { void successFromTheLeaseHolderIsRecorded() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); Instant done = NOW.plusSeconds(10); // when - inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", done)); + inTransaction(() -> store().succeed(held, "/results/a", done)); // then assertThat(find("op-1", "alice", done).orElseThrow()) @@ -309,28 +343,106 @@ public abstract class DurableOperationStoreContract { void staleWorkerCannotRecordAResultOverItsSuccessor() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim stale = claim("worker-a", NOW).orElseThrow(); Instant later = NOW.plus(LEASE).plusSeconds(1); claim("worker-b", later); // when boolean recorded = - inTransaction( - () -> store().succeed(id("op-1"), "worker-a", "/results/a", later.plusSeconds(1))); + inTransaction(() -> store().succeed(stale, "/results/a", later.plusSeconds(1))); // then assertThat(recorded).isFalse(); } + @Test + void heartbeatOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = inTransaction(() -> store().heartbeat(earlier, LEASE, later.plusSeconds(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void progressOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = + inTransaction(() -> store().reportProgress(earlier, OperationProgressSnapshot.of(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void successOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = + inTransaction(() -> store().succeed(earlier, "/results/earlier", later.plusSeconds(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void failureOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = inTransaction(() -> store().fail(earlier, TIMED_OUT, later.plusSeconds(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void resultOfTheRunThatTookOverSurvivesItsSameWorkerPredecessor() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + DurableOperationClaim current = claim("worker-a", later).orElseThrow(); + inTransaction(() -> store().succeed(current, "/results/current", later.plusSeconds(1))); + + // when + inTransaction(() -> store().succeed(earlier, "/results/earlier", later.plusSeconds(2))); + + // then + assertThat(find("op-1", "alice", later.plusSeconds(2)).orElseThrow().resultReference()) + .contains("/results/current"); + } + @Test void failureFromTheLeaseHolderIsRecorded() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); OperationFailure failure = new OperationFailure("DEPENDENCY_TIMEOUT", "timed out", true); // when - inTransaction(() -> store().fail(id("op-1"), "worker-a", failure, NOW.plusSeconds(10))); + inTransaction(() -> store().fail(held, failure, NOW.plusSeconds(10))); // then assertThat(find("op-1", "alice", NOW.plusSeconds(10)).orElseThrow()) @@ -369,8 +481,8 @@ public abstract class DurableOperationStoreContract { void cancelNeverReversesASuccess() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); - inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", NOW.plusSeconds(10))); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); + inTransaction(() -> store().succeed(held, "/results/a", NOW.plusSeconds(10))); // when cancel("op-1", "alice", NOW.plusSeconds(20)); @@ -407,19 +519,35 @@ public abstract class DurableOperationStoreContract { } @Test - void expirySweepExpiresAgedOutWorkAndLeavesFinishedWorkAlone() { + void reclaimLeavesALiveLeaseAlone() { // given submit(submission("op-1", null, HASH_A)); claim("worker-a", NOW); - inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", NOW.plusSeconds(10))); + + // when + List reclaimed = + inTransaction(() -> store().reclaimExpiredLeases(NOW.plusSeconds(30))); + + // then + assertThat(reclaimed).isEmpty(); + } + + @Test + void expirySweepExpiresAgedOutWorkAndLeavesFinishedWorkAlone() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); + inTransaction(() -> store().succeed(held, "/results/a", NOW.plusSeconds(10))); submit(submission("op-2", null, HASH_B)); + submit(submission("op-3", null, HASH_C)); + claim("worker-b", NOW.plusSeconds(20)); // when List expired = inTransaction(() -> store().expireStaleOperations(NOW.plus(TTL).plus(TTL))); // then - assertThat(expired).containsExactly(id("op-2")); + assertThat(expired).containsExactlyInAnyOrder(id("op-2"), id("op-3")); } private DurableOperationSubmitResult submit(DurableOperationSubmission submission) { @@ -430,7 +558,7 @@ public abstract class DurableOperationStoreContract { return inTransaction(() -> store().find(id(operationId), principal, now)); } - private Optional claim(String workerId, Instant now) { + private Optional claim(String workerId, Instant now) { return inTransaction(() -> store().claimNext(workerId, LEASE, now)); } diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java index 019193c3..3c76c99a 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java @@ -1,6 +1,7 @@ package dev.caskeleton.application.operation.testkit; import dev.caskeleton.application.operation.DurableOperation; +import dev.caskeleton.application.operation.DurableOperationClaim; import dev.caskeleton.application.operation.DurableOperationId; import dev.caskeleton.application.operation.DurableOperationState; import dev.caskeleton.application.operation.DurableOperationStorePort; @@ -26,8 +27,10 @@ import java.util.concurrent.ConcurrentHashMap; *

Written to the same rules rather than to whatever makes the tests pass: a submission scope * that already exists returns the existing operation, a lease is checked on every write, and a * terminal operation is never reopened. A fake that was laxer than the adapter would let the use - * case tests certify behaviour the real store refuses. {@link DurableOperationStoreContract} holds - * this fake and the adapter to the same cases. + * case tests certify behaviour the real store refuses. Every claim increments the operation's + * fence, and a write is accepted only from the claim holding the current fence, as the adapter's + * {@code lease_fence} column does. {@link DurableOperationStoreContract} holds this fake and the + * adapter to the same cases. * *

Built over a {@link DirectTransactionPort}, every write made inside one of its transactions is * undone when that transaction rolls back, as the database undoes it; built without one, writes @@ -41,6 +44,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor private final Map byScope = new ConcurrentHashMap<>(); private final Map submissions = new ConcurrentHashMap<>(); + private final Map fences = new ConcurrentHashMap<>(); private final DirectTransactionPort transactions; /** A store whose writes stay as they are made. */ @@ -103,7 +107,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public Optional claimNext( + public Optional claimNext( String workerId, Duration leaseDuration, Instant now) { return byId.values().stream() .filter( @@ -126,19 +130,22 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor Optional.empty(), Optional.of(new OperationLease(workerId, now, now.plus(leaseDuration)))); write(claimed.operationId(), claimed); - return submissions.get(claimed.operationId()); + long previous = fences.getOrDefault(claimed.operationId(), 0L); + fences.put(claimed.operationId(), previous + 1); + onRollback(() -> fences.put(claimed.operationId(), previous)); + return new DurableOperationClaim( + submissions.get(claimed.operationId()), workerId, previous + 1); }); } @Override - public boolean heartbeat( - DurableOperationId operationId, String workerId, Duration leaseDuration, Instant now) { - DurableOperation operation = byId.get(operationId); - if (!holdsLease(operation, workerId, now)) { + public boolean heartbeat(DurableOperationClaim claim, Duration leaseDuration, Instant now) { + DurableOperation operation = byId.get(claim.operationId()); + if (!holdsLease(operation, claim, now)) { return false; } write( - operationId, + claim.operationId(), replace( operation, operation.state(), @@ -152,14 +159,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public boolean reportProgress( - DurableOperationId operationId, String workerId, OperationProgressSnapshot progress) { - DurableOperation operation = byId.get(operationId); - if (operation == null || !ownedBy(operation, workerId)) { + public boolean reportProgress(DurableOperationClaim claim, OperationProgressSnapshot progress) { + DurableOperation operation = byId.get(claim.operationId()); + if (!heldBy(operation, claim)) { return false; } write( - operationId, + claim.operationId(), replace( operation, operation.state(), @@ -173,17 +179,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public boolean succeed( - DurableOperationId operationId, - String workerId, - String resultReference, - Instant completedAt) { - DurableOperation operation = byId.get(operationId); - if (operation == null || !ownedBy(operation, workerId)) { + public boolean succeed(DurableOperationClaim claim, String resultReference, Instant completedAt) { + DurableOperation operation = byId.get(claim.operationId()); + if (!heldBy(operation, claim)) { return false; } write( - operationId, + claim.operationId(), replace( operation, DurableOperationState.SUCCEEDED, @@ -197,17 +199,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public boolean fail( - DurableOperationId operationId, - String workerId, - OperationFailure failure, - Instant completedAt) { - DurableOperation operation = byId.get(operationId); - if (operation == null || !ownedBy(operation, workerId)) { + public boolean fail(DurableOperationClaim claim, OperationFailure failure, Instant completedAt) { + DurableOperation operation = byId.get(claim.operationId()); + if (!heldBy(operation, claim)) { return false; } write( - operationId, + claim.operationId(), replace( operation, DurableOperationState.FAILED, @@ -316,6 +314,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor byId.clear(); byScope.clear(); submissions.clear(); + fences.clear(); } // Same as the adapter: a poll after the TTL sees EXPIRED even before the sweeper runs. @@ -334,13 +333,15 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor Optional.empty()); } - private static boolean holdsLease(DurableOperation operation, String workerId, Instant now) { - return operation != null && ownedBy(operation, workerId) && !operation.leaseExpiredAt(now); + private boolean holdsLease(DurableOperation operation, DurableOperationClaim claim, Instant now) { + return heldBy(operation, claim) && !operation.leaseExpiredAt(now); } - private static boolean ownedBy(DurableOperation operation, String workerId) { - return operation.state() == DurableOperationState.RUNNING - && operation.lease().map(lease -> lease.workerId().equals(workerId)).orElse(false); + private boolean heldBy(DurableOperation operation, DurableOperationClaim claim) { + return operation != null + && operation.state() == DurableOperationState.RUNNING + && operation.lease().map(lease -> lease.workerId().equals(claim.workerId())).orElse(false) + && fences.getOrDefault(claim.operationId(), 0L) == claim.fence(); } private static DurableOperation replace( diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java index 4577fec0..ad90e67d 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java @@ -44,6 +44,16 @@ public interface CleanupQueuePort { /** Records a failed attempt and when to try again, if {@code item}'s claim is still current. */ void markFailed(CleanupItem item, String reasonCode, Instant nextAttemptAt); + /** + * Gives a claimed item back unattempted, if {@code item}'s claim is still current. + * + *

For an item the batch claimed and then did not start — its byte budget or its claim's lease + * ran out first. The claim ends and the item is claimable again at {@code nextAttemptAt}; the + * attempt counter does not move, because nothing was tried. Recorded as a failure instead, every + * such release spent an attempt, and a busy queue's tail was abandoned without ever being tried. + */ + void release(CleanupItem item, Instant nextAttemptAt); + /** * Parks an item for good, if {@code item}'s claim is still current. * diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java index d28a7d40..71033678 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java @@ -49,6 +49,20 @@ public final class DefaultCleanupService implements CleanupService { */ public static final Duration CLAIM_LEASE = Duration.ofMinutes(10); + /** + * No claimed item is started once less than this is left of the batch's claim lease. + * + *

A batch claims up to {@code maxItems} under one lease and works them one after another, so + * the last item starts long after the claim. Past the lease, the reaper may already have handed + * that item to another worker, and both delete it and settle its quota. The outbox relay checks + * its batch against its lease when it is assembled and the notification scheduler claims only + * what it can start; cleanup has no bound on one delete to check against, so it stops starting + * items instead, and releases the rest for the next batch the way the byte budget does (round9 + * review 2026-11-12, RR9-01-23). Two minutes is the time one delete against slow storage is + * allowed to finish inside its lease. + */ + static final Duration LAST_START_BEFORE_LEASE_ENDS = Duration.ofMinutes(2); + private final CleanupQueuePort queue; private final CleanupContentPort contentGateway; private final FileMetadataPort metadataStore; @@ -95,6 +109,7 @@ public final class DefaultCleanupService implements CleanupService { // claiming means the recovered items are eligible in this same pass. transactions.inWrite(() -> queue.reclaimExpiredClaims(now, maxItems)); List due = transactions.inWrite(() -> queue.claimDue(now, maxItems, CLAIM_LEASE)); + Instant lastStart = now.plus(CLAIM_LEASE).minus(LAST_START_BEFORE_LEASE_ENDS); int deleted = 0; int skippedActiveLease = 0; @@ -113,10 +128,16 @@ public final class DefaultCleanupService implements CleanupService { failed++; continue; } + if (!clock.instant().isBefore(lastStart)) { + // The claim's lease is nearly spent; an item started now could still be running when the + // reaper hands it to another worker. Given back untried, like the byte budget's. + release(item, now); + continue; + } if (reclaimed >= maxBytes) { - // The byte budget is spent; the remaining claims are released for the next batch rather - // than being marked done, so nothing is silently dropped. - markFailed(item, "BATCH_BYTE_BUDGET_EXHAUSTED", now); + // The byte budget is spent; the remaining claims are given back untried rather than + // being marked done, so nothing is silently dropped and no attempt is spent on them. + release(item, now); continue; } Outcome outcome = process(item, now); @@ -330,6 +351,11 @@ public final class DefaultCleanupService implements CleanupService { }); } + /** Gives a claimed item back untried, in a write transaction of its own. */ + private void release(CleanupItem item, Instant nextAttemptAt) { + transactions.inWrite(() -> queue.release(item, nextAttemptAt)); + } + /** * Reschedules an item, or parks it once this failure spends its retry budget. * diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java index dd0bd88d..ead9afb2 100644 --- a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java +++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java @@ -26,7 +26,10 @@ import dev.caskeleton.application.fileserver.testkit.ReadyFileFixture; import dev.caskeleton.application.fileserver.testkit.RecordingQuotaReclaimGateway; import java.time.Clock; import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.List; import java.util.OptionalLong; import java.util.UUID; import org.junit.jupiter.api.DisplayName; @@ -321,7 +324,91 @@ class CleanupServiceTest { CleanupBatchResult result = cleanup.runBatch(100, 1); assertThat(result.deleted()).isEqualTo(1); - assertThat(queue.failed().values()).contains("BATCH_BYTE_BUDGET_EXHAUSTED"); + assertThat(List.of(queue.released().size(), queue.failed().size())) + .as("the deferred item is given back untried; recorded as failed it spent an attempt") + .containsExactly(1, 0); + } + + @Test + void itemsTheClaimLeaseNoLongerCoversAreReleasedNotStarted() { + // given + // Each staging delete takes just over half the time an item may still start in (the lease less + // its last two minutes): the first two items run, the third is handed back to the queue. + MovableClock slowStorage = new MovableClock(FileserverFixtures.NOW); + Duration perDelete = + DefaultCleanupService.CLAIM_LEASE + .minus(DefaultCleanupService.LAST_START_BEFORE_LEASE_ENDS) + .dividedBy(2) + .plusSeconds(1); + content.afterEachStagingDiscard(() -> slowStorage.advance(perDelete)); + CleanupService timed = + new DefaultCleanupService( + queue, + content, + metadata, + sessions, + quota, + Duration.ofMinutes(5), + FileserverMetricsPort.noop(), + transactions, + slowStorage); + for (int item = 0; item < 3; item++) { + cancelledStagingItem(); + } + + // when + CleanupBatchResult result = timed.runBatch(100, 1L << 30); + + // then + assertThat(List.of(result.deleted(), queue.released().size(), queue.failed().size())) + .as( + "the third item comes back untried rather than failed; that a release leaves the" + + " attempt counter where it was is the queue contract's") + .containsExactly(2, 1, 0); + } + + private void cancelledStagingItem() { + UploadId uploadId = UploadId.of(UUID.randomUUID()); + FileId fileId = FileId.of(UUID.randomUUID()); + sessions.create( + new UploadSessionDraft( + uploadId, + fileId, + UploadMode.SINGLE_SHOT, + OptionalLong.of(SIZE), + FileserverFixtures.NOW.plusSeconds(3600))); + sessions.terminate(uploadId); + content.storeStaging(uploadId); + queue.enqueue(CleanupRequest.forStaging(CleanupType.CANCELLED_STAGING, fileId, uploadId)); + } + + /** A clock the slow-storage gateway moves forward. */ + private static final class MovableClock extends Clock { + + private Instant now; + + private MovableClock(Instant start) { + this.now = start; + } + + private void advance(Duration duration) { + now = now.plus(duration); + } + + @Override + public ZoneId getZone() { + return ZoneOffset.UTC; + } + + @Override + public Clock withZone(ZoneId zone) { + return this; + } + + @Override + public Instant instant() { + return now; + } } @Test diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java index 0db36011..b9cc169b 100644 --- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java +++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java @@ -119,6 +119,40 @@ public abstract class CleanupQueuePortContract { }); } + @Test + void aReleasedItemIsClaimableAgainWithItsAttemptUnchanged() { + // given + enqueue(contentCleanup()); + CleanupItem item = claim(due(), 10).get(0); + settle(() -> queue().release(item, due())); + + // when + List again = claim(due(), 10); + + // then + assertThat(again) + .singleElement() + .extracting(CleanupItem::cleanupId, CleanupItem::attempt) + .containsExactly(item.cleanupId(), 0); + } + + @Test + void aReleaseUnderAReclaimedTokenChangesNothing() { + // given + enqueue(contentCleanup()); + CleanupItem stale = claim(due(), 10).get(0); + settle(() -> queue().release(stale, due())); + var unused = claim(due(), 10); + + // when + settle(() -> queue().release(stale, due())); + + // then + assertThat(claim(due(), 10)) + .as("the stale release would hand the current claim's item to a second worker") + .isEmpty(); + } + @Test @DisplayName("완료된 항목은 다시 claim되지 않는다") void aDoneItemIsNeverClaimedAgain() { diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java index 8a485a3c..3b842068 100644 --- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java +++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java @@ -7,6 +7,7 @@ import dev.caskeleton.application.fileserver.api.content.DeleteResult; import dev.caskeleton.application.fileserver.cleanup.CleanupContentPort; import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; import java.util.Set; /** Tracks which physical objects still exist so a test can prove nothing live was removed. */ @@ -16,6 +17,7 @@ public final class FakeCleanupContentGateway implements CleanupContentPort { private final Set stagingObjects = new LinkedHashSet<>(); private final Set quarantined = new LinkedHashSet<>(); private final List preconditions = new java.util.ArrayList<>(); + private Runnable afterStagingDiscard = () -> {}; public void store(ContentKey key) { objects.add(key); @@ -46,9 +48,20 @@ public final class FakeCleanupContentGateway implements CleanupContentPort { return DeleteResult.removed(precondition.expectedSize().orElse(0)); } + /** + * Runs {@code hook} after every staging discard, for a test that needs the storage to be slow — + * advancing its clock — or to fail mid-batch. + * + * @param hook what a discard does after removing the staging object + */ + public void afterEachStagingDiscard(Runnable hook) { + afterStagingDiscard = Objects.requireNonNull(hook, "hook"); + } + @Override public void discardStaging(UploadId uploadId) { stagingObjects.remove(uploadId); + afterStagingDiscard.run(); } public void quarantine(ContentKey key) { diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java index 50edd600..0dc81d84 100644 --- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java +++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java @@ -35,6 +35,7 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { private final Map rows = new LinkedHashMap<>(); private final List done = new ArrayList<>(); private final Map failed = new LinkedHashMap<>(); + private final List released = new ArrayList<>(); private final Map abandoned = new LinkedHashMap<>(); private final List reclaimSweeps = new ArrayList<>(); private final List claimSweeps = new ArrayList<>(); @@ -108,6 +109,15 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { } } + @Override + public void release(CleanupItem item, Instant nextAttemptAt) { + Row row = rows.get(item.cleanupId()); + if (row != null && item.claimToken().equals(row.claimToken)) { + row.release(nextAttemptAt); + released.add(item); + } + } + @Override public void markAbandoned(CleanupItem item, String reasonCode) { if (settle(item, Status.ABANDONED, clock.instant())) { @@ -151,6 +161,11 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { return Map.copyOf(failed); } + /** Items the caller gave back untried, in order. */ + public List released() { + return List.copyOf(released); + } + /** Items the caller parked for good, with the reason given. */ public Map abandoned() { return Map.copyOf(abandoned); @@ -180,6 +195,13 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { this.nextAttemptAt = nextAttemptAt; } + private void release(Instant nextAttemptAt) { + this.status = attempt == 0 ? Status.PENDING : Status.FAILED; + this.nextAttemptAt = nextAttemptAt; + this.claimToken = null; + this.leaseUntil = null; + } + private void settle(Status status, Instant nextAttemptAt) { this.status = status; this.attempt++; diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java index bd902930..40006855 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java @@ -11,6 +11,10 @@ import java.util.Optional; * the work is done; and the same id with a different command is a mistake worth reporting rather * than an idempotent repeat. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param outcome what the caller may do * @param existing the earlier result, when there is one */ @@ -37,7 +41,7 @@ public record AdminOperationClaim(Outcome outcome, OptionalA requeue is one write conditional on the abandoned lease recovery found (owner, fence, state, + * expiry), not a read followed by an unconditional transition. The pass runs on every instance, and + * between finding a job and writing it another instance's recovery may have requeued it and a new + * worker claimed it, or the holder may have finished; the unconditional write put such a job back + * to {@code READY_TO_DISPATCH} and it was dispatched again (round9 review 2026-11-12, RR9-01-12). A + * requeue that matches nothing is not counted as handled. + * *

Every write opens its own transaction; the pass itself runs on a bare scheduler thread. The * stores behind these ports are modifying JPA queries, which throw outside a transaction — so the * pass used to fail on its first statement, every time, and the scheduler only logged it. It lives @@ -110,14 +116,14 @@ public final class LeaseRecoveryService { // Retiring expired deliveries rides along with recovery because both answer the same operator // question — "why is this row still here?" — and neither needs a scheduler of its own. int handled = transactions.inWrite(() -> leases.expireOverdue(batchSize)); - List abandoned = leases.expiredDispatching(batchSize, staleAfter); - for (var recipientDeliveryId : abandoned) { + List abandoned = leases.expiredDispatching(batchSize, staleAfter); + for (var lease : abandoned) { + RecipientDeliveryId recipientDeliveryId = lease.recipientDeliveryId(); var attemptsOfDelivery = attempts.attemptsOf(recipientDeliveryId); if (attemptsOfDelivery.isEmpty()) { // Crash before the attempt row: no provider call can have happened, so this is the one // safe requeue. Leaving it DISPATCHING stranded the delivery permanently. - requeue(recipientDeliveryId); - handled++; + handled += requeue(lease); continue; } if (attemptsOfDelivery.stream().allMatch(attempt -> attempt.completedAt().isPresent())) { @@ -127,8 +133,7 @@ public final class LeaseRecoveryService { // the delivery, and dispatch derives the next step from exactly those facts — an accepted // submission stops as COMPLETED, an ambiguous one goes to reconciliation — so handing it // back to dispatch cannot send a second copy of something the provider accepted. - requeue(recipientDeliveryId); - handled++; + handled += requeue(lease); continue; } for (var attempt : attemptsOfDelivery) { @@ -141,25 +146,25 @@ public final class LeaseRecoveryService { // identically, so every abandoned attempt had to be reconciled. The pre-call row the // dispatcher writes never lands here: it records the start as unknown, because it is // still the stored state while the provider call runs. - requeue(recipientDeliveryId); + handled += requeue(lease); } else { // Outside any transaction: reconciliation asks the provider over the network and opens // its own boundaries for what it writes. reconciliation.reconcile(attempt.id()); + handled++; } - handled++; } } } return handled; } - private void requeue(RecipientDeliveryId recipientDeliveryId) { - transactions.inWrite( - () -> - deliveries.transition( - recipientDeliveryId, - RecipientDeliveryState.READY_TO_DISPATCH, - java.util.Optional.of(clock.instant()))); + /** Requeues the abandoned dispatch; 1 when it was still abandoned and moved, 0 otherwise. */ + private int requeue(RecipientLease abandoned) { + return transactions + .inWrite(() -> deliveries.requeueAbandoned(abandoned, clock.instant())) + .isPresent() + ? 1 + : 0; } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java index 101c9f35..c82fa5aa 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java @@ -49,4 +49,28 @@ public interface RecipientDeliveryStorePort { RecipientDeliveryState state, Optional nextDispatchAt, RecipientLease lease); + + /** + * Returns a job a dead worker left dispatching to the queue, only while it is still the abandoned + * dispatch recovery found. + * + *

One conditional write on the abandoned lease — same owner, same fence, still {@code + * DISPATCHING}, lease not extended past what recovery saw — rather than a read followed by an + * unconditional transition. Recovery runs on every instance, and between finding the job and + * writing it another instance may have requeued it and a new worker claimed it, or the original + * holder may have finished; an unconditional write put such a job back to {@code + * READY_TO_DISPATCH} and it was dispatched again. + * + *

The target state is the operation's meaning, as {@code EXPIRED} is {@code + * RecipientLeaseStorePort.expireOverdue}'s, so it is not an argument. What recovery decides is + * which operation applies: this one when no attempt may have reached the provider, the + * reconciliation path otherwise. + * + * @param abandoned the lease as {@link RecipientLeaseStorePort#expiredDispatching} returned it + * @param nextDispatchAt when the job becomes due again + * @return the requeued job, or empty when it is no longer that abandoned dispatch and nothing was + * written + */ + Optional requeueAbandoned( + RecipientLease abandoned, Instant nextDispatchAt); } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java index 0b9962a3..5660e4d3 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java @@ -16,6 +16,10 @@ import java.util.Objects; * that no longer exists and updates nothing. It travels with the lease rather than being re-read, * because re-reading is the race. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param recipientDeliveryId the job this lease is for * @param owner the holder, including the process incarnation that claimed it * @param fence the claim generation; a lower fence has been superseded @@ -29,11 +33,11 @@ public record RecipientLease( Objects.requireNonNull(owner, "owner"); Objects.requireNonNull(leaseUntil, "leaseUntil"); if (owner.isBlank()) { - throw new IllegalArgumentException("owner"); + throw new IllegalStateException("a lease needs an owner"); } if (fence < 1) { // A zero fence is the value an unclaimed row holds; a lease can never legitimately carry it. - throw new IllegalArgumentException("fence"); + throw new IllegalStateException("fence must be >= 1, was " + fence); } } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java index 634fabd7..da251a5d 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java @@ -1,6 +1,5 @@ package dev.caskeleton.application.notification.platform.dispatch; -import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; import java.time.Duration; import java.util.List; import java.util.Optional; @@ -25,6 +24,9 @@ public interface RecipientLeaseStorePort { /** * Extend a lease that is still being worked on. * + *

The template's dispatcher never calls it: a dispatch runs inside the lease it claimed. It is + * here for a deployment whose own dispatch outlasts one lease. + * * @return the extended lease, or empty when this holder no longer owns the job */ Optional renew(RecipientLease lease, Duration leaseDuration); @@ -40,8 +42,24 @@ public interface RecipientLeaseStorePort { /** Whether this lease is still the current one, for checking before an external side effect. */ boolean stillHeld(RecipientLease lease); - /** Jobs left {@code DISPATCHING} by a worker that died, for reconciliation. */ - List expiredDispatching(int limit, Duration olderThan); + /** + * Jobs left {@code DISPATCHING} by a worker that died, for recovery. + * + *

Each comes back as the lease it was abandoned under — owner, fence and expiry as they stood + * when it was found. Recovery decides what to do from attempts it reads afterwards, and the lease + * is what lets its write say "only if this is still that abandoned dispatch": another instance's + * recovery, a new claim or the holder itself may have moved the job in between. + * + *

Only jobs a claim leased: an owner and a fence of at least one. A {@code DISPATCHING} job + * without them is no lease anyone holds — one left before the fence existed, or one moved there + * without a claim — and is not returned; offered, it would be a lease nobody can build, and the + * pass would fail on it every time. + * + * @param limit how many to return + * @param olderThan how long a lease must have been expired before the job counts as abandoned + * @return the abandoned leases, oldest expiry first + */ + List expiredDispatching(int limit, Duration olderThan); /** * Retire deliveries whose window closed before anyone claimed them. diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java index c8545b62..e0d3e401 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java @@ -24,6 +24,10 @@ import java.util.UUID; * the store's writes after the claim match on it, so an instance that held the job past its lease * cannot reschedule or remove it once another instance has claimed it since. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param id the job identity * @param attemptId the attempt whose outcome is unknown * @param providerProfileId the profile that must answer, since capability is per profile @@ -55,10 +59,10 @@ public record ReconciliationJob( Objects.requireNonNull(createdAt, "createdAt"); Objects.requireNonNull(updatedAt, "updatedAt"); if (attempts < 0) { - throw new IllegalArgumentException("attempts must not be negative"); + throw new IllegalStateException("attempts must not be negative"); } if (claimFence < 0) { - throw new IllegalArgumentException("claimFence must not be negative"); + throw new IllegalStateException("claimFence must not be negative"); } } } diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java index 7ff29358..ed508150 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java @@ -186,6 +186,40 @@ class LeaseRecoveryServiceTest { assertThat(recovery.recoverOnce()).isEqualTo(3); } + @Test + void aDeliveryItsHolderFinishedAfterRecoveryFoundItIsNotRequeued() { + // given + RecipientDeliveryId id = jobOf(NOW, Optional.empty()); + RecipientLease lease = queue.claim("worker-a", 10, Duration.ofSeconds(30)).getFirst(); + workerDied(); + // The holder completes the job between recovery finding it abandoned and recovery writing it + // — the window another instance or a slow worker uses in production. + attempts.beforeAttemptsOf( + delivery -> { + var unused = + queue.transitionHeldBy( + delivery, RecipientDeliveryState.COMPLETED, Optional.empty(), lease); + }); + LeaseRecoveryService racing = + new LeaseRecoveryService( + queue, + attempts, + queue, + reconciled::add, + new DirectTransactionPort(), + CLOCK, + Duration.ofMinutes(1), + 10); + + // when + racing.recoverOnce(); + + // then + assertThat(stateOf(id)) + .as("the holder was only slow; recovery's requeue must not reopen what it finished") + .isEqualTo(RecipientDeliveryState.COMPLETED); + } + private static dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence notStarted() { return dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java new file mode 100644 index 00000000..dd33283f --- /dev/null +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java @@ -0,0 +1,76 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.notification.platform.admin.AdminOperationClaim; +import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; +import dev.caskeleton.application.notification.platform.api.ProviderProfileId; +import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * The leases, jobs and claims a notification store builds from its rows refuse a bad value as each + * record says. + */ +class StoreBuiltRecordsTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + + static Stream refusedValues() { + RecipientDeliveryId delivery = new RecipientDeliveryId(UUID.randomUUID()); + return Stream.of( + Arguments.of( + "owner 없는 수신자 lease", + (ThrowingCallable) () -> new RecipientLease(delivery, " ", 1, NOW), + "a lease needs an owner"), + Arguments.of( + "fence 0인 수신자 lease", + (ThrowingCallable) () -> new RecipientLease(delivery, "worker-1", 0, NOW), + "fence must be >= 1, was 0"), + Arguments.of( + "시도 수가 음수인 맞추기 job", + (ThrowingCallable) () -> job(-1, 0), + "attempts must not be negative"), + Arguments.of( + "fence가 음수인 맞추기 job", + (ThrowingCallable) () -> job(0, -1), + "claimFence must not be negative"), + Arguments.of( + "결과 없는 운영자 재생", + (ThrowingCallable) + () -> new AdminOperationClaim(AdminOperationClaim.Outcome.REPLAY, Optional.empty()), + "a replay must carry the result it is replaying")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("refusedValues") + void aValueAStoreCannotHaveBuiltIsAServerFault( + String name, ThrowingCallable build, String message) { + // when + Throwable refused = catchThrowable(build); + + // then + assertThat(refused).isInstanceOf(IllegalStateException.class).hasMessageContaining(message); + } + + private static ReconciliationJob job(int attempts, long claimFence) { + return new ReconciliationJob( + UUID.randomUUID(), + new DeliveryAttemptId(UUID.randomUUID()), + new ProviderProfileId("email-main"), + NOW, + attempts, + Optional.empty(), + NOW, + NOW, + claimFence); + } +} diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java index ae257085..65bc4d4d 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; /** * The fake of the attempt store and the two read models built on it, held to {@link @@ -47,6 +48,8 @@ public final class InMemoryDeliveryAttemptStore /** Stored attempts, in insertion order, as their rows read back. */ private final Map rows = new LinkedHashMap<>(); + private volatile Consumer beforeAttemptsOf = id -> {}; + private final Map requestIdHashes = new LinkedHashMap<>(); private final Map engagement = new LinkedHashMap<>(); private final Map suppression = new LinkedHashMap<>(); @@ -140,7 +143,23 @@ public final class InMemoryDeliveryAttemptStore } @Override - public synchronized List attemptsOf( + public List attemptsOf(RecipientDeliveryId recipientDeliveryId) { + // Outside the monitor: the hook writes to the recipient queue, as another worker would. + beforeAttemptsOf.accept(recipientDeliveryId); + return attemptsFor(recipientDeliveryId); + } + + /** + * Runs {@code hook} before every {@link #attemptsOf} read, for a test that needs another worker + * to act between a caller finding a delivery and reading its attempts. + * + * @param hook what happens, given the delivery being read + */ + public void beforeAttemptsOf(Consumer hook) { + beforeAttemptsOf = Objects.requireNonNull(hook, "hook"); + } + + private synchronized List attemptsFor( RecipientDeliveryId recipientDeliveryId) { return rows.values().stream() .filter(row -> row.recipientDeliveryId().equals(recipientDeliveryId)) @@ -155,7 +174,7 @@ public final class InMemoryDeliveryAttemptStore @Override public synchronized int nextAttemptNo(RecipientDeliveryId recipientDeliveryId) { - return attemptsOf(recipientDeliveryId).stream() + return attemptsFor(recipientDeliveryId).stream() .mapToInt(DeliveryAttemptRecord::attemptNo) .max() .orElse(0) diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java index 89517a47..7714a522 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java @@ -160,7 +160,7 @@ public final class InMemoryRecipientQueue } @Override - public List expiredDispatching(int limit, Duration olderThan) { + public List expiredDispatching(int limit, Duration olderThan) { Objects.requireNonNull(olderThan, "olderThan"); if (limit < 1) { throw new IllegalArgumentException("limit"); @@ -170,9 +170,18 @@ public final class InMemoryRecipientQueue return jobs() .filter(job -> job.state() == RecipientDeliveryState.DISPATCHING) .filter(job -> job.leaseUntil().filter(until -> until.isBefore(threshold)).isPresent()) + // Only what a claim wrote: an owner and a fence. The JPA store skips the rest the same + // way. + .filter(job -> job.leaseOwner().isPresent() && fences.getOrDefault(job.id(), 0L) > 0) .sorted(Comparator.comparing((RecipientDeliveryRecord job) -> job.leaseUntil().get())) .limit(limit) - .map(RecipientDeliveryRecord::id) + .map( + job -> + new RecipientLease( + job.id(), + job.leaseOwner().orElseThrow(), + fences.getOrDefault(job.id(), 0L), + job.leaseUntil().orElseThrow())) .toList(); } } @@ -294,6 +303,35 @@ public final class InMemoryRecipientQueue } } + @Override + public Optional requeueAbandoned( + RecipientLease abandoned, Instant nextDispatchAt) { + Objects.requireNonNull(abandoned, "abandoned"); + Objects.requireNonNull(nextDispatchAt, "nextDispatchAt"); + synchronized (requests) { + return heldBy(abandoned) + .filter(job -> job.state() == RecipientDeliveryState.DISPATCHING) + .filter( + job -> + job.leaseUntil() + .filter(until -> !until.isAfter(abandoned.leaseUntil())) + .isPresent()) + .map( + job -> { + RecipientDeliveryRecord moved = + copy( + job, + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(nextDispatchAt), + job.leaseOwner(), + job.leaseUntil(), + clock.instant()); + requests.replace(moved); + return moved; + }); + } + } + // ---- rows ------------------------------------------------------------------------------------- private Stream jobs() { diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java index 7ceefe82..5ec3dead 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java @@ -244,6 +244,84 @@ public abstract class RecipientDeliveryStoreContract { List.of(Optional.empty(), Optional.empty()), RecipientDeliveryState.DISPATCHING, 0); } + @Test + void anAbandonedDispatchIsRequeued() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease abandoned = claim(); + clock.moveTo(NOW.plus(LEASE).plusSeconds(1)); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(abandoned, clock.instant())); + + // then + assertThat( + requeued.map(stored -> List.of(stored.id(), stored.state(), stored.nextDispatchAt()))) + .contains( + List.of( + job.id(), + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(NOW.plus(LEASE).plusSeconds(1)))); + } + + @Test + void requeueOfAJobItsHolderFinishedWritesNothing() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease lease = claim(); + clock.moveTo(NOW.plus(LEASE).plusSeconds(1)); + inTransaction( + () -> + deliveries() + .transitionHeldBy( + job.id(), RecipientDeliveryState.COMPLETED, Optional.empty(), lease)); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(lease, clock.instant())); + + // then + assertThat(List.of(requeued, deliveries().find(job.id()).orElseThrow().state())) + .containsExactly(Optional.empty(), RecipientDeliveryState.COMPLETED); + } + + @Test + void requeueOfAJobAnotherClaimTookWritesNothing() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease abandoned = claim(); + clock.moveTo(NOW.plus(LEASE).plusSeconds(1)); + inTransaction(() -> deliveries().requeueAbandoned(abandoned, clock.instant())); + claim(); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(abandoned, clock.instant())); + + // then + assertThat(List.of(requeued, deliveries().find(job.id()).orElseThrow().state())) + .as("the second recovery found the same abandoned lease; the new claim raised the fence") + .containsExactly(Optional.empty(), RecipientDeliveryState.DISPATCHING); + } + + @Test + void requeueOfAJobWhoseLeaseWasRenewedWritesNothing() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease found = claim(); + clock.moveTo(NOW.plus(LEASE).minusSeconds(1)); + inTransaction(() -> leases().renew(found, LEASE)); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(found, clock.instant())); + + // then + assertThat(List.of(requeued, deliveries().find(job.id()).orElseThrow().state())) + .containsExactly(Optional.empty(), RecipientDeliveryState.DISPATCHING); + } + private RecipientDeliveryRecord givenAQueuedJob() { NotificationRequestRecord request = NotificationRows.request(TENANT, "idem-" + UUID.randomUUID(), NOW); diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java index 0a97a7c1..4ddfb077 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java @@ -278,25 +278,77 @@ public abstract class RecipientLeaseStoreContract { } @Test - @DisplayName("lease가 기준보다 오래 전에 끝난 DISPATCHING 작업만 복구 대상으로 나온다") + @DisplayName("lease가 기준보다 오래 전에 끝난 DISPATCHING 작업만, 버려질 때의 lease로 복구 대상에 나온다") void onlyJobsWhoseLeaseEndedLongEnoughAgoAreAbandoned() { // given - RecipientDeliveryId abandoned = given(RecipientDeliveryState.READY_TO_DISPATCH, NOW, null); - claim("worker-a"); + given(RecipientDeliveryState.READY_TO_DISPATCH, NOW, null); + RecipientLease abandoned = claim("worker-a").getFirst(); clock.moveTo(NOW.plus(LEASE).plusSeconds(30)); given(RecipientDeliveryState.READY_TO_DISPATCH, NOW.plus(LEASE), null); claim("worker-b"); clock.moveTo(NOW.plus(LEASE).plus(Duration.ofMinutes(2))); // when - List expired = leases().expiredDispatching(10, Duration.ofMinutes(1)); + List expired = leases().expiredDispatching(10, Duration.ofMinutes(1)); // then assertThat(expired) - .as("a job whose lease is still running belongs to a live worker") + .as( + "a job whose lease is still running belongs to a live worker; the abandoned one comes" + + " back as the lease it was abandoned under") .containsExactly(abandoned); } + @Test + @DisplayName("claim이 쓰지 않은 DISPATCHING 작업(owner는 있고 fence가 없음)은 복구 대상에 나오지 않는다") + void aDispatchingJobNoClaimWroteIsNotOfferedForRecovery() { + // given + // The shape a row left dispatching before the fence existed has: an owner and a lease expiry + // written by hand, and the fence column's default. + NotificationRequestRecord request = + NotificationRows.request(TENANT, "idem-" + UUID.randomUUID(), clock.instant()); + RecipientDeliveryRecord job = + NotificationRows.job( + request, + RecipientDeliveryState.DISPATCHING, + Optional.of(NOW), + Optional.empty(), + clock.instant()); + RecipientDeliveryRecord leasedByHand = + new RecipientDeliveryRecord( + job.id(), + job.notificationId(), + job.tenantId(), + job.recipientRef(), + job.locale(), + job.timeZone(), + job.routingPlan(), + job.routeCursor(), + job.state(), + job.submissionOutcome(), + job.deliveryOutcome(), + job.evidenceLevel(), + job.ambiguousAttemptExists(), + job.duplicateRisk(), + job.nextDispatchAt(), + job.expiresAt(), + Optional.of("legacy-worker"), + Optional.of(NOW.minus(Duration.ofHours(1))), + job.attemptCount(), + job.lastFailureCategory(), + job.createdAt(), + job.updatedAt()); + inTransaction(() -> requests().insert(request, List.of(leasedByHand))); + + // when + List expired = leases().expiredDispatching(10, Duration.ofMinutes(1)); + + // then + assertThat(expired) + .as("a lease the application cannot build would fail every recovery pass after it") + .isEmpty(); + } + private RecipientDeliveryId given( RecipientDeliveryState state, Instant dueAt, Instant expiresAt) { NotificationRequestRecord request = diff --git a/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml b/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml index 959da41d..1aaac56e 100644 --- a/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml +++ b/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml @@ -43,8 +43,9 @@ app: default-locale: ${APP_NOTIFICATION_PLATFORM_DEFAULT_LOCALE:ko-KR} dispatch: claim-batch-size: ${APP_NOTIFICATION_PLATFORM_CLAIM_BATCH_SIZE:50} - # The lease must outlast a provider call plus its timeout, or a live worker's delivery is - # claimed by a second one while the first is still waiting on the provider. + # The lease must outlast a provider call, or a live worker's delivery is claimed by a second + # one while the first is still waiting on the provider. Startup refuses a lease that is not + # longer than every enabled provider profile's timeout. lease-duration: ${APP_NOTIFICATION_PLATFORM_LEASE_DURATION:2m} poll-interval: ${APP_NOTIFICATION_PLATFORM_POLL_INTERVAL:1s} max-global-concurrency: ${APP_NOTIFICATION_PLATFORM_MAX_CONCURRENCY:64} From 4da5143906bdea2c08bb39ce44a8738b5a87f3d0 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 13 Nov 2026 13:17:33 +0900 Subject: [PATCH 2/8] =?UTF-8?q?fix(persistence)!:=20=ED=8A=B8=EB=9E=9C?= =?UTF-8?q?=EC=9E=AD=EC=85=98=20=EC=8B=9C=ED=95=9C=EA=B3=BC=20=EC=A0=95?= =?UTF-8?q?=EB=A6=AC=20=EA=B2=BD=EB=A1=9C=EB=A5=BC=20=EB=8B=AB=EB=8A=94?= =?UTF-8?q?=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 남은 트랜잭션·정리 결함 일곱. live event log는 지우는 곳이 없어 끝없이 자란다. app.lock.wait-time은 읽는 곳이 없다. 알림 scheduler는 poll마다 상태 쿼리 넷을 돌리고, projection 지연은 닫힌 event까지 읽는다. stateless runner 트랜잭션에 시한이 없다. tenant pool 예산은 새 pool의 크기를 모른 채 열어 연결 합계가 상한을 넘는다. runbook SQL 설명이 lane 넷 중 셋만 말한다. 고른 방법: LiveEventReaper(1000행씩, 트랜잭션 30초)와 공용 BatchedDeletes를 더한다. app.realtime.replay.*는 LiveEventReplaySettings가 엄격하게 묶고, 조립 설정이 SchedulingConfigurer로 reaper를 reaper-interval(기본 10분)마다 등록한다. LockSettings는 lease-ttl 하나만 받는다(모르는 키로 기동 실패). scheduler는 backlogDepth() 한 문장만 읽고, 그 WHERE는 claim과 같은 SQL 조각(RecipientClaimSql.CLAIMABLE_AT_NOW)이다. projection 지연은 V14 부분 인덱스로 읽는다(lock_timeout, V21 선례대로 CONCURRENTLY 아님). runner 기본 30초(SpringTransaction 쓰기 시한과 같음을 시험이 본다), tenant 예산은 계획된 pool 크기로 열기 전에 본다. 버린 대안: V14를 CONCURRENTLY로(Flyway 트랜잭션 안에서 못 돈다, 큰 DB는 안내대로 미리 만든다), reaper @Scheduled에 SpEL로 settings bean 이름 읽기(문자열 bean 조회는 검사가 못 본다). 위험·한계: LockSettings·TenantDataSourceRegistry·TenantPoolBudget의 공개 생성자가 바뀐다. V14는 큰 notification_provider_event에서 쓰기를 잠깐 막는다(안내 4행에 미리 만드는 법과 INVALID 인덱스 확인). backlog 수치는 창이 닫힌 delivery를 더는 세지 않는다. 검증: desktop RED(pr1-red3) 새 시험 실패 포함. GREEN은 PR 본문. Refs: RR9-01-03, RR9-01-07, RR9-01-09, RR9-01-10, RR9-01-14, RR9-01-25, RR9-01-26 BREAKING CHANGE: JpaLiveEventReplay.sweepExpired 삭제, app.realtime.replay 아래 모르는 키와 app.lock.wait-time은 기동 실패, HibernateStatelessSessionRunner 트랜잭션 30초 상한, TenantPoolBudget·TenantDataSourceRegistry 시그니처 변경. docs/upgrade/2026-11-round9-claim-fence.md --- docs/architecture/jpa-api-surface.txt | 4 +- docs/upgrade/2026-11-round9-claim-fence.md | 31 ++- .../dispatch/NotificationSchedulerWorker.java | 6 +- .../outbound/persistence-jpa/README.md | 16 ++ .../autoconfigure/JpaAdapterBeansConfig.java | 43 ++++- .../database/TenantDataSourceRegistry.java | 23 ++- .../database/TenantPoolBudget.java | 39 +++- .../HibernateStatelessSessionRunner.java | 47 ++++- .../liveevent/JpaLiveEventReplay.java | 10 - .../liveevent/LiveEventJpaRepository.java | 23 ++- .../liveevent/LiveEventReaper.java | 53 ++++++ .../liveevent/LiveEventReplaySettings.java | 55 ++++++ .../persistence/lock/LockSettings.java | 31 +-- .../module-defaults/persistence-jpa.yml | 10 +- .../JpaLiveEventReplayContractTest.java | 41 ++++ .../notification/NotificationFixtures.java | 3 +- .../ServingStateContractTest.java | 177 +++++++++++++++++- ...StatelessSessionRunnerIntegrationTest.java | 51 +++++ .../TenantPoolCapacityContractTest.java | 68 ++++--- .../PostgreSqlOptionalStreamLifecycle.java | 4 +- .../persistence/readiness/RunbookSql.java | 10 +- .../JpaLiveEventReplayActivationTest.java | 123 +++++++++++- .../ExperimentalFailClosedTest.java | 21 ++- .../stateless/StatelessRunnerTimeoutTest.java | 45 +++++ .../liveevent/LiveEventReaperTest.java | 53 ++++++ .../DistributedLockPersistenceConfigTest.java | 14 ++ .../resources/db/applied-migrations.sha256 | 1 + .../JdbcNotificationServingState.java | 88 ++++++--- ...notification_provider_event_open_index.sql | 26 +++ ...otificationServingStateOpenEventsTest.java | 42 +++++ .../observation/NotificationServingState.java | 2 +- .../NotificationServingStatePort.java | 18 ++ 32 files changed, 1038 insertions(+), 140 deletions(-) create mode 100644 src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaper.java create mode 100644 src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReplaySettings.java create mode 100644 src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java create mode 100644 src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java create mode 100644 src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql create mode 100644 src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java diff --git a/docs/architecture/jpa-api-surface.txt b/docs/architecture/jpa-api-surface.txt index 0063f347..06fc95b3 100644 --- a/docs/architecture/jpa-api-surface.txt +++ b/docs/architecture/jpa-api-surface.txt @@ -5,7 +5,7 @@ # root yet. # Update only after review with: # ./gradlew :adapter:outbound:persistence-jpa:updateJpaApiSurface -PapproveJpaApiSurfaceChange -# types: 166 +# types: 168 dev.caskeleton.adapter.outbound.persistence.api.PersistenceOperationName dev.caskeleton.adapter.outbound.persistence.api.capability.CapabilitySupport dev.caskeleton.adapter.outbound.persistence.api.capability.JpaCapability @@ -104,6 +104,8 @@ dev.caskeleton.adapter.outbound.persistence.idempotency.postgresql.IdempotencySc dev.caskeleton.adapter.outbound.persistence.idempotency.postgresql.PostgreSqlIdempotencyStore dev.caskeleton.adapter.outbound.persistence.liveevent.JpaLiveEventReplay dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventJpaRepository +dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReaper +dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReplaySettings dev.caskeleton.adapter.outbound.persistence.liveevent.entity.LiveEventEntity dev.caskeleton.adapter.outbound.persistence.lock.DistributedLockPersistenceConfig dev.caskeleton.adapter.outbound.persistence.lock.LockRegistryDistributedLock diff --git a/docs/upgrade/2026-11-round9-claim-fence.md b/docs/upgrade/2026-11-round9-claim-fence.md index 99ae53f9..940011b1 100644 --- a/docs/upgrade/2026-11-round9-claim-fence.md +++ b/docs/upgrade/2026-11-round9-claim-fence.md @@ -1,13 +1,14 @@ # 업그레이드 안내 — 2026-11 claim·fence·트랜잭션 시한·정리 -- 범위: durable operation·알림 수신자·fileserver cleanup·멱등성 기록의 lease, fence, 조건부 쓰기, 정리를 고친 - 변경. 포트 시그니처와 기동 조건이 바뀐 곳이 있다. 아래 표가 그 목록이다. -- 근거: 2026-11-12 전체 리뷰(`docs/reviews/2026-11-12-round9-review/`)의 claim·fence(B8) 발견 +- 범위: durable operation·알림 수신자·fileserver cleanup·멱등성 기록·live event·분산 락·stateless runner의 lease, + fence, 조건부 쓰기, 정리, 시간 상한을 고친 변경. 포트 시그니처와 기동 조건이 바뀐 곳이 있다. 아래 표가 그 목록이다. +- 근거: 2026-11-12 전체 리뷰(`docs/reviews/2026-11-12-round9-review/`)의 claim·fence(B8)와 트랜잭션 시한·정리(B9) 발견 ## 1. 누가 읽나 이 템플릿을 `upstream` remote로 두고 merge해 받는 파생 서비스가 읽는다. 특히 durable operation worker를 직접 쓴 서비스, -알림 recipient store를 직접 구현한 서비스, 알림 provider timeout을 설정한 서비스다. +알림 recipient store를 직접 구현한 서비스, `app.lock.wait-time`이나 알림 provider timeout을 설정한 서비스, experimental +tenant pool 예산을 쓰는 서비스다. ## 2. 할 일 @@ -16,17 +17,22 @@ | 1 | durable operation worker는 `claimNext`가 돌려주는 `DurableOperationClaim`을 들고 있다가 `heartbeat(claim, …)`·`reportProgress(claim, …)`·`succeed(claim, …)`·`fail(claim, …)`에 넘긴다. 예전에는 `(operationId, workerId, …)`였다 | 컴파일이 실패한다 | durable operation worker를 쓴 서비스 | | 2 | `DurableOperationStorePort`를 직접 구현했으면 claim마다 오르는 fence를 저장하고, 네 쓰기가 owner와 fence를 함께 확인하게 한다. `DurableOperationStoreContract`의 새 사례(같은 worker id가 다시 claim한 뒤 앞 실행의 쓰기는 거절)로 확인한다 | 재시작한 worker나 한 프로세스의 두 실행이 같은 id를 쓰면 앞 실행의 결과가 넘겨받은 실행 위에 기록된다 | 자기 durable operation store를 둔 서비스 | | 3 | persistence-jpa의 `V23__durable_operation_lease_fence.sql`(`durable_operation.lease_fence BIGINT NOT NULL DEFAULT 0`)이 기본 stream에 더해진다. 같은 번호의 서비스 마이그레이션이 있으면 번호를 옮긴다. `durable_operation`을 건드린 트랜잭션이 열려 있으면 5초 뒤 실패하고 V22에 남는다(다시 배포) | Flyway가 번호 충돌로 기동하지 않는다 | persistence-jpa를 쓰는 모든 서비스 | -| 4 | `RecipientLeaseStorePort.expiredDispatching`이 `List`(버려질 때의 owner·fence·만료 시각)를 돌려주고, `RecipientDeliveryStorePort`에 `requeueAbandoned(lease, nextDispatchAt)`가 생겼다. 직접 구현했으면 상태가 `DISPATCHING`이고 owner·fence가 같고 lease가 늘지 않았을 때만 `READY_TO_DISPATCH`로 바꾼다(`RecipientDeliveryStoreContract` 새 사례). owner가 없거나 fence가 0인 `DISPATCHING` 행(notification V4 전에 남은 행)은 복구가 더는 고르지 않는다. 그런 행은 `SELECT id FROM notification_recipient_delivery WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 찾는다. 배포 전에 `UPDATE notification_recipient_delivery SET lease_owner = COALESCE(lease_owner, 'pre-v4'), lease_fence = lease_fence + 1, lease_until = now() - interval '1 day', version = version + 1, updated_at = now() WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 owner를 채우고 fence를 올려 복구 패스에 넘긴다. 복구는 attempt를 보고 정한다: attempt가 없으면 다시 보낼 차례로 돌리고, 끝나지 않은 attempt가 있으면 맞추기로 넘긴다(그 뒤는 `docs/notification/configuration-reference.md` 「발송」의 lease 문단). 손으로 `READY_TO_DISPATCH`로 바꾸지 않는다. 발송 중에 멈춘 행이면 같은 알림이 한 번 더 나간다. owner가 없는 행은 stuck lease 지표에도 잡히지 않는다 | 컴파일이 실패한다. 조건 없이 바꾸면 끝났거나 다시 claim된 delivery가 다시 발송된다 | 알림 store를 직접 구현한 서비스 | -| 5 | 알림 provider profile의 `timeout`이 `app.notification.platform.dispatch.lease-duration`보다 짧은지 확인한다. 짧은 것은 필요조건일 뿐이다: lease는 claim 때 시작해서 attempt 기록·렌더링·provider 허가 대기도 lease를 쓴다. lease는 timeout보다 넉넉히 길게 둔다. 이 timeout이 provider 호출 전체를 묶는 것은 SES(API 호출 시간 상한)다. SMTP는 `spring.mail` socket timeout이 명령마다 묶을 뿐이라 이 검사만으로는 호출 전체가 lease 안에 끝난다고 말할 수 없다 | 짧지 않으면 기동이 실패한다(`notification provider profile '…': timeout (…) must be shorter than …lease-duration`). 전에는 provider 호출 하나가 lease보다 길어도 기동했다. 그때 생기는 일은 `docs/notification/configuration-reference.md` 「발송」의 lease 문단이다 | 알림 capability를 켠 서비스 | -| 6 | `IdempotencyReaper`를 직접 만들던 코드는 `new IdempotencyReaper(repository)`로 바꾼다(`Clock` 인자 삭제). 만료 비교는 DB 시계(`now()`)이고 1000행씩 지운다 | 컴파일이 실패한다 | reaper를 직접 조립한 서비스 | -| 7 | fileserver cleanup 묶음의 `max-items`를 정할 때, 한 묶음이 claim lease(10분)의 첫 8분 안에 시작할 수 있는 만큼만 처리된다는 것을 안다. 나머지는 시도하지 않은 채 queue에 돌아가 다음 묶음이 다시 claim한다. 시도 횟수는 그대로다(바이트 예산이 바닥나 돌려준 항목도 이제 같다. 전에는 시도 한 번으로 세어 8번째에 보류됐다). 직접 구현한 `CleanupQueuePort`는 새 연산 `release`를 구현한다(`CleanupQueuePortContract` 새 사례) | 느린 저장소에서 큰 `max-items`는 그 사유로 도는 항목이 늘어난다. 전에는 lease가 끝난 뒤에도 남은 항목을 처리해 다른 worker와 겹쳤다 | fileserver capability를 켠 서비스 | +| 4 | 알림 stream에 `V14__notification_provider_event_open_index.sql`(부분 인덱스 `ix_notification_provider_event_open`)이 더해진다. `notification_provider_event`가 이미 큰 운영 DB는 배포 전에 같은 인덱스를 `CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_notification_provider_event_open ON notification_provider_event (received_at) WHERE projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL;`로 먼저 만든다. 미리 만든 인덱스가 실패해 남았으면(`SELECT indisvalid FROM pg_index WHERE indexrelid = 'ix_notification_provider_event_open'::regclass`가 false) `DROP INDEX CONCURRENTLY ix_notification_provider_event_open` 뒤 다시 만든다. `IF NOT EXISTS`는 쓸 수 없는 인덱스도 있는 것으로 보고 넘어간다 | 마이그레이션이 인덱스를 만드는 동안 provider callback 저장이 기다린다(provider가 callback을 다시 보낸다) | 알림 capability를 켠 서비스 | +| 5 | `RecipientLeaseStorePort.expiredDispatching`이 `List`(버려질 때의 owner·fence·만료 시각)를 돌려주고, `RecipientDeliveryStorePort`에 `requeueAbandoned(lease, nextDispatchAt)`가 생겼다. 직접 구현했으면 상태가 `DISPATCHING`이고 owner·fence가 같고 lease가 늘지 않았을 때만 `READY_TO_DISPATCH`로 바꾼다(`RecipientDeliveryStoreContract` 새 사례). owner가 없거나 fence가 0인 `DISPATCHING` 행(notification V4 전에 남은 행)은 복구가 더는 고르지 않는다. 그런 행은 `SELECT id FROM notification_recipient_delivery WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 찾는다. 배포 전에 `UPDATE notification_recipient_delivery SET lease_owner = COALESCE(lease_owner, 'pre-v4'), lease_fence = lease_fence + 1, lease_until = now() - interval '1 day', version = version + 1, updated_at = now() WHERE delivery_state = 'DISPATCHING' AND (lease_owner IS NULL OR lease_fence = 0)`로 owner를 채우고 fence를 올려 복구 패스에 넘긴다. 복구는 attempt를 보고 정한다: attempt가 없으면 다시 보낼 차례로 돌리고, 끝나지 않은 attempt가 있으면 맞추기로 넘긴다(그 뒤는 `docs/notification/configuration-reference.md` 「발송」의 lease 문단). 손으로 `READY_TO_DISPATCH`로 바꾸지 않는다. 발송 중에 멈춘 행이면 같은 알림이 한 번 더 나간다. owner가 없는 행은 stuck lease 지표에도 잡히지 않는다 | 컴파일이 실패한다. 조건 없이 바꾸면 끝났거나 다시 claim된 delivery가 다시 발송된다 | 알림 store를 직접 구현한 서비스 | +| 6 | 알림 provider profile의 `timeout`이 `app.notification.platform.dispatch.lease-duration`보다 짧은지 확인한다. 짧은 것은 필요조건일 뿐이다: lease는 claim 때 시작해서 attempt 기록·렌더링·provider 허가 대기도 lease를 쓴다. lease는 timeout보다 넉넉히 길게 둔다. 이 timeout이 provider 호출 전체를 묶는 것은 SES(API 호출 시간 상한)다. SMTP는 `spring.mail` socket timeout이 명령마다 묶을 뿐이라 이 검사만으로는 호출 전체가 lease 안에 끝난다고 말할 수 없다 | 짧지 않으면 기동이 실패한다(`notification provider profile '…': timeout (…) must be shorter than …lease-duration`). 전에는 provider 호출 하나가 lease보다 길어도 기동했다. 그때 생기는 일은 `docs/notification/configuration-reference.md` 「발송」의 lease 문단이다 | 알림 capability를 켠 서비스 | +| 7 | `app.lock.wait-time`을 설정에서 지운다. 기다릴 시간은 `tryAcquire(key, waitTime, leaseTtl)`의 인자다 | `app.lock` 아래 모르는 키라 기동이 실패한다. 전에도 읽는 곳이 없었다 | `app.lock.wait-time`을 설정한 서비스 | +| 8 | `JpaLiveEventReplay.sweepExpired()`를 부르던 코드를 지운다. replay를 켜면 `LiveEventReaper`가 `app.realtime.replay.reaper-interval`(기본 `PT10M`)마다 `retained_until`이 지난 행을 1000행씩 지운다. `app.realtime.replay.*`는 이제 `LiveEventReplaySettings`가 엄격하게 묶는다: 그 아래 모르는 키, 0 이하의 `retention`·`reaper-interval`은 기동을 실패시킨다 | 컴파일이 실패한다. 잘못 쓴 키는 기동이 실패한다. 전에는 지우는 곳이 없어 `live_event_log`가 끝없이 자랐다 | `app.realtime.replay.enabled=true`인 서비스 | +| 9 | `IdempotencyReaper`를 직접 만들던 코드는 `new IdempotencyReaper(repository)`로 바꾼다(`Clock` 인자 삭제). 만료 비교는 DB 시계(`now()`)이고 1000행씩 지운다 | 컴파일이 실패한다 | reaper를 직접 조립한 서비스 | +| 10 | `HibernateStatelessSessionRunner`의 트랜잭션은 이제 30초 상한이 있다(`DEFAULT_TIMEOUT`). 더 긴 일이면 셋째 인자로 상한을 넘기거나 일을 나눈다 | 30초를 넘는 문장이 취소되고 일 전체가 롤백된다. 전에는 상한이 없었다 | stateless runner를 쓰는 서비스 | +| 11 | experimental `TenantPoolBudget.requireCapacity`·`hasCapacity`가 새 pool의 연결 수를 셋째 인자로 받는다. pool 수만 볼 때는 `requirePoolSlot(openPools)`를 쓴다. `TenantDataSourceRegistry`의 둘째 인자는 이제 열린 pool의 크기를 재는 함수가 아니라 tenant의 계획된 pool 크기(`ToIntFunction`)다. 예산은 pool을 열기 전에 그 크기로 본다 | 컴파일이 실패한다. 전에는 새 pool 크기를 모른 채 열어 연결 합계가 상한을 넘었다 | experimental tenant pool을 쓰는 서비스 | +| 12 | fileserver cleanup 묶음의 `max-items`를 정할 때, 한 묶음이 claim lease(10분)의 첫 8분 안에 시작할 수 있는 만큼만 처리된다는 것을 안다. 나머지는 시도하지 않은 채 queue에 돌아가 다음 묶음이 다시 claim한다. 시도 횟수는 그대로다(바이트 예산이 바닥나 돌려준 항목도 이제 같다. 전에는 시도 한 번으로 세어 8번째에 보류됐다). 직접 구현한 `CleanupQueuePort`는 새 연산 `release`를 구현한다(`CleanupQueuePortContract` 새 사례) | 느린 저장소에서 큰 `max-items`는 그 사유로 도는 항목이 늘어난다. 전에는 lease가 끝난 뒤에도 남은 항목을 처리해 다른 worker와 겹쳤다 | fileserver capability를 켠 서비스 | ## 3. 바뀐 동작 (따라 할 일 없음) -- reaper 둘(outbox·멱등성)의 트랜잭션은 `SpringTransaction.DEFAULT_TIMEOUT_SECONDS`(30초)로 묶인다. +- reaper 셋(outbox·멱등성·live event)의 트랜잭션은 `SpringTransaction.DEFAULT_TIMEOUT_SECONDS`(30초)로 묶인다. `app.persistence-jpa.transaction.*`를 바꿔도 reaper 시한은 그대로다. 한 트랜잭션이 1000행 한 묶음만 지운다. - 저장소가 행에서 만드는 `DurableOperationClaim`·`OperationLease`·`RecipientLease`·`OutboxLease`·`ClaimedOutboxEvent`· - `ReconciliationJob`·`AdminOperationClaim`이 값을 거절할 때 `IllegalStateException`을 던진다 + `ReconciliationJob`·`AdminOperationClaim`과 stateless runner의 timeout이 값을 거절할 때 `IllegalStateException`을 던진다 (전에는 `IllegalArgumentException`). 요청에서 오는 값이 아니라 서버 결함이다. 이 IAE를 잡던 코드는 ISE를 잡는다. - 알림 lease 회복은 버려질 때의 lease 그대로일 때만 delivery를 다시 넣는다. 다른 인스턴스가 이미 다시 넣었거나 holder가 끝냈으면 아무것도 쓰지 않고 처리 수에 세지 않는다. @@ -36,3 +42,8 @@ - 멱등성 reaper는 다른 트랜잭션이 쥔 기록을 건너뛰고, 바깥 `DELETE`에서도 만료를 다시 본다. 같은 키 요청이 만료된 기록을 되살리는 동안 돈 reap이 그 claim을 지우지 않는다. - fileserver recovery 대기열 enqueue는 `INSERT … ON CONFLICT` 한 문장이다. 같은 파일을 동시에 넣어도 25P02로 실패하지 않는다. +- 알림 scheduler는 poll마다 backlog 한 문장만 읽는다(전에는 넷). health snapshot의 projection 지연은 열린 provider event만 + 읽는다. +- 알림 backlog(`backlogDepth`, scheduler의 queue-depth gauge, health 임계값)는 claim이 지금 가져갈 delivery만 센다. claim과 + 같은 SQL 조각이다. 전에는 창이 닫힌(`expires_at`이 지난) delivery도 셌고, lease가 지금 막 끝나는 행(`lease_until = now`)도 + 셌다. 만료가 쌓인 플랫폼은 배포 직후 backlog 수치가 내려간다. 보낼 일이 줄어든 것이 아니라 보낼 수 없는 행을 빼고 센 것이다. diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java index d761c923..f54fa001 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java @@ -81,10 +81,10 @@ public final class NotificationSchedulerWorker implements AutoCloseable { // by the claim batch size — so a queue of ten and a queue of ten million published the same // number, and the one metric named "queue depth" was the one that could not show a queue // growing. + // The depth alone: the full serving snapshot is several statements and is the health + // endpoint's, not something to run on every poll. metrics.gauge( - NotificationMetricName.QUEUE_DEPTH, - Map.of(), - (double) servingState.currentState().backlogDepth()); + NotificationMetricName.QUEUE_DEPTH, Map.of(), (double) servingState.backlogDepth()); for (RecipientLease lease : claimed) { globalConcurrency.acquireUninterruptibly(); diff --git a/src/adapter/outbound/persistence-jpa/README.md b/src/adapter/outbound/persistence-jpa/README.md index 9269264e..299beed7 100644 --- a/src/adapter/outbound/persistence-jpa/README.md +++ b/src/adapter/outbound/persistence-jpa/README.md @@ -389,6 +389,22 @@ PUBLISHED backlog 를 한 transaction 으로 지우면 그 사이 모든 row loc cadence 측정 필요). retention 한 값은 reaper-local 이라 `@Value` 로 받지만, canonical 6-property 문서는 app-bootstrap `OutboxSettings` / `application.yml` 에 있다. +## liveevent — `JpaLiveEventReplay`와 `LiveEventReaper` + +`app.realtime.replay.enabled=true`이고 vendor가 PostgreSQL이면 replay store와 reaper가 함께 조립된다. 설정은 +`LiveEventReplaySettings`가 `app.realtime.replay.*`를 엄격하게 묶는다. 모르는 키는 기동을 실패시킨다. + +- `retention`: 한 event를 다시 받을 수 있는 시간. 기본값이 없다. 모듈 기본값이 `APP_REALTIME_REPLAY_RETENTION`(1시간)으로 + 채운다. 0이나 음수는 기동 실패다. +- `reaper-interval`: `LiveEventReaper`가 retention이 지난 행을 지우는 간격(기본 10분, 모듈 기본값이 같은 값을 적는다). 0이나 + 음수는 기동 실패다. reaper 자체에는 일정이 없다. 조립 설정이 이 값으로 `reap()`을 등록한다(첫 실행도 한 간격 뒤). + reaper를 직접 만드는 서비스는 일정도 직접 건다. + +reaper는 `BatchedDeletes`로 1,000행씩, 배치마다 따로 commit하고, 배치가 덜 차면 멈춘다. 배치 문장은 `SKIP LOCKED`라 여러 +replica가 같이 돌아도 서로 기다리지 않는다. 트랜잭션 시한은 `SpringTransaction.DEFAULT_TIMEOUT_SECONDS`(30초)다. replay 읽기는 +`retained_until`로 지난 행을 이미 거르므로, reaper가 늦어도 지난 event를 내주지는 않는다. reaper가 없으면 테이블이 끝없이 +자란다(2026-11-13 전에는 지우는 곳이 없었다). + ## operation — `JpaDurableOperationStore` ### claim과 fence (`V23`) diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java index 193187e5..438f0978 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaAdapterBeansConfig.java @@ -8,6 +8,8 @@ import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.IdempotencyRe import dev.caskeleton.adapter.outbound.persistence.idempotency.jpa.IdempotencyRecordJpaRepository; import dev.caskeleton.adapter.outbound.persistence.liveevent.JpaLiveEventReplay; import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventJpaRepository; +import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReaper; +import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReplaySettings; import dev.caskeleton.adapter.outbound.persistence.lock.DistributedLockPersistenceConfig; import dev.caskeleton.adapter.outbound.persistence.operation.DurableOperationJpaRepository; import dev.caskeleton.adapter.outbound.persistence.operation.JpaDurableOperationStore; @@ -26,7 +28,11 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.core.env.Environment; import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.SchedulingConfigurer; +import org.springframework.scheduling.config.FixedDelayTask; import org.springframework.transaction.PlatformTransactionManager; /** @@ -120,11 +126,15 @@ class JpaAdapterBeansConfig { } /** - * Retained live-event history: on when replay is switched on, and only on PostgreSQL (the - * position allocator is an {@code INSERT … ON CONFLICT … RETURNING}). The retention has no - * default on purpose: a replay window nobody chose is one nobody sized. + * Retained live-event history and the reaper that removes it past retention: on when replay is + * switched on, and only on PostgreSQL (the position allocator is an {@code INSERT … ON CONFLICT … + * RETURNING}). The settings are {@link LiveEventReplaySettings}, bound strictly: the retention is + * one hour unless the deployment sets {@code APP_REALTIME_REPLAY_RETENTION} (the module + * defaults), the reaper runs every {@code reaper-interval}. Scheduling is enabled here because + * the reaper is part of the store, as for the idempotency reaper. */ @Configuration(proxyBeanMethods = false) + @EnableScheduling @ConditionalOnProperty(prefix = "app.realtime.replay", name = "enabled", havingValue = "true") @ConditionalOnProperty( prefix = "app.persistence", @@ -133,13 +143,36 @@ class JpaAdapterBeansConfig { matchIfMissing = true) static class LiveEventReplay { + @Bean + LiveEventReplaySettings liveEventReplaySettings(Environment environment) { + return LiveEventReplaySettings.bind(environment); + } + @Bean JpaLiveEventReplay jpaLiveEventReplay( LiveEventJpaRepository repository, JdbcOperations jdbc, Clock clock, - @Value("${app.realtime.replay.retention}") Duration retention) { - return new JpaLiveEventReplay(repository, jdbc, clock, retention); + LiveEventReplaySettings settings) { + return new JpaLiveEventReplay(repository, jdbc, clock, settings.retention()); + } + + @Bean + LiveEventReaper liveEventReaper(LiveEventJpaRepository repository, Clock clock) { + return new LiveEventReaper(repository, clock); + } + + /** + * Runs the reaper every bound {@code reaper-interval}, first one interval after startup. The + * interval is handed over as the settings' value, so it has one default and a zero has already + * failed startup in the settings rather than scheduling a loop. + */ + @Bean + SchedulingConfigurer liveEventReaperSchedule( + LiveEventReaper reaper, LiveEventReplaySettings settings) { + Duration interval = settings.reaperInterval(); + return registrar -> + registrar.addFixedDelayTask(new FixedDelayTask(reaper::reap, interval, interval)); } } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantDataSourceRegistry.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantDataSourceRegistry.java index 90737a29..2551c93c 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantDataSourceRegistry.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantDataSourceRegistry.java @@ -23,26 +23,32 @@ import javax.sql.DataSource; public final class TenantDataSourceRegistry implements AutoCloseable { private final Map pools = new LinkedHashMap<>(); + private final Map poolSizes = new LinkedHashMap<>(); private final Function poolFactory; - private final ToIntFunction poolSize; + private final ToIntFunction plannedPoolSize; private final TenantPoolBudget budget; /** * @param poolFactory opens a pool for a tenant from its secret-backed connection profile - * @param poolSize reports a pool's maximum connection count, for the global budget + * @param plannedPoolSize the maximum connection count a tenant's pool will have, read from its + * connection profile before the pool is opened, for the global budget */ public TenantDataSourceRegistry( Function poolFactory, - ToIntFunction poolSize, + ToIntFunction plannedPoolSize, TenantPoolBudget budget) { this.poolFactory = Objects.requireNonNull(poolFactory, "poolFactory"); - this.poolSize = Objects.requireNonNull(poolSize, "poolSize"); + this.plannedPoolSize = Objects.requireNonNull(plannedPoolSize, "plannedPoolSize"); this.budget = Objects.requireNonNull(budget, "budget"); } /** * The tenant's pool, opening it if the budget allows. * + *

The budget is checked against the pool's planned size before the pool is opened: a pool + * opens its first connections as it is built, so one checked afterwards had already taken the + * server past the ceiling, if only until it was closed again. + * * @throws IllegalStateException when the global pool budget is exhausted */ public synchronized DataSource require(TenantId tenant) { @@ -51,9 +57,12 @@ public final class TenantDataSourceRegistry implements AutoCloseable { if (existing != null) { return existing; } - budget.requireCapacity(pools.size(), allocatedConnections()); + budget.requirePoolSlot(pools.size()); + int planned = plannedPoolSize.applyAsInt(tenant); + budget.requireCapacity(pools.size(), allocatedConnections(), planned); DataSource opened = poolFactory.apply(tenant); pools.put(tenant, opened); + poolSizes.put(tenant, planned); return opened; } @@ -72,6 +81,7 @@ public final class TenantDataSourceRegistry implements AutoCloseable { */ public synchronized void evict(TenantId tenant) { DataSource removed = pools.remove(tenant); + poolSizes.remove(tenant); if (removed instanceof AutoCloseable closeable) { TenantDataSourceLifecycle.drainAndClose(closeable); } @@ -84,7 +94,7 @@ public final class TenantDataSourceRegistry implements AutoCloseable { /** How many connections all open pools may hold between them. */ public synchronized int allocatedConnections() { - return pools.values().stream().mapToInt(poolSize).sum(); + return poolSizes.values().stream().mapToInt(Integer::intValue).sum(); } /** The tenants with an open pool. */ @@ -99,5 +109,6 @@ public final class TenantDataSourceRegistry implements AutoCloseable { .map(AutoCloseable.class::cast) .forEach(TenantDataSourceLifecycle::drainAndClose); pools.clear(); + poolSizes.clear(); } } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantPoolBudget.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantPoolBudget.java index b3abfb1d..c26fa39d 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantPoolBudget.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/experimental/database/TenantPoolBudget.java @@ -23,19 +23,44 @@ public record TenantPoolBudget(int maxOpenPools, int maxConnectionsAcrossPools) } /** - * Fails when another tenant pool would exceed either ceiling. + * Fails when another tenant pool would exceed the pool count, before that pool is opened. * - * @throws IllegalStateException when the budget is exhausted + * @param openPools how many pools are open now + * @throws IllegalStateException when no other pool fits */ - public void requireCapacity(int openPools, int allocatedConnections) { - if (openPools >= maxOpenPools || allocatedConnections >= maxConnectionsAcrossPools) { + public void requirePoolSlot(int openPools) { + if (openPools >= maxOpenPools) { throw new IllegalStateException("tenant pool budget exhausted"); } } - /** Whether another tenant pool fits inside both ceilings. */ - public boolean hasCapacity(int openPools, int allocatedConnections) { - return openPools < maxOpenPools && allocatedConnections < maxConnectionsAcrossPools; + /** + * Fails when a new tenant pool of {@code newPoolConnections} would exceed either ceiling. + * + *

The new pool's own size is part of the question. Checking only that the pools already open + * stay under the ceiling let a ten-connection pool open with one connection to spare, and the + * total then stood nine above the ceiling the budget exists to keep (round9 review 2026-11-12, + * RR9-01-26). + * + * @param openPools how many pools are open now, the new one not included + * @param allocatedConnections how many connections the open pools may hold between them + * @param newPoolConnections how many the new pool may hold + * @throws IllegalStateException when the budget cannot take the new pool + */ + public void requireCapacity(int openPools, int allocatedConnections, int newPoolConnections) { + if (!hasCapacity(openPools, allocatedConnections, newPoolConnections)) { + throw new IllegalStateException("tenant pool budget exhausted"); + } + } + + /** Whether a new tenant pool of {@code newPoolConnections} fits inside both ceilings. */ + public boolean hasCapacity(int openPools, int allocatedConnections, int newPoolConnections) { + if (newPoolConnections < 1) { + // The size is the server's own configuration, not the caller's input: a state error. + throw new IllegalStateException("a tenant pool holds at least one connection"); + } + return openPools < maxOpenPools + && (long) allocatedConnections + newPoolConnections <= maxConnectionsAcrossPools; } /** The maximum number of tenants that can be open at once. */ diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/HibernateStatelessSessionRunner.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/HibernateStatelessSessionRunner.java index 00f8aa30..d80927af 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/HibernateStatelessSessionRunner.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/HibernateStatelessSessionRunner.java @@ -1,5 +1,6 @@ package dev.caskeleton.adapter.outbound.persistence.hibernate.stateless; +import java.time.Duration; import java.util.Objects; import java.util.Set; import java.util.function.Function; @@ -19,23 +20,63 @@ import org.hibernate.Transaction; * *

Only registered work names may run, and each carries a declared row cap. An admin bulk path * without a cap is how a maintenance job becomes an outage. + * + *

The transaction has the same time bound as the application's write transactions (30 seconds + * unless the constructor is given another, {@code docs/work/reference/decisions.md} 2.5). The row + * cap can only be checked once the work reports what it touched, so without a bound a runaway + * statement held its locks for as long as it ran before the cap could roll it back. Hibernate hands + * every statement of the work the time left, and the database cancels the one that runs past it + * (round9 review 2026-11-12, RR9-01-25). */ public final class HibernateStatelessSessionRunner { + /** + * The default write-transaction bound, which this runner's transactions share: {@code + * SpringTransaction.DEFAULT_TIMEOUT_SECONDS}. Restated rather than read from it, because the + * {@code hibernate} package does not depend on {@code transaction} (JpaModuleBoundaryTest); + * {@code StatelessRunnerTimeoutTest} holds the two equal. A deployment that moves its write bound + * passes the same value to the three-argument constructor. + */ + public static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30); + private final SessionFactory sessionFactory; private final Set registeredWork; + private final int timeoutSeconds; + /** A runner whose transactions are bounded by {@link #DEFAULT_TIMEOUT}. */ public HibernateStatelessSessionRunner( SessionFactory sessionFactory, Set registeredWork) { + this(sessionFactory, registeredWork, DEFAULT_TIMEOUT); + } + + /** + * A runner whose transactions are bounded by {@code timeout}. + * + * @param timeout a whole, positive number of seconds; Hibernate's transaction timeout has no + * finer unit, so a fraction would be dropped without a word + */ + public HibernateStatelessSessionRunner( + SessionFactory sessionFactory, Set registeredWork, Duration timeout) { this.sessionFactory = Objects.requireNonNull(sessionFactory, "sessionFactory"); this.registeredWork = Set.copyOf(Objects.requireNonNull(registeredWork, "registeredWork")); + Objects.requireNonNull(timeout, "timeout"); + if (timeout.isNegative() + || timeout.isZero() + || timeout.toNanosPart() != 0 + || timeout.toSeconds() > Integer.MAX_VALUE) { + throw new IllegalStateException( + "a stateless transaction timeout must be a positive whole number of seconds, was " + + timeout); + } + this.timeoutSeconds = Math.toIntExact(timeout.toSeconds()); } /** * Runs {@code work} in its own stateless session and transaction. * *

The work reports how many rows it touched, and exceeding {@code maxRows} rolls the - * transaction back. + * transaction back. A statement still running when the transaction's time is up is cancelled, and + * the failure rolls the transaction back. * * @param maxRows the row cap for this unit of work * @throws StatelessRowCapExceededException when the work touched more, after rolling back @@ -54,7 +95,9 @@ public final class HibernateStatelessSessionRunner { } try (StatelessSession session = sessionFactory.openStatelessSession()) { - Transaction transaction = session.beginTransaction(); + Transaction transaction = session.getTransaction(); + transaction.setTimeout(timeoutSeconds); + transaction.begin(); try { StatelessWorkResult result = work.apply(session); if (result.affectedRows() > maxRows) { diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplay.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplay.java index 26e345af..d2762a63 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplay.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplay.java @@ -134,16 +134,6 @@ public final class JpaLiveEventReplay implements LiveEventReplayPort { .toList(); } - /** - * Removes events past their retention. - * - * @return how many went - */ - public int sweepExpired() { - requireCallerTransaction("sweepExpired"); - return repository.sweepExpired(clock.instant()); - } - private static void requireCallerTransaction(String operation) { CallerTransaction.require("live-event " + operation); } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventJpaRepository.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventJpaRepository.java index cf9ea4ce..9df87a3f 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventJpaRepository.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventJpaRepository.java @@ -1,6 +1,7 @@ package dev.caskeleton.adapter.outbound.persistence.liveevent; import dev.caskeleton.adapter.outbound.persistence.liveevent.entity.LiveEventEntity; +import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; import java.time.Instant; import java.util.List; import org.springframework.data.domain.Limit; @@ -8,6 +9,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; /** * Spring Data access for {@code live_event_log}. @@ -66,12 +68,27 @@ public interface LiveEventJpaRepository Long latestRetained(@Param("streamId") String streamId, @Param("now") Instant now); /** - * Removes expired rows. + * Deletes at most {@code limit} rows retained no later than {@code now}, in a transaction of its + * own unless the caller already has one; returns the count. + * + *

Bounded, and skipping rows another replica's reaper is deleting, for the reason the outbox + * purge is bounded: a backlog of expired history is one long transaction holding every row lock + * otherwise. {@code retained_until} is written from the application clock, so it is compared with + * the application clock. * * @param now rows retained no later than this are removed + * @param limit the most rows one statement removes * @return how many went */ + // Bounded by the default transaction bound: a statement stuck behind a lock gives the connection + // back instead of holding it until the pool is empty. + @Transactional(timeout = SpringTransaction.DEFAULT_TIMEOUT_SECONDS) @Modifying - @Query("delete from LiveEventEntity e where e.retainedUntil <= :now") - int sweepExpired(@Param("now") Instant now); + @Query( + value = + "DELETE FROM live_event_log WHERE (stream_id, position) IN (" + + "SELECT stream_id, position FROM live_event_log WHERE retained_until <= :now" + + " FOR UPDATE SKIP LOCKED LIMIT :limit)", + nativeQuery = true) + int deleteExpiredBatch(@Param("now") Instant now, @Param("limit") int limit); } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaper.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaper.java new file mode 100644 index 00000000..6e57b633 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaper.java @@ -0,0 +1,53 @@ +package dev.caskeleton.adapter.outbound.persistence.liveevent; + +import dev.caskeleton.adapter.outbound.persistence.transaction.BatchedDeletes; +import java.time.Clock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Periodically deletes {@code live_event_log} rows past their retention. + * + *

A replay read already filters on {@code retained_until}, so an expired row is never served; + * without this nothing ever removed one, and the table grew by every event ever published for as + * long as replay was on (round9 review 2026-11-12, RR9-01-03). It is assembled with the replay + * store, as the idempotency and outbox reapers are with theirs, rather than left to each service to + * schedule: a duty that every deployment must remember is one some deployment forgets. + * + *

Deletes in batches of {@link #BATCH_SIZE}, each committed on its own, until a batch comes back + * short. Every replica runs it; the batch skips rows another replica's batch holds. + * + *

It carries no schedule of its own. The replay configuration registers {@link #reap()} at the + * bound {@link LiveEventReplaySettings#reaperInterval()}, so the interval reaches the schedule as a + * value rather than through a bean name an expression looks up. A deployment that builds a reaper + * itself schedules it too. + */ +public class LiveEventReaper { + + /** Rows per delete statement and transaction. */ + static final int BATCH_SIZE = BatchedDeletes.BATCH_SIZE; + + private static final Logger log = LoggerFactory.getLogger(LiveEventReaper.class); + + private final LiveEventJpaRepository repository; + private final Clock clock; + + public LiveEventReaper(LiveEventJpaRepository repository, Clock clock) { + this.repository = repository; + this.clock = clock; + } + + /** + * Deletes every row retained no later than now; returns the count. + * + *

Not transactional itself: each batch is its own transaction. + */ + public int reap() { + int purged = + BatchedDeletes.untilShort(limit -> repository.deleteExpiredBatch(clock.instant(), limit)); + if (purged > 0) { + log.debug("live event reaper purged {} expired row(s)", purged); + } + return purged; + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReplaySettings.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReplaySettings.java new file mode 100644 index 00000000..dd62e36b --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReplaySettings.java @@ -0,0 +1,55 @@ +package dev.caskeleton.adapter.outbound.persistence.liveevent; + +import dev.caskeleton.adapter.outbound.support.StrictSettingsBinder; +import java.time.Duration; +import org.springframework.core.env.Environment; + +/** + * The replay store's settings, bound strictly from {@code app.realtime.replay.*}: a misspelt key + * fails startup rather than leaving a default in force. + * + *

The reaper's interval was read only by the {@code @Scheduled} placeholder, with its default + * written twice and nothing to refuse a zero (round9 review 2026-11-12). It is bound here with the + * rest of the store's settings and read from this record by the reaper's schedule. + * + * @param enabled whether the replay store is assembled; the configuration's condition reads it + * @param retention how long an event stays replayable. No default: how long a reconnecting client + * may resume is the deployment's decision, made against what it is willing to store + * @param reaperInterval how often {@link LiveEventReaper} deletes rows past retention; 10 minutes + * unless set, which the module defaults mirror + */ +public record LiveEventReplaySettings( + boolean enabled, Duration retention, Duration reaperInterval) { + + static final String PREFIX = "app.realtime.replay"; + + private static final Duration DEFAULT_REAPER_INTERVAL = Duration.ofMinutes(10); + + public LiveEventReplaySettings { + if (retention == null) { + throw new IllegalArgumentException( + "app.realtime.replay.retention must be set when replay is enabled: how long a client may" + + " resume is the deployment's decision"); + } + if (retention.isNegative() || retention.isZero()) { + throw new IllegalArgumentException( + "app.realtime.replay.retention must be positive, was " + retention); + } + if (reaperInterval == null) { + reaperInterval = DEFAULT_REAPER_INTERVAL; + } else if (reaperInterval.isNegative() || reaperInterval.isZero()) { + throw new IllegalArgumentException( + "app.realtime.replay.reaper-interval must be positive, was " + reaperInterval); + } + } + + /** + * Binds {@link #PREFIX} strictly. + * + * @param environment the environment to bind from + * @return the bound settings + */ + public static LiveEventReplaySettings bind(Environment environment) { + return StrictSettingsBinder.bind(environment, PREFIX, LiveEventReplaySettings.class); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/lock/LockSettings.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/lock/LockSettings.java index 54cc10d6..5d19b24a 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/lock/LockSettings.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/lock/LockSettings.java @@ -5,41 +5,28 @@ import java.time.Duration; import org.springframework.core.env.Environment; /** - * Distributed-lock tuning knobs bound from {@code app.lock.*} (yaml-only defaults, not {@code - * APP_*} env keys). {@code waitTime} = max wait before timeout (default 3s); {@code leaseTtl} = max - * lock hold before auto-expiry (default 30s, must be ≥ {@code waitTime}). See README "lock". + * Distributed-lock tuning bound from {@code app.lock.*} (yaml-only defaults, not {@code APP_*} env + * keys). {@code leaseTtl} = the longest a lock may be held before the provider expires it (default + * 30s); a caller's {@code tryAcquire} may ask for that long or less. See README "lock". + * + *

There is no wait setting. How long to wait is every caller's own argument to {@code + * tryAcquire}; an {@code app.lock.wait-time} used to be bound here, checked against the lease, and + * read by nothing (round9 review 2026-11-12, RR9-01-07). It now fails startup as an unknown key, so + * a deployment that set it learns that it never did anything. */ -public record LockSettings(Duration waitTime, Duration leaseTtl) { +public record LockSettings(Duration leaseTtl) { static final String PREFIX = "app.lock"; - private static final Duration DEFAULT_WAIT_TIME = Duration.ofSeconds(3); private static final Duration DEFAULT_LEASE_TTL = Duration.ofSeconds(30); public LockSettings { - if (waitTime == null) { - waitTime = DEFAULT_WAIT_TIME; - } else if (waitTime.isNegative() || waitTime.isZero()) { - throw new IllegalArgumentException( - "app.lock.wait-time must be positive (> 0), was " + waitTime); - } if (leaseTtl == null) { leaseTtl = DEFAULT_LEASE_TTL; } else if (leaseTtl.isNegative() || leaseTtl.isZero()) { throw new IllegalArgumentException( "app.lock.lease-ttl must be positive (> 0), was " + leaseTtl); } - // Cross-field invariant: a lease shorter than the max wait is nonsensical. - // If the TTL fires before the waitTime elapses, a second holder could acquire - // the lock before the first holder's protected work is complete. - if (leaseTtl.compareTo(waitTime) < 0) { - throw new IllegalArgumentException( - "app.lock.lease-ttl (" - + leaseTtl - + ") must be >= app.lock.wait-time (" - + waitTime - + ") — a lease shorter than the max wait is nonsensical"); - } } /** diff --git a/src/adapter/outbound/persistence-jpa/src/main/resources/config/module-defaults/persistence-jpa.yml b/src/adapter/outbound/persistence-jpa/src/main/resources/config/module-defaults/persistence-jpa.yml index 9b9ff3ce..64a0eb39 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/resources/config/module-defaults/persistence-jpa.yml +++ b/src/adapter/outbound/persistence-jpa/src/main/resources/config/module-defaults/persistence-jpa.yml @@ -199,10 +199,10 @@ app: # true | false (whether to run Flyway migrations automatically at startup) migration-on-startup: ${APP_MIGRATION_ON_STARTUP:true} # feature-distributed-lock-contract D5 — distributed lock acquisition contract (try-lock + - # finite wait + lease TTL). Bound to LockSettings. Plain values, not APP_* env keys; the code - # defaults in LockSettings mirror these. + # finite wait + lease TTL). Bound to LockSettings. A plain value, not an APP_* env key; the code + # default in LockSettings mirrors it. The wait is each caller's tryAcquire argument, so there is + # no wait setting (app.lock.wait-time was read by nothing and is now an unknown key). lock: - wait-time: 3s lease-ttl: 30s realtime: replay: @@ -210,6 +210,10 @@ app: # events into the store and exposes replay/resume semantics. enabled: ${APP_REALTIME_REPLAY_ENABLED:false} retention: ${APP_REALTIME_REPLAY_RETENTION:1h} + # ISO-8601 duration — how often LiveEventReaper deletes rows past retention, 1 000 a batch. + # A plain value like the outbox reaper's interval. Bound strictly by LiveEventReplaySettings, + # whose default mirrors it; zero or negative fails startup. + reaper-interval: PT10M outbox: # feature-domain-event-outbox-contract I11 — the tuning values are literal defaults; # NO env placeholders (spec: 신규 env key 없음). Declared, validated and strictly bound by diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplayContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplayContractTest.java index b8b1b3cf..aa2e38ed 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplayContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplayContractTest.java @@ -1,5 +1,7 @@ package dev.caskeleton.adapter.outbound.persistence.liveevent; +import static org.assertj.core.api.Assertions.assertThat; + import dev.caskeleton.adapter.outbound.persistence.liveevent.entity.LiveEventEntity; import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; @@ -13,12 +15,17 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.time.Clock; import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; import java.util.Map; +import java.util.UUID; import javax.sql.DataSource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.springframework.context.annotation.AnnotationConfigApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -98,6 +105,40 @@ class JpaLiveEventReplayContractTest extends LiveEventReplayContract { return RETENTION; } + @Test + void theReaperRemovesRowsPastTheirRetentionAndKeepsTheRest() { + // given + String stream = "reaper-" + UUID.randomUUID(); + Instant start = Instant.parse("2026-11-03T00:00:00Z"); + appendAt(stream, "expired", start); + appendAt(stream, "retained", start.plus(RETENTION).minusSeconds(60)); + LiveEventReaper reaper = + new LiveEventReaper( + context.getBean(LiveEventJpaRepository.class), + Clock.fixed(start.plus(RETENTION).plusSeconds(1), ZoneOffset.UTC)); + + // when + var unused = reaper.reap(); + + // then + assertThat( + new JdbcTemplate(support.dataSource()) + .queryForList( + "SELECT payload FROM live_event_log WHERE stream_id = ?", String.class, stream)) + .containsExactly("retained"); + } + + private void appendAt(String streamId, String payload, Instant at) { + JpaLiveEventReplay writer = + new JpaLiveEventReplay( + context.getBean(LiveEventJpaRepository.class), + new JdbcTemplate(support.dataSource()), + Clock.fixed(at, ZoneOffset.UTC), + RETENTION); + var unused = + context.getBean(TransactionPort.class).inWrite(() -> writer.append(streamId, payload)); + } + private static void execute(String sql) throws SQLException { try (Connection connection = support.connection(); Statement statement = connection.createStatement()) { diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationFixtures.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationFixtures.java index d88d0217..12625a8f 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationFixtures.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/NotificationFixtures.java @@ -49,7 +49,8 @@ final class NotificationFixtures { migration("jpa/notification-platform", "V10__variables_payload_envelope_guard.sql"), migration("jpa/notification-platform", "V11__notification_reconciliation_claim_lease.sql"), migration("jpa/notification-platform", "V12__notification_suppression_target_unique.sql"), - migration("jpa/notification-platform", "V13__notification_request_acceptance.sql")); + migration("jpa/notification-platform", "V13__notification_request_acceptance.sql"), + migration("jpa/notification-platform", "V14__notification_provider_event_open_index.sql")); } /** diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/ServingStateContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/ServingStateContractTest.java index b10deb33..20154219 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/ServingStateContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/ServingStateContractTest.java @@ -8,6 +8,7 @@ import dev.caskeleton.application.notification.platform.observation.Notification import java.io.IOException; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; @@ -15,6 +16,9 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.UUID; import javax.sql.DataSource; import org.junit.jupiter.api.AfterAll; @@ -23,7 +27,12 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.PreparedStatementCreator; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.core.SqlProvider; import org.springframework.jdbc.datasource.SingleConnectionDataSource; /** @@ -100,6 +109,39 @@ class ServingStateContractTest { assertThat(measure().backlogDepth()).isZero(); } + @Test + void aDeliveryWhoseWindowClosedIsNotBacklog() throws SQLException { + // given + // Due, but past its expiry: the claim refuses it and the expiry sweep retires it unsent. + insertDelivery( + "READY_TO_DISPATCH", NOW.minus(Duration.ofMinutes(5)), null, null, NOW.minusSeconds(1)); + + // when + long depth = measure().backlogDepth(); + + // then + assertThat(depth).isZero(); + } + + @ParameterizedTest(name = "[{index}] lease_until = NOW에서 {0}초 → backlog {1}") + @CsvSource({"-1, 1", "0, 0"}) + @DisplayName("lease는 NOW 전에 끝났을 때만 끝난 것이다: claim과 같은 lease_until < now") + void aLeaseHasRunOutOnlyOnceItEndedBeforeNow(long offsetSeconds, long expectedDepth) + throws SQLException { + // given + insertDelivery( + "READY_TO_DISPATCH", + NOW.minus(Duration.ofMinutes(1)), + "worker-1", + NOW.plusSeconds(offsetSeconds)); + + // when + long depth = measure().backlogDepth(); + + // then + assertThat(depth).isEqualTo(expectedDepth); + } + @Test @DisplayName("a delivery held under a live lease is not backlog") void aDeliveryUnderALiveLeaseIsNotBacklog() throws SQLException { @@ -113,22 +155,24 @@ class ServingStateContractTest { } @Test - @DisplayName("a lease whose holder never came back is counted as stuck") void anExpiredLeaseIsStuck() throws SQLException { + // given insertDelivery( "DISPATCHING", NOW.minus(Duration.ofMinutes(20)), "worker-1", NOW.minus(Duration.ofMinutes(10))); + // when NotificationServingState state = measure(); - assertThat(state.stuckLeases()) - .as("a worker died mid-dispatch, so a provider call may have happened with no record") - .isEqualTo(1); - assertThat(state.backlogDepth()) - .as("an expired lease is claimable again, but this row is DISPATCHING, not due") - .isZero(); + // then + assertThat(state) + .as( + "a worker died mid-dispatch, so a provider call may have happened with no record; a" + + " dispatching row is never claimed, so it is not backlog") + .extracting(NotificationServingState::stuckLeases, NotificationServingState::backlogDepth) + .containsExactly(1L, 0L); } @Test @@ -162,6 +206,108 @@ class ServingStateContractTest { assertThat(state.oldestReconciliationAge()).isEqualTo(Duration.ofHours(2)); } + @Test + void theSchedulersBacklogReadIsOneStatement() throws SQLException { + // given + insertDelivery("READY_TO_DISPATCH", NOW.minus(Duration.ofMinutes(30)), null, null); + + // when + List outcome; + try (Connection connection = support.connection()) { + RecordingJdbc jdbc = new RecordingJdbc(new SingleConnectionDataSource(connection, true)); + long depth = new JdbcNotificationServingState(jdbc, CLOCK).backlogDepth(); + outcome = List.of(depth, jdbc.statements.size()); + } + + // then + assertThat(outcome) + .as("the scheduler asks this on every poll; the full snapshot is four statements") + .containsExactly(1L, 1); + } + + @Test + void theProjectionLagReadsOpenEventsThroughTheirIndex() throws SQLException { + // given + insertProviderEvent("PENDING", NOW.minus(Duration.ofMinutes(45)), false); + + // when + String plan; + try (Connection connection = support.connection()) { + RecordingJdbc jdbc = new RecordingJdbc(new SingleConnectionDataSource(connection, true)); + var unused = new JdbcNotificationServingState(jdbc, CLOCK).currentState(); + RecordedStatement lag = + jdbc.statements.stream() + .filter(statement -> statement.sql().contains("notification_provider_event")) + .findFirst() + .orElseThrow(); + plan = explainWithoutSequentialScans(connection, lag); + } + + // then + assertThat(plan) + .as("the event table is never purged; only the index over the open events bounds this read") + .contains("ix_notification_provider_event_open"); + } + + /** The plan PostgreSQL picks for {@code statement} when it may not scan a table sequentially. */ + private static String explainWithoutSequentialScans( + Connection connection, RecordedStatement statement) throws SQLException { + boolean autoCommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + try { + try (Statement off = connection.createStatement()) { + off.execute("SET LOCAL enable_seqscan = off"); + } + StringBuilder plan = new StringBuilder(); + try (PreparedStatement explain = connection.prepareStatement("EXPLAIN " + statement.sql())) { + for (int index = 0; index < statement.args().size(); index++) { + explain.setObject(index + 1, statement.args().get(index)); + } + try (ResultSet rows = explain.executeQuery()) { + while (rows.next()) { + plan.append(rows.getString(1)).append('\n'); + } + } + } + return plan.toString(); + } finally { + connection.rollback(); + connection.setAutoCommit(autoCommit); + } + } + + /** A statement the serving state ran, as it ran it. */ + private record RecordedStatement(String sql, List args) {} + + /** Plain JDBC that keeps the statements it was given, so a case reads the shipped SQL. */ + private static final class RecordingJdbc extends JdbcTemplate { + + private final List statements = new ArrayList<>(); + + private RecordingJdbc(DataSource dataSource) { + super(dataSource); + } + + @Override + public T queryForObject(String sql, RowMapper rowMapper, Object... args) { + statements.add(new RecordedStatement(sql, Arrays.asList(args))); + return super.queryForObject(sql, rowMapper, args); + } + + @Override + public T queryForObject(String sql, Class requiredType, Object... args) { + statements.add(new RecordedStatement(sql, Arrays.asList(args))); + return super.queryForObject(sql, requiredType, args); + } + + /** The named-parameter backlog read arrives here; its parameters stay inside the creator. */ + @Override + public List query(PreparedStatementCreator creator, RowMapper rowMapper) { + statements.add(new RecordedStatement(((SqlProvider) creator).getSql(), List.of())); + return super.query(creator, rowMapper); + } + } + private NotificationServingState measure() throws SQLException { // try-with-resources: the pool holds four connections, and a measurement that keeps one is a // measurement that starves the next test. @@ -174,6 +320,16 @@ class ServingStateContractTest { private void insertDelivery( String state, Instant nextDispatchAt, String leaseOwner, Instant leaseUntil) throws SQLException { + insertDelivery(state, nextDispatchAt, leaseOwner, leaseUntil, null); + } + + private void insertDelivery( + String state, + Instant nextDispatchAt, + String leaseOwner, + Instant leaseUntil, + Instant expiresAt) + throws SQLException { UUID requestId = UUID.randomUUID(); try (Connection connection = support.connection()) { try (PreparedStatement statement = @@ -197,17 +353,18 @@ class ServingStateContractTest { + " id, notification_id, tenant_id, recipient_ref, routing_plan_json," + " route_cursor, delivery_state, submission_outcome, delivery_outcome," + " evidence_level, attempt_count, next_dispatch_at, lease_owner, lease_until," - + " created_at, updated_at)" + + " expires_at, created_at, updated_at)" + " VALUES (?, ?, 'tenant-a', 'user-1', '[]'::jsonb, 0, ?, 'NOT_SUBMITTED'," - + " 'UNKNOWN', 'PLATFORM_QUEUED', 0, ?, ?, ?, ?, ?)")) { + + " 'UNKNOWN', 'PLATFORM_QUEUED', 0, ?, ?, ?, ?, ?, ?)")) { statement.setObject(1, UUID.randomUUID()); statement.setObject(2, requestId); statement.setString(3, state); statement.setTimestamp(4, Timestamp.from(nextDispatchAt)); statement.setString(5, leaseOwner); statement.setTimestamp(6, leaseUntil == null ? null : Timestamp.from(leaseUntil)); - statement.setTimestamp(7, Timestamp.from(NOW)); + statement.setTimestamp(7, expiresAt == null ? null : Timestamp.from(expiresAt)); statement.setTimestamp(8, Timestamp.from(NOW)); + statement.setTimestamp(9, Timestamp.from(NOW)); statement.executeUpdate(); } } diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/HibernateStatelessSessionRunnerIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/HibernateStatelessSessionRunnerIntegrationTest.java index 3ac505ad..282e467c 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/HibernateStatelessSessionRunnerIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/HibernateStatelessSessionRunnerIntegrationTest.java @@ -2,12 +2,16 @@ package dev.caskeleton.adapter.outbound.persistence.platform; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.persistence.hibernate.stateless.HibernateStatelessSessionRunner; import dev.caskeleton.adapter.outbound.persistence.hibernate.stateless.StatelessRowCapExceededException; import dev.caskeleton.adapter.outbound.persistence.hibernate.stateless.StatelessWorkName; import dev.caskeleton.adapter.outbound.persistence.hibernate.stateless.StatelessWorkResult; import dev.caskeleton.adapter.outbound.persistence.testkit.id.SequenceEntity; +import java.time.Duration; +import java.util.List; +import java.util.Locale; import java.util.Set; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -104,6 +108,53 @@ class HibernateStatelessSessionRunnerIntegrationTest { .hasMessageContaining("positive row cap"); } + @Test + void workStillRunningWhenTheTransactionTimeIsUpIsCancelledAndRolledBack() { + // given + var runner = + new HibernateStatelessSessionRunner( + jpa.sessionFactory(), Set.of(WORK), Duration.ofSeconds(1)); + long before = countRows(); + + // when + Throwable failure = + catchThrowable( + () -> + runner.execute( + WORK, + 10, + session -> { + session.insert(new SequenceEntity("timed-out")); + var unused = + session + .createNativeQuery("SELECT 1 FROM pg_sleep(5)", Integer.class) + .getSingleResult(); + return StatelessWorkResult.of(1, 1); + })); + + // then + assertThat(List.of(failure != null && causedByTimeout(failure), countRows())) + .as( + "the transaction's bound cancels the statement; nothing the work wrote is kept: %s", + failure) + .containsExactly(true, before); + } + + /** Whether the failure is the database or Hibernate stopping work that ran out of time. */ + private static boolean causedByTimeout(Throwable failure) { + for (Throwable current = failure; current != null; current = current.getCause()) { + String message = String.valueOf(current.getMessage()).toLowerCase(Locale.ROOT); + if (current instanceof org.hibernate.QueryTimeoutException + || current instanceof jakarta.persistence.QueryTimeoutException + || message.contains("canceling statement due to statement timeout") + || message.contains("canceling statement due to user request") + || message.contains("transaction timeout expired")) { + return true; + } + } + return false; + } + private static long countRows() { return jpa.inTransaction( entityManager -> diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/experimental/TenantPoolCapacityContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/experimental/TenantPoolCapacityContractTest.java index b2ef5043..4b53bf5e 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/experimental/TenantPoolCapacityContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/experimental/TenantPoolCapacityContractTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.platform.experimental; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; @@ -12,6 +13,7 @@ import dev.caskeleton.application.tenant.TenantId; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; import java.util.List; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -51,9 +53,9 @@ class TenantPoolCapacityContractTest { void budgetRefusesTheTenantThatWouldExceedTheCeiling() { var budget = new TenantPoolBudget(4, 8); - budget.requireCapacity(3, 6); - assertThat(budget.hasCapacity(4, 8)).isFalse(); - assertThatThrownBy(() -> budget.requireCapacity(4, 8)) + budget.requireCapacity(3, 6, 2); + assertThat(budget.hasCapacity(3, 7, 2)).isFalse(); + assertThatThrownBy(() -> budget.requireCapacity(3, 7, 2)) .isInstanceOf(IllegalStateException.class); } @@ -77,30 +79,50 @@ class TenantPoolCapacityContractTest { void openingPastTheBudgetIsRefusedBeforeAPoolIsCreated() { var budget = new TenantPoolBudget(2, 4); try (var registry = - new TenantDataSourceRegistry( - TenantPoolCapacityContractTest::poolFor, - TenantPoolCapacityContractTest::poolSizeOf, - budget)) { + new TenantDataSourceRegistry(this::poolFor, tenant -> POOL_SIZE_PER_TENANT, budget)) { registry.openTenants(2); assertThat(registry.openPools()).isEqualTo(2); assertThat(registry.allocatedConnections()).isEqualTo(4); assertThatThrownBy(() -> registry.openTenants(3)).isInstanceOf(IllegalStateException.class); - assertThat(registry.openPools()) - .as("a refused tenant must not leave a half-built pool behind") - .isEqualTo(2); + assertThat(List.of(registry.openPools(), opened.size())) + .as("a refused tenant must not leave a half-built pool behind, or have one built at all") + .containsExactly(2, 2); } } + @Test + void aTenantWhosePoolWouldPassTheConnectionCeilingIsRefusedBeforeItsPoolOpens() { + // given + // Two pools of two use four of five connections; the third pool would make six. + var budget = new TenantPoolBudget(4, 5); + List outcome; + try (var registry = + new TenantDataSourceRegistry(this::poolFor, tenant -> POOL_SIZE_PER_TENANT, budget)) { + registry.openTenants(2); + + // when + Throwable refused = catchThrowable(() -> registry.require(new TenantId("tenant-2"))); + outcome = + List.of( + refused instanceof IllegalStateException, + registry.openPools(), + registry.allocatedConnections(), + opened.size()); + } + + // then + assertThat(outcome) + .as("the third pool is refused on its planned size, so it never opens a connection") + .containsExactly(true, 2, 4, 2); + } + @Test @DisplayName("every opened tenant pool reaches the real server") void everyOpenedTenantPoolReachesTheServer() throws SQLException { var budget = new TenantPoolBudget(3, 6); try (var registry = - new TenantDataSourceRegistry( - TenantPoolCapacityContractTest::poolFor, - TenantPoolCapacityContractTest::poolSizeOf, - budget)) { + new TenantDataSourceRegistry(this::poolFor, tenant -> POOL_SIZE_PER_TENANT, budget)) { registry.openTenants(3); for (TenantId tenant : List.copyOf(registry.openTenantIds())) { @@ -118,10 +140,7 @@ class TenantPoolCapacityContractTest { void evictingATenantReturnsItsConnectionsToTheBudget() { var budget = new TenantPoolBudget(2, 4); try (var registry = - new TenantDataSourceRegistry( - TenantPoolCapacityContractTest::poolFor, - TenantPoolCapacityContractTest::poolSizeOf, - budget)) { + new TenantDataSourceRegistry(this::poolFor, tenant -> POOL_SIZE_PER_TENANT, budget)) { registry.openTenants(2); TenantId evicted = List.copyOf(registry.openTenantIds()).get(0); @@ -133,18 +152,19 @@ class TenantPoolCapacityContractTest { } } - private static javax.sql.DataSource poolFor(TenantId tenant) { + /** Every pool this test's factory built, so a test can see one was never opened. */ + private final List opened = new ArrayList<>(); + + private javax.sql.DataSource poolFor(TenantId tenant) { var config = new HikariConfig(); config.setJdbcUrl(support.jdbcUrl()); config.setUsername(support.username()); config.setPassword(support.password()); config.setMaximumPoolSize(POOL_SIZE_PER_TENANT); config.setPoolName("tenant-" + tenant.value()); - return new HikariDataSource(config); - } - - private static int poolSizeOf(javax.sql.DataSource pool) { - return ((HikariDataSource) pool).getMaximumPoolSize(); + HikariDataSource pool = new HikariDataSource(config); + opened.add(pool); + return pool; } private static int serverMaxConnections() throws SQLException { diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOptionalStreamLifecycle.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOptionalStreamLifecycle.java index f25dfaee..6a47fc8d 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOptionalStreamLifecycle.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOptionalStreamLifecycle.java @@ -104,12 +104,14 @@ final class PostgreSqlOptionalStreamLifecycle { // CONFLICT instead of a find-then-insert two webhooks could both win. // V13 records whether a stored request was accepted or dropped by a deduplication window, // so a retry with the same idempotency key gets the same receipt (review 05-23c). + // V14 indexes the provider events still waiting for something, so the projection lag reads + // them and not the whole history (round9 review 2026-11-12, RR9-01-14). // // This list is the stream's contract, not a note about its length: the lifecycle below // disables, re-enables and interrupts the stream and asserts the applied set is unchanged // each time. So a migration added to the stream belongs here, and the version that landed // without being added is why the lane failed the first time anybody ran it. - List.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13"), + List.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14"), List.of()); } diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/RunbookSql.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/RunbookSql.java index ba1855cd..6e4f3c8c 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/RunbookSql.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/RunbookSql.java @@ -8,9 +8,13 @@ import java.util.Map; /** * The SQL a runbook gives an operator, read out of the runbook itself. Every lane that runs a - * runbook's SQL reads it through this one class: so far the outbox runbooks and the migration - * runbook's recovery of a failed concurrent index migration (re-review 2026-11-03 round 8, - * RR8-03-03: the class kept an outbox name after the migration runbook began to use it). + * runbook's SQL reads it through this one class: so far the outbox runbooks ({@code + * PostgreSqlOutboxDispositionRunbookIntegrationTest}, {@code + * PostgreSqlOutboxOrderKeyRunbookIntegrationTest}), the migration runbook's recovery of a failed + * concurrent index migration ({@code PostgreSqlConcurrentIndexMigrationIntegrationTest}) and the + * lock-timeout runbook's holder lookup ({@code PostgreSqlDistributedLockIntegrationTest}). + * Re-review 2026-11-03 round 8, RR8-03-03: the class kept an outbox name after the migration + * runbook began to use it; round9 review 2026-11-12, RR9-01-10: this list missed the lock lane. * *

A runbook block is marked {@code } and followed by a {@code sql} * fence. The lanes that run these blocks against real PostgreSQL read them from the document rather diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java index 98739e4d..a483b2fe 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java @@ -1,18 +1,34 @@ package dev.caskeleton.adapter.outbound.persistence.autoconfigure; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import dev.caskeleton.adapter.outbound.persistence.liveevent.JpaLiveEventReplay; import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventJpaRepository; +import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReaper; import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.springframework.boot.context.properties.bind.UnboundConfigurationPropertiesException; import org.springframework.boot.convert.ApplicationConversionService; import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.scheduling.config.FixedDelayTask; +import org.springframework.scheduling.config.ScheduledTask; +import org.springframework.scheduling.config.ScheduledTaskHolder; class JpaLiveEventReplayActivationTest { @@ -34,6 +50,110 @@ class JpaLiveEventReplayActivationTest { }); } + @Test + void theReaperIsAssembledWithTheReplayStore() { + runner + .withPropertyValues( + "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H") + .run( + context -> + assertThat(context) + .as("without the reaper nothing removes a row past its retention") + .hasSingleBean(LiveEventReaper.class)); + } + + @Test + void noReaperIsAssembledWithoutTheReplayStore() { + runner.run(context -> assertThat(context).doesNotHaveBean(LiveEventReaper.class)); + } + + @Test + void aZeroReaperIntervalFailsStartup() { + runner + .withPropertyValues( + "app.realtime.replay.enabled=true", + "app.realtime.replay.retention=PT1H", + "app.realtime.replay.reaper-interval=PT0S") + .run( + context -> + assertThat(context.getStartupFailure()) + .rootCause() + .hasMessageContaining("app.realtime.replay.reaper-interval must be positive")); + } + + @Test + void aMisspeltReplayKeyFailsStartup() { + runner + .withPropertyValues( + "app.realtime.replay.enabled=true", + "app.realtime.replay.retention=PT1H", + "app.realtime.replay.reaper-intervall=PT1M") + .run( + context -> + assertThat(context.getStartupFailure()) + .rootCause() + .isInstanceOf(UnboundConfigurationPropertiesException.class) + .hasMessageContaining("reaper-intervall")); + } + + @ParameterizedTest(name = "[{index}] reaper-interval {0} → {1}") + @CsvSource( + value = {"UNSET, PT10M", "PT3M, PT3M"}, + nullValues = "UNSET") + @DisplayName("reaper는 묶인 reaper-interval(기본 PT10M)마다 돌고, 첫 회도 그 간격 뒤다") + void theReaperRunsAtTheBoundInterval(String configured, Duration expected) { + // given + ApplicationContextRunner replay = + runner.withPropertyValues( + "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H"); + if (configured != null) { + replay = replay.withPropertyValues("app.realtime.replay.reaper-interval=" + configured); + } + + // when + List> schedules = new ArrayList<>(); + replay.run(context -> schedules.addAll(fixedDelaySchedules(context))); + + // then + assertThat(schedules) + .as("the schedule is the bound settings' interval, its default included") + .containsExactly(List.of(expected, expected)); + } + + @Test + void theScheduledTaskIsTheReaper() { + // given + ApplicationContextRunner replay = + runner.withPropertyValues( + "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H"); + + // when + List reapedThrough = new ArrayList<>(); + replay.run( + context -> { + fixedDelayTasks(context).forEach(task -> task.getRunnable().run()); + reapedThrough.add(context.getBean(LiveEventJpaRepository.class)); + }); + + // then + verify(reapedThrough.getFirst()).deleteExpiredBatch(any(Instant.class), anyInt()); + } + + /** Interval and initial delay of every fixed-delay task the context scheduled. */ + private static List> fixedDelaySchedules(ApplicationContext context) { + return fixedDelayTasks(context).stream() + .map(task -> List.of(task.getIntervalDuration(), task.getInitialDelayDuration())) + .toList(); + } + + private static List fixedDelayTasks(ApplicationContext context) { + return context.getBean(ScheduledTaskHolder.class).getScheduledTasks().stream() + .map(ScheduledTask::getTask) + .filter(FixedDelayTask.class::isInstance) + .map(FixedDelayTask.class::cast) + .toList(); + } + @Test void enabledReplayCapabilityRequiresExplicitRetention() { runner @@ -42,7 +162,8 @@ class JpaLiveEventReplayActivationTest { context -> { assertThat(context).hasFailed(); assertThat(context.getStartupFailure()) - .hasMessageContaining("app.realtime.replay.retention"); + .rootCause() + .hasMessageContaining("app.realtime.replay.retention must be set"); }); } diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java index a560adf4..74694d1f 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.experimental; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.persistence.experimental.database.TenantPoolBudget; import dev.caskeleton.adapter.outbound.persistence.experimental.replica.ConsistencyAwareDataSourceRouter; @@ -103,14 +104,28 @@ class ExperimentalFailClosedTest { void tenantPoolBudgetIsBounded() { var budget = new TenantPoolBudget(2, 20); - assertThat(budget.hasCapacity(1, 10)).isTrue(); - assertThatThrownBy(() -> budget.requireCapacity(2, 10)) + assertThat(budget.hasCapacity(1, 10, 10)).isTrue(); + assertThatThrownBy(() -> budget.requireCapacity(2, 10, 1)) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("tenant pool budget"); - assertThatThrownBy(() -> budget.requireCapacity(1, 20)) + assertThatThrownBy(() -> budget.requireCapacity(1, 20, 1)) .isInstanceOf(IllegalStateException.class); } + @Test + void aNewPoolLargerThanTheConnectionsLeftIsRefused() { + // given + var budget = new TenantPoolBudget(4, 20); + + // when + Throwable refused = catchThrowable(() -> budget.requireCapacity(1, 15, 10)); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("tenant pool budget exhausted"); + } + /** A lag monitor with a fixed answer, including "I do not know". */ private record FixedLagMonitor(Optional replayedThrough, Optional lag) implements ReplicaLagMonitor {} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java new file mode 100644 index 00000000..1fb7f801 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java @@ -0,0 +1,45 @@ +package dev.caskeleton.adapter.outbound.persistence.hibernate.stateless; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; +import static org.mockito.Mockito.mock; + +import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; +import java.time.Duration; +import java.util.Set; +import org.hibernate.SessionFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * The stateless runner's transaction bound is the write bound every other transaction here has. + * + *

The runner restates the value instead of reading it, because the {@code hibernate} package may + * not depend on {@code transaction} (JpaModuleBoundaryTest). A test may depend on both, so it is + * the place the two copies are held to each other (round9 review 2026-11-12, RR9-01-25). + */ +class StatelessRunnerTimeoutTest { + + @Test + void theRunnersDefaultIsSpringTransactionsWriteDefault() { + assertThat(HibernateStatelessSessionRunner.DEFAULT_TIMEOUT) + .isEqualTo(SpringTransaction.Timeouts.DEFAULT.write()); + } + + @ParameterizedTest + @ValueSource(strings = {"PT0S", "PT-1S", "PT1.5S", "PT2147483648S"}) + void aTimeoutThatIsNotAWholePositiveNumberOfSecondsIsTheAssemblysFault(String timeout) { + // when + Throwable refused = + catchThrowable( + () -> + new HibernateStatelessSessionRunner( + mock(SessionFactory.class), Set.of(), Duration.parse(timeout))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("must be a positive whole number of seconds"); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java new file mode 100644 index 00000000..b43a316e --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java @@ -0,0 +1,53 @@ +package dev.caskeleton.adapter.outbound.persistence.liveevent; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link LiveEventReaper}: rows retained no later than the clock's now go, in + * bounded batches until one comes back short. + * + *

The repository is a Spring Data boundary, so Mockito stands in for it. The statement runs + * against PostgreSQL in {@code JpaLiveEventReplayContractTest}. + */ +class LiveEventReaperTest { + + private static final Instant NOW = Instant.parse("2026-11-13T00:00:00Z"); + private static final int BATCH = LiveEventReaper.BATCH_SIZE; + + private final LiveEventJpaRepository repository = mock(LiveEventJpaRepository.class); + private final LiveEventReaper reaper = + new LiveEventReaper(repository, Clock.fixed(NOW, ZoneOffset.UTC)); + + @Test + void reapKeepsDeletingBatchesUntilOneComesBackShortAndReturnsTheTotal() { + // given + when(repository.deleteExpiredBatch(NOW, BATCH)).thenReturn(BATCH, 4); + + // when + int purged = reaper.reap(); + + // then + assertThat(purged).isEqualTo(BATCH + 4); + } + + @Test + void eachBatchIsItsOwnDeleteStatement() { + // given + when(repository.deleteExpiredBatch(NOW, BATCH)).thenReturn(BATCH, 4); + + // when + var unused = reaper.reap(); + + // then + verify(repository, times(2)).deleteExpiredBatch(NOW, BATCH); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java index 72fe3def..471a36ba 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java @@ -49,6 +49,20 @@ class DistributedLockPersistenceConfigTest { }); } + @Test + @DisplayName("읽는 곳이 없던 app.lock.wait-time은 모르는 키로 기동을 실패시킨다") + void theRemovedWaitTimeKeyFailsStartup() { + // when + contextRunner + .withPropertyValues("app.lock.wait-time=3s") + .run( + context -> { + // then + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()).hasStackTraceContaining("wait-time"); + }); + } + @Test void multiInstanceBuildsTheJdbcProviderAndNotTheInProcessProvider() { contextRunner diff --git a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 index 32edf890..93a51fe4 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 +++ b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 @@ -14,6 +14,7 @@ be51fabd07220f9ef885534ba36ada5f1491baf24cdfb54d25d93a19ddc29926 jpa/notification-platform/V11__notification_reconciliation_claim_lease.sql 43f9429f91c07528154ec01b721649b2c25fa98857ac154c7c08f4655634657f jpa/notification-platform/V12__notification_suppression_target_unique.sql d01aaf59f66834358abf18246faa9a9eb8dfa9f42097ca9d968c44edff564630 jpa/notification-platform/V13__notification_request_acceptance.sql +8a7dc3604d44f1438b39dfec3aa688f792e7363cd3c1b74bf0c799a6b53f9f0d jpa/notification-platform/V14__notification_provider_event_open_index.sql 1ac7cc1c2509868eb0ce202e1917bb36846f98f17ad4922abe3fac0ef128199b jpa/notification-platform/V1__notification_platform_core.sql 0add67aa23580e51712f84fabe9b09e08401ef708cfa1b3292b120fdb5c12fa2 jpa/notification-platform/V2__notification_platform_contact_template_policy.sql f5b98598a59cc4376f5d233b410f8a5ae9f3cd94156291ebdf2d0da5a66f34ef jpa/notification-platform/V3__notification_platform_inbox_admin.sql diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java index 5a81e94b..72a52a8e 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java @@ -1,13 +1,17 @@ package dev.caskeleton.adapter.outbound.persistence.notification.platform; +import dev.caskeleton.adapter.outbound.persistence.notification.platform.repository.RecipientClaimSql; import dev.caskeleton.application.notification.platform.observation.NotificationServingState; import dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort; import java.sql.Timestamp; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Map; import java.util.Objects; import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; /** * Measures the platform's serving state from the durable rows. @@ -17,34 +21,44 @@ import org.springframework.jdbc.core.JdbcOperations; * identical. These are the questions an operator actually asks during an incident: how far behind * are we, how long has the oldest thing waited, and is anything stuck. * - *

One statement per question rather than one joined statement. They are read independently, a - * slow one does not delay the others, and each index the queries rely on is the same index the - * corresponding worker already uses to claim. + *

One statement per question rather than one joined statement. They are read independently and a + * slow one does not delay the others. Every statement is bounded by an index: the backlog and + * stuck-lease counts by the predicates the claim uses, the reconciliation count by the job's due + * time, and the projection lag by {@code ix_notification_provider_event_open} (V14), which holds + * only the events still waiting for something — the provider-event table itself is never purged, so + * the lag query used to read all of it (round9 review 2026-11-12, RR9-01-14). + * + *

The scheduler reads only {@link #backlogDepth()}, on every poll; the full snapshot is for the + * health endpoint. */ public final class JdbcNotificationServingState implements NotificationServingStatePort { /** - * Deliveries that are due and unclaimed. + * Deliveries the claim would take now. * - *

The lease predicate matches the claim's: a row whose lease has expired is available again, - * so it is backlog. Counting it as in-flight is how a stuck worker hides a growing queue. + *

The {@code WHERE} is the claim's own fragment ({@link RecipientClaimSql#CLAIMABLE_AT_NOW}): + * the same states, due, not expired, and a lease that is absent or has run out. A row whose lease + * has expired is available again, so it is backlog; counting it as in-flight is how a stuck + * worker hides a growing queue. An expired delivery is not backlog: the expiry sweep retires it + * unsent. The fragment's parameter is named, so this statement is too. */ private static final String BACKLOG = """ SELECT count(*) AS depth, - COALESCE(EXTRACT(EPOCH FROM (? - min(next_dispatch_at))), 0) AS oldest_seconds + COALESCE(EXTRACT(EPOCH FROM (:now - min(next_dispatch_at))), 0) AS oldest_seconds FROM notification_recipient_delivery - WHERE delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING') - AND next_dispatch_at IS NOT NULL - AND next_dispatch_at <= ? - AND (lease_until IS NULL OR lease_until <= ?) - """; + WHERE\s""" + + RecipientClaimSql.CLAIMABLE_AT_NOW; /** * Leases whose holder never came back. * - *

Distinct from backlog: these rows are claimable again, but the fact that they had to expire - * means a worker died mid-dispatch, and a provider call may have happened without a record. + *

Distinct from backlog: these rows are still dispatching, which the claim never takes; lease + * recovery requeues or reconciles them one lease after they ran out. A row left from before + * notification V4 (an owner but fence 0) is counted here and not recovered; {@code + * docs/upgrade/2026-11-round9-claim-fence.md}, row 5, says how to hand it to recovery. That a + * lease had to run out means a worker died or stalled mid-dispatch, and a provider call may have + * happened without a record. */ private static final String STUCK_LEASES = """ @@ -56,7 +70,21 @@ public final class JdbcNotificationServingState implements NotificationServingSt AND delivery_state = 'DISPATCHING' """; - /** Provider events stored and not yet applied to a delivery. */ + /** + * The provider events still waiting for something: the lag query's {@code WHERE}, and the + * predicate of the partial index {@code V14} builds for it. One string, held to the migration by + * {@code JdbcNotificationServingStateOpenEventsTest}: a query the index does not cover reads the + * whole table again, which is what the index was made to stop. + */ + static final String OPEN_EVENTS = + "projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL"; + + /** + * Provider events stored and not yet applied to a delivery. + * + *

The {@code WHERE} is exactly the predicate of {@code ix_notification_provider_event_open}: + * every row any of the four counts can see, and none of the applied, matched history. + */ private static final String PROJECTION_LAG = """ SELECT @@ -66,7 +94,9 @@ public final class JdbcNotificationServingState implements NotificationServingSt COALESCE(EXTRACT(EPOCH FROM (? - min(received_at) FILTER (WHERE projection_status = 'PENDING'))), 0) AS oldest_seconds FROM notification_provider_event - """; + WHERE + """ + + OPEN_EVENTS; /** Questions to providers that are due and unanswered. */ private static final String RECONCILIATION_LAG = @@ -78,26 +108,36 @@ public final class JdbcNotificationServingState implements NotificationServingSt """; private final JdbcOperations jdbc; + private final NamedParameterJdbcOperations named; private final Clock clock; public JdbcNotificationServingState(JdbcOperations jdbc, Clock clock) { this.jdbc = Objects.requireNonNull(jdbc, "jdbc"); + this.named = new NamedParameterJdbcTemplate(jdbc); this.clock = Objects.requireNonNull(clock, "clock"); } + @Override + public long backlogDepth() { + Measurement backlog = measureBacklog(Timestamp.from(clock.instant())); + return backlog == null ? 0 : backlog.count(); + } + + /** The backlog query alone: the one statement the scheduler's queue-depth gauge needs. */ + private Measurement measureBacklog(Timestamp at) { + return named.queryForObject( + BACKLOG, + Map.of("now", at), + (rows, rowNumber) -> + new Measurement(rows.getLong("depth"), rows.getDouble("oldest_seconds"))); + } + @Override public NotificationServingState currentState() { Instant now = clock.instant(); Timestamp at = Timestamp.from(now); - Measurement backlog = - jdbc.queryForObject( - BACKLOG, - (rows, rowNumber) -> - new Measurement(rows.getLong("depth"), rows.getDouble("oldest_seconds")), - at, - at, - at); + Measurement backlog = measureBacklog(at); Long stuck = jdbc.queryForObject(STUCK_LEASES, Long.class, at); ProjectionMeasurement projection = jdbc.queryForObject( diff --git a/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql b/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql new file mode 100644 index 00000000..2d2416a2 --- /dev/null +++ b/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql @@ -0,0 +1,26 @@ +-- An index over the provider events still waiting for something. +-- +-- The serving-state projection lag counts pending, failed and unmatched events. It had no WHERE and +-- read the whole notification_provider_event table, which nothing ever purges, and the scheduler +-- asked for it on every poll (round9 review 2026-11-12, RR9-01-14). The query now names exactly +-- this predicate, so it reads only these rows; applied, matched events — the history that grows — +-- are outside the index. +-- +-- Built inside this migration's transaction, not CONCURRENTLY, as V21 of the core stream builds its +-- index: a transactional file fails whole and leaves nothing to repair, where a failed concurrent +-- build leaves an INVALID index and a failed history row an operator must clear by hand +-- (docs/runbooks/migration-failed.md). The cost is that the build holds SHARE on +-- notification_provider_event: provider callbacks wait for it, and providers retry a callback that +-- does not get an answer. On a table already large in production, create the same index +-- CONCURRENTLY by hand before deploying; IF NOT EXISTS then makes this migration a no-op. + +-- Bound the wait for the lock (SHARE on notification_provider_event, held until commit). Unbounded, +-- it queues behind any open transaction that touched the table, and every callback insert queues +-- behind the migration. Five seconds, then the migration fails, rolls back whole and the stream +-- stays at V13: deploy again once that transaction has ended (docs/jpa/migration-guide.md "Bound +-- the lock wait"). +SET LOCAL lock_timeout = '5s'; + +CREATE INDEX IF NOT EXISTS ix_notification_provider_event_open + ON notification_provider_event (received_at) + WHERE projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL; diff --git a/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java b/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java new file mode 100644 index 00000000..0805a0d6 --- /dev/null +++ b/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java @@ -0,0 +1,42 @@ +package dev.caskeleton.adapter.outbound.persistence.notification.platform; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; +import org.junit.jupiter.api.Test; + +/** + * The projection-lag query and the partial index built for it name the same events. + * + *

The index is what keeps the lag query, which the scheduler runs on every poll, from reading + * the whole provider-event history. A predicate that drifted on either side would leave the query + * outside the index and scanning the table again, with nothing failing. + */ +class JdbcNotificationServingStateOpenEventsTest { + + private static final String V14 = + "db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql"; + + @Test + void theLagQueryAndItsPartialIndexNameTheSameEvents() throws IOException { + // given + String migration; + try (InputStream sql = getClass().getClassLoader().getResourceAsStream(V14)) { + migration = new String(Objects.requireNonNull(sql, V14).readAllBytes(), UTF_8); + } + int index = migration.indexOf("CREATE INDEX"); + + // when + String indexPredicate = + migration + .substring( + migration.indexOf("WHERE ", index) + "WHERE ".length(), migration.lastIndexOf(';')) + .trim(); + + // then + assertThat(indexPredicate).isEqualTo(JdbcNotificationServingState.OPEN_EVENTS); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java index ce48cf07..7ddece3f 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java @@ -12,7 +12,7 @@ import java.util.Objects; * being applied reported exactly the same numbers as an idle healthy one. Readiness that cannot * distinguish those two states is not readiness. * - * @param backlogDepth deliveries due now and not yet claimed + * @param backlogDepth deliveries a claim would take now: due, not expired, not under a live lease * @param oldestDueAge how long the oldest of them has been due * @param stuckLeases leases whose holder has not finished and whose expiry has passed * @param pendingProjections provider events stored and not yet applied diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java index 79a817aa..f6ae4f15 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java @@ -6,7 +6,25 @@ public interface NotificationServingStatePort { /** * Measure the backlog, the stuck leases and the projection and reconciliation lag. * + *

Several questions, for the health snapshot. A caller that needs only the queue depth asks + * {@link #backlogDepth()} instead: the scheduler reads the depth on every poll, and asking all of + * them that often ran four statements a poll, one of them over the whole provider-event table + * (round9 review 2026-11-12, RR9-01-14). + * * @return the current state */ NotificationServingState currentState(); + + /** + * How many deliveries a claim would take now — due, not expired, and not under a live lease: the + * {@link NotificationServingState#backlogDepth()} of {@link #currentState()}, measured alone. + * + *

The default takes it from the full snapshot, so an implementation that can only measure + * everything at once still answers; a store that can count the backlog by itself overrides it. + * + * @return the backlog depth + */ + default long backlogDepth() { + return currentState().backlogDepth(); + } } From 154cb4b7959916b66fcd99b18380967d59b83924 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 13 Nov 2026 16:21:16 +0900 Subject: [PATCH 3/8] =?UTF-8?q?fix(outbound)!:=20=EB=B0=9C=ED=96=89=20?= =?UTF-8?q?=EB=8C=80=EA=B8=B0=C2=B7SMTP=20=EB=B0=9C=EC=86=A1=C2=B7?= =?UTF-8?q?=EC=BA=90=EC=8B=9C=20=EC=8B=9C=EA=B0=84=20=EC=83=81=ED=95=9C?= =?UTF-8?q?=EC=9D=84=20=EC=A7=80=ED=82=A8=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit outbox 발행 대기가 publish()가 돌아온 뒤부터 재서, 호출을 붙잡은 publisher는 longestAttempt를 넘겨도 relay를 묶는다. SMTP 발송 pool은 대기열이 찰 때까지 thread 하나로 돌고, profile들의 max-concurrency가 pool이 담는 수를 넘어도 기동한다. 넘친 발송은 결과 불명으로 남는다. SMTP profile timeout은 명령 하나만 묶어, 느린 relay에서 발송이 lease를 넘으면 lease 복구가 아직 도는 발송을 운영자 확인으로 넘긴다. Redis 캐시 command-timeout에 lane 대기 2초가 더해지고, 값 크기 상한은 설정할 길이 없다. 고른 방법: 발행 대기 마감은 publish() 전에 잡고 남은 시간만 기다린다. 기다리는 중 interrupt는 결과 불명, 이미 interrupt된 thread와 0 이하 longestAttempt는 보내지 않고 REJECTED_RETRYABLE로 돌려준다. SMTP pool은 dispatch-concurrency개 thread와 그 네 배 대기열로 고정한다. 한 family의 켜진 profile 전체를 보는 ProviderRuntimeAssembler.requireTogether를 두고, SMTP는 max-concurrency 합이 pool보다 크거나 lease가 timeout 두 배 이하면 조립을 실패시킨다. profile timeout은 발송 한 번을 두 번 묶는다: pool 대기가 넘으면 보내지 않고 재시도, 발송이 넘으면 PROVIDER_RESPONSE_LOST 로 남겨 다시 보내지 않는다. 두 시한은 completeOnTimeout이라 취소된다. 기본 timeout은 10초에서 30초로, socket timeout 합보다 길어야 한다. Redis 연산 전체를 command-timeout 하나로 묶고 app.capabilities.cache.maximum-value-bytes를 다시 묶는다(1..16 MiB). 버린 대안: 발송 timeout에 재시도(같은 메일을 두 번 보낼 수 있다), 캐시 상한을 상수로 두기(큰 값을 쓰는 서비스가 바꿀 길이 없다), assembler가 assemble마다 합을 누적하기(조립 순서에 상태가 남는다). 위험·한계: 기동 조건이 늘었다(SMTP max-concurrency 합, lease와 SMTP timeout, 캐시 값 상한 범위). 대기 시한이 돌려주는 stage는 일부러 버린다 (Error Prone 규칙대로 `unused`에 받고, SpotBugs DLS는 config/spotbugs/ exclude.xml에 그 메서드만 제외한다. Jetty abortAtDeadline 선례). send pool을 바꿔 끼운 배포는 부른 thread에서 돌리지 않는 executor여야 한다. outbox 새 거절 코드의 runbook 줄은 runbook 묶음에 넘긴다. claim·fence 커밋 위에 쌓았다. 검증: desktop RED(고치기 전 트리 + 새 시험, pr2-red2) 새 시험 실패. GREEN은 PR 본문의 verify.sh pr 결과. Refs: RR9-01-05, RR9-01-15, RR9-01-16, RR9-01-17, RR9-01-18 BREAKING CHANGE: SMTP profile들의 max-concurrency 합이 send pool(thread와 대기열)을 넘거나 dispatch.lease-duration이 SMTP timeout의 두 배 이하면 기동이 실패한다. SMTP 기본 timeout이 30초이고, 넘은 발송은 결과 불명이 된다. SmtpNotificationProviderAdapter 생성자가 timeout을 받는다. Redis 캐시 값이 maximum-value-bytes(기본 1 MiB)를 넘으면 기록하지 않는다. docs/upgrade/2026-11-round9-outbound.md --- .env.example | 3 +- config/spotbugs/exclude.xml | 9 ++ docs/notification/configuration-reference.md | 33 +++- docs/registries/env-keys.yaml | 11 +- docs/upgrade/2026-11-round9-claim-fence.md | 2 +- docs/upgrade/2026-11-round9-outbound.md | 29 ++++ src/adapter/outbound/cache-redis/README.md | 7 +- .../cache/redis/SemanticRedisCall.java | 24 +++ .../redis/cache/RedisCacheRegionAdapter.java | 73 ++++++++- .../redis/capability/RedisCacheConfig.java | 3 +- .../capability/RedisCapabilitySettings.java | 26 +++ .../config/module-defaults/cache-redis.yml | 3 + .../cache/RedisCacheRegionAdapterTest.java | 127 ++++++++++++++- .../RedisCacheValueCeilingSettingTest.java | 40 +++++ src/adapter/outbound/messaging/AGENTS.md | 4 +- src/adapter/outbound/messaging/README.md | 11 ++ .../outbox/OutboxMessagePublishAdapter.java | 69 +++++++- .../OutboxMessagePublishAdapterTest.java | 150 ++++++++++++++++++ .../NotificationProviderAssembly.java | 28 ++++ .../NotificationSmtpProviderConfig.java | 20 ++- .../ProviderRuntimeAssembler.java | 18 +++ .../SmtpProviderRuntimeAssembler.java | 101 +++++++++++- .../smtp/SmtpNotificationProviderAdapter.java | 140 +++++++++++++++- .../NotificationProviderAssemblyTest.java | 118 ++++++++++++++ .../NotificationSmtpExecutorTest.java | 58 +++++++ .../SmtpProviderRuntimeAssemblerTest.java | 114 ++++++++++++- .../SmtpNotificationProviderAdapterTest.java | 130 ++++++++++++++- .../event/IntegrationEventPublishPort.java | 9 +- .../FakeIntegrationEventPublisher.java | 65 +++++++- .../config/module-defaults/notification.yml | 2 +- 30 files changed, 1384 insertions(+), 43 deletions(-) create mode 100644 docs/upgrade/2026-11-round9-outbound.md create mode 100644 src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisCacheValueCeilingSettingTest.java create mode 100644 src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationSmtpExecutorTest.java diff --git a/.env.example b/.env.example index 1c68b40e..09bb6b74 100644 --- a/.env.example +++ b/.env.example @@ -171,6 +171,7 @@ APP_CACHE_REDIS_COMMAND_TIMEOUT=200ms APP_CACHE_REDIS_POSITIVE_SOFT_TTL=30s APP_CACHE_REDIS_MINIMUM_HARD_TTL=1s APP_CACHE_REDIS_SEMANTIC_REGION=default +APP_CACHE_REDIS_MAXIMUM_VALUE_BYTES=1048576 APP_MESSAGING_BROKER=kafka APP_MESSAGING_PRODUCER_ID= APP_MESSAGING_KAFKA_BROKERS= @@ -293,7 +294,7 @@ APP_NOTIFICATION_PLATFORM_SMTP_ENABLED=false APP_NOTIFICATION_PLATFORM_SMTP_PRIMARY=true APP_NOTIFICATION_PLATFORM_SMTP_ENVIRONMENT=local APP_NOTIFICATION_PLATFORM_SMTP_CREDENTIAL_PROFILE=default -APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT=10s +APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT=30s APP_NOTIFICATION_PLATFORM_SMTP_MAX_CONCURRENCY=4 APP_NOTIFICATION_PLATFORM_SMTP_RATE_PER_SECOND=10 APP_NOTIFICATION_PLATFORM_SMTP_SENDER_IDENTITY=no-reply@example.invalid diff --git a/config/spotbugs/exclude.xml b/config/spotbugs/exclude.xml index 2ee07473..156d6eae 100644 --- a/config/spotbugs/exclude.xml +++ b/config/spotbugs/exclude.xml @@ -359,6 +359,15 @@ + + + + + + + + methods for the same reason. Every use case carries @UseCaseCapability and is not final by + the use-case contract (src/application/core/AGENTS.md), @SystemInvoked and query use cases + included, so the annotation matches them too; together with the *UseCase name, so a class + that is not a use case does not hide a finding behind the annotation. --> + + + + diff --git a/docs/manual/05-idempotency.md b/docs/manual/05-idempotency.md index fce96ff2..8c8cd538 100644 --- a/docs/manual/05-idempotency.md +++ b/docs/manual/05-idempotency.md @@ -44,6 +44,27 @@ DB 커밋 중 연결 끊김 ───────────────── executor 조립과 실제 PostgreSQL 저장소 왕복은 `PostgreSqlIdempotencyProviderIntegrationTest`(app-bootstrap `integrationTest`)다. KEYED 유스케이스를 HTTP부터 끝까지 이은 업무 기능 예는 템플릿에 없다. +### executor를 쓰지 않는 중복 판정 둘 (2026-11-13 결정) + +템플릿에는 `IdempotencyExecutor` 말고 중복을 판정하는 곳이 둘 더 있다. 둘 다 일부러 executor로 옮기지 않는다. +executor는 **응답을 다시 돌려주는 기록**이다: HTTP `Idempotency-Key` 헤더로 들어오고, 원본 키 대신 scope digest만 +저장하고, 기록은 TTL이 지나면 지운다. 아래 둘은 **업무 행 자체의 정체성**이라 수명과 판정 대상이 다르다. + +| 어디 | 키 | 같은 키·같은 요청 | 같은 키·다른 요청 | 수명 | +| --- | --- | --- | --- | --- | +| executor(KEYED 유스케이스) | `Idempotency-Key` 헤더의 scope digest | 저장된 응답 재생 | 422 `IDEMPOTENT_REQUEST_MISMATCH` | TTL | +| 알림 받기(`AcceptNotificationApplicationUseCase`) | 본문의 `idempotencyKey`. 없으면 알림 plan 지문 | 처음 만든 알림의 receipt | `IDEMPOTENCY_CONFLICT`(요청 지문 비교) | 알림 행과 같다 | +| durable operation 제출(`SubmitDurableOperationUseCase`) | 제출 scope(tenant·principal·operation 이름)와 payload 지문 | 처음 받은 operation | 지문이 키에 들어 있어 다른 키다. 새 operation | operation 보존 기간 | + +- 알림: 키는 tenant 안에서 같은 알림을 한 번만 만드는 업무 키다. 저장소의 유일 제약이 동시 제출을 막고, 진 쪽은 이긴 행을 + 읽어 같은 receipt로 수렴한다. 키를 주지 않은 호출자도 중복을 막도록 plan 지문으로 키를 만든다. executor로 옮기면 TTL + 뒤 같은 알림을 다시 만들고, 키 없는 호출은 중복을 막지 못한다. 저장한 키 값을 어떻게 다룰지(원본 저장)는 보안 검토 + 항목이다. +- durable operation: 결정 15·26(서버가 id를 만들고 payload 지문으로 재제출을 판정)이 정한다. capability의 `KEYED`는 + "같은 scope·같은 payload로 다시 보내면 같은 작업"이라는 뜻이고, 그 판정은 operation 저장소의 scope 유일성이 한다. +- 새 기능이 이 둘을 따라 할 일은 거의 없다. 클라이언트가 응답을 못 받아 다시 보내는 쓰기는 위 순서대로 executor를 쓴다. + 업무 행 자체가 영구히 하나여야 하면(04장 유일 제약) 그 행의 키로 판정한다. + ## 클라이언트에게 무엇을 약속하나 | 응답 | 의미 | 클라이언트 행동 | diff --git a/docs/notification/api-surface-snapshot.txt b/docs/notification/api-surface-snapshot.txt index 691f0444..f5622441 100644 --- a/docs/notification/api-surface-snapshot.txt +++ b/docs/notification/api-surface-snapshot.txt @@ -2,7 +2,7 @@ # Every top-level public type under the platform packages. Growth is a reviewed change. # Update only after review with: # ./gradlew :adapter:outbound:notification:updateNotificationApiSurface -PapproveNotificationApiSurfaceChange -# types: 364 +# types: 359 dev.caskeleton.adapter.outbound.notification.NotificationConfig dev.caskeleton.adapter.outbound.notification.NotificationRoutesSettings dev.caskeleton.adapter.outbound.notification.core.FailOpenNotificationProvider @@ -136,7 +136,6 @@ dev.caskeleton.application.notification.platform.api.NotificationPlan dev.caskeleton.application.notification.platform.api.NotificationReceipt dev.caskeleton.application.notification.platform.api.NotificationSnapshot dev.caskeleton.application.notification.platform.api.NotificationVariable -dev.caskeleton.application.notification.platform.api.ProviderEventId dev.caskeleton.application.notification.platform.api.ProviderId dev.caskeleton.application.notification.platform.api.ProviderProfileId dev.caskeleton.application.notification.platform.api.RecipientDeliveryId @@ -168,7 +167,6 @@ dev.caskeleton.application.notification.platform.api.error.CallbackValidationExc dev.caskeleton.application.notification.platform.api.error.FailureCategory dev.caskeleton.application.notification.platform.api.error.IdempotencyConflictException dev.caskeleton.application.notification.platform.api.error.NotificationException -dev.caskeleton.application.notification.platform.api.error.NotificationExpiredException dev.caskeleton.application.notification.platform.api.error.NotificationFailureCode dev.caskeleton.application.notification.platform.api.error.NotificationFailureDescriptor dev.caskeleton.application.notification.platform.api.error.NotificationValidationException @@ -329,14 +327,11 @@ dev.caskeleton.application.notification.platform.port.in.SubmitNotificationComma dev.caskeleton.application.notification.platform.port.in.SubmitNotificationUseCase dev.caskeleton.application.notification.platform.provider.AttachmentAccessContext dev.caskeleton.application.notification.platform.provider.AttachmentResolver -dev.caskeleton.application.notification.platform.provider.BatchNotificationProviderAdapter -dev.caskeleton.application.notification.platform.provider.CollapseCapability dev.caskeleton.application.notification.platform.provider.EvidenceCertainty dev.caskeleton.application.notification.platform.provider.EvidenceFact dev.caskeleton.application.notification.platform.provider.NotificationProviderAdapter dev.caskeleton.application.notification.platform.provider.ProviderCallNotStartedException dev.caskeleton.application.notification.platform.provider.ProviderCapabilities -dev.caskeleton.application.notification.platform.provider.ProviderCollapseMapping dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence dev.caskeleton.application.notification.platform.provider.ProviderFailure dev.caskeleton.application.notification.platform.provider.ProviderProfileSnapshot diff --git a/docs/upgrade/2026-11-round9-application.md b/docs/upgrade/2026-11-round9-application.md new file mode 100644 index 00000000..e985b88c --- /dev/null +++ b/docs/upgrade/2026-11-round9-application.md @@ -0,0 +1,22 @@ +# 업그레이드 안내 — 2026-11 application 구조와 작은 정리 + +- 범위: durable operation 제출이 id를 유스케이스에서 만들고 tenant를 타입으로 받게 한 변경, 쓰지 않던 공개 타입 일곱의 + 삭제, 그 밖의 작은 정리. 공개 타입과 생성자가 바뀐 곳이 있다. 아래 표가 그 목록이다. +- 근거: 2026-11-12 전체 리뷰(`docs/reviews/2026-11-12-round9-review/`)의 application 작은 것(B15b) 발견 + +## 1. 누가 읽나 + +이 템플릿을 `upstream` remote로 두고 merge해 받는 파생 서비스가 읽는다. 특히 durable operation을 직접 제출·조회하는 +서비스와 아래 표의 지운 타입을 쓴 서비스다. + +## 2. 할 일 + +| # | 할 일 | 안 하면 | 해당 서비스 | +| --- | --- | --- | --- | +| 1 | `OperationCommand`에서 operation id 인자를 지우고, tenant를 `Optional`(`application.tenant.TenantId`)로 넘긴다. `GetDurableOperationQuery`·`CancelDurableOperationCommand`의 tenant도 같다. `SubmitDurableOperationUseCase`는 생성자에 `DurableOperationIdFactory`를 받아 id를 직접 만든다. `DurableOperationController`는 id factory를 받지 않는다 | 컴파일이 실패한다 | durable operation을 직접 제출·조회하거나 그 use case를 조립한 서비스 | +| 2 | 쓰는 곳이 없던 공개 타입 일곱을 지웠다: `application.transaction.ReadConsistency`, `FileserverSpans`, 알림의 `ProviderEventId`·`NotificationExpiredException`·`BatchNotificationProviderAdapter`·`CollapseCapability`·`ProviderCollapseMapping` | 그 타입을 쓴 코드는 컴파일이 실패한다. 템플릿 안에는 쓰는 곳이 없었다 | 그 타입을 쓴 서비스 | + +## 3. 바뀐 동작 (따라 할 일 없음) + +- `GetDurableOperationUseCase`·`PublishPendingOutboxEventsUseCase`가 final이 아니다. +- 알림 받기의 `idempotencyKey`가 `IdempotencyExecutor`와 다른 이유는 manual 05 「executor를 쓰지 않는 중복 판정 둘」에 있다. diff --git a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/operation/DurableOperationController.java b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/operation/DurableOperationController.java index cdf7a8ab..c58d7e53 100644 --- a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/operation/DurableOperationController.java +++ b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/operation/DurableOperationController.java @@ -7,7 +7,6 @@ import dev.caskeleton.application.idempotency.RequestFingerprint; import dev.caskeleton.application.operation.CancelDurableOperationCommand; import dev.caskeleton.application.operation.CancelDurableOperationUseCase; import dev.caskeleton.application.operation.DurableOperationId; -import dev.caskeleton.application.operation.DurableOperationIdFactory; import dev.caskeleton.application.operation.GetDurableOperationQuery; import dev.caskeleton.application.operation.GetDurableOperationUseCase; import dev.caskeleton.application.operation.OperationAcceptance; @@ -19,6 +18,7 @@ import jakarta.validation.Valid; import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.Objects; +import java.util.Optional; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.authentication.AuthenticationCredentialsNotFoundException; @@ -51,7 +51,6 @@ public final class DurableOperationController { private final SubmitDurableOperationUseCase submit; private final GetDurableOperationUseCase get; private final CancelDurableOperationUseCase cancel; - private final DurableOperationIdFactory operationIds; /** * A controller over the three operation use cases. @@ -59,15 +58,12 @@ public final class DurableOperationController { * @param submit accepts work * @param get reads one operation * @param cancel requests cancellation - * @param operationIds mints the id of a new operation (a UUIDv7: it is also the event id) */ public DurableOperationController( SubmitDurableOperationUseCase submit, GetDurableOperationUseCase get, - CancelDurableOperationUseCase cancel, - DurableOperationIdFactory operationIds) { + CancelDurableOperationUseCase cancel) { this.submit = Objects.requireNonNull(submit, "submit"); - this.operationIds = Objects.requireNonNull(operationIds, "operationIds"); this.get = Objects.requireNonNull(get, "get"); this.cancel = Objects.requireNonNull(cancel, "cancel"); } @@ -75,10 +71,9 @@ public final class DurableOperationController { /** * Accepts an operation. * - *

{@code 202} with the operation's {@code Location}. The id is minted here, by {@link - * DurableOperationIdFactory}; a resubmission of the same work by the same caller is recognised by - * its payload fingerprint and answered with the operation that already exists, {@code - * alreadyAccepted} true and the same {@code Location}. + *

{@code 202} with the operation's {@code Location}. The use case mints the id; a resubmission + * of the same work by the same caller is recognised by its payload fingerprint and answered with + * the operation that already exists, {@code alreadyAccepted} true and the same {@code Location}. */ @PostMapping public ResponseEntity> submit( @@ -88,7 +83,6 @@ public final class DurableOperationController { OperationAcceptance acceptance = submit.handle( new OperationCommand( - operationIds.newOperationId(), request.operationName(), principalId(principal), tenantOf(principal), @@ -155,9 +149,9 @@ public final class DurableOperationController { return principal.idpUserId(); } - // The operation use cases take the tenant as a plain string; null means "not tenant scoped". - private static String tenantOf(AuthenticatedPrincipal principal) { - return principal == null ? null : principal.tenant().map(TenantId::value).orElse(null); + // Empty means "not tenant scoped", the same as a token without a tenant claim. + private static Optional tenantOf(AuthenticatedPrincipal principal) { + return principal == null ? Optional.empty() : principal.tenant(); } private static ResponseEntity> notFound() { diff --git a/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/operation/DurableOperationControllerTest.java b/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/operation/DurableOperationControllerTest.java index 4039ff1a..ae20c86d 100644 --- a/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/operation/DurableOperationControllerTest.java +++ b/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/operation/DurableOperationControllerTest.java @@ -70,10 +70,10 @@ class DurableOperationControllerTest { outbox, new FakeIntegrationEventEncoder(new DurableOperationSubmittedContract()), transactions, + new UuidV7DurableOperationIdFactory(), clock), new GetDurableOperationUseCase(operations, transactions, clock), - new CancelDurableOperationUseCase(operations, transactions, clock), - new UuidV7DurableOperationIdFactory()); + new CancelDurableOperationUseCase(operations, transactions, clock)); mvc = MockMvcBuilders.standaloneSetup(controller) .setControllerAdvice( @@ -141,10 +141,14 @@ class DurableOperationControllerTest { MockMvcBuilders.standaloneSetup( new DurableOperationController( new SubmitDurableOperationUseCase( - operations, outbox, new FakeIntegrationEventEncoder(), transactions, clock), + operations, + outbox, + new FakeIntegrationEventEncoder(), + transactions, + new UuidV7DurableOperationIdFactory(), + clock), new GetDurableOperationUseCase(operations, transactions, clock), - new CancelDurableOperationUseCase(operations, transactions, clock), - new UuidV7DurableOperationIdFactory())) + new CancelDurableOperationUseCase(operations, transactions, clock))) .setControllerAdvice( new GlobalExceptionHandler(SpanErrorRecorder.NOOP), new EnvelopeBodyAdvice()) .setCustomArgumentResolvers(new AuthenticationPrincipalArgumentResolver()) diff --git a/src/adapter/outbound/identifier/README.md b/src/adapter/outbound/identifier/README.md index 4ad90bb5..c4c13f0a 100644 --- a/src/adapter/outbound/identifier/README.md +++ b/src/adapter/outbound/identifier/README.md @@ -37,7 +37,8 @@ leaf에 둔다. 그때 이 절을 고친다. 만든다. operation id는 제출 이벤트의 event id가 되고, messaging platform `MessageId`는 UUIDv7만 받는다. 예전에는 web controller가 `UUID.randomUUID()`(v4)로 만들어 모든 제출 이벤트가 발행 시점에 거부되어 DEAD가 됐다(리뷰 2026-11-01 06-06, Kafka relay lane이 발견). -`composition:durable-operation`이 bean으로 조립한다. +`composition:durable-operation`이 bean으로 조립하고, `SubmitDurableOperationUseCase`가 제출마다 부른다. 2026-11-13 +전에는 controller가 불러 그 id를 command에 실었다. ## 조립 diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/operation/JpaDurableOperationStore.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/operation/JpaDurableOperationStore.java index 44caf0a3..b6fd27c0 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/operation/JpaDurableOperationStore.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/operation/JpaDurableOperationStore.java @@ -46,8 +46,8 @@ public class JpaDurableOperationStore implements DurableOperationStorePort { // Insert-or-nothing on the scope, then read whichever row holds it. Catching the unique // violation instead does not work on PostgreSQL: the violation aborts the caller's transaction // (25P02), and the read of the winning row is refused along with everything after it. The - // inserted count is the only thing that tells a retry under the same operation id from a - // creation. + // inserted count is the only thing that tells a resubmission, which finds the scope already + // held by an earlier call's operation, from a creation. int inserted = repository.submitIfAbsent( submission.operationId().value(), diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSubmissionOutboxIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSubmissionOutboxIntegrationTest.java index 55db6eb9..77366f9f 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSubmissionOutboxIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSubmissionOutboxIntegrationTest.java @@ -31,6 +31,7 @@ import java.time.ZoneOffset; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import javax.sql.DataSource; import org.flywaydb.core.Flyway; import org.junit.jupiter.api.AfterAll; @@ -151,10 +152,9 @@ class DurableOperationSubmissionOutboxIntegrationTest { private static OperationCommand command() { return new OperationCommand( - ID, "reports.generate", "alice", - null, + Optional.empty(), new RequestFingerprint("a".repeat(64)), "{\"range\":\"2026-08\"}", Duration.ofDays(1)); @@ -224,6 +224,9 @@ class DurableOperationSubmissionOutboxIntegrationTest { new JpaOutboxStore(outbox, (now, limit) -> List.of()), new JsonIntegrationEventEncoder(List.of(new DurableOperationSubmittedContract())), transactions, + // One id every time: a resubmission's minted id is discarded, so the tests read the + // first submission's id back whichever call they make. + () -> ID, Clock.fixed(NOW, ZoneOffset.UTC)); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyClaimAttempt.java b/src/application/core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyClaimAttempt.java index aeff40dc..35885ba0 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyClaimAttempt.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyClaimAttempt.java @@ -13,6 +13,10 @@ import java.util.regex.Pattern; * until the record expired (round6 RR6-02-01, {@code IdempotencyStoreContract}). One attempt is one * sender at a time: a handle from before a takeover is stale, but two calls sending the same * attempt at once are not told apart — retry after the call has returned, not beside it. + * + *

The owner token is the store's: 32 secure-random bytes as 64 lower-case hex characters, which + * both stores compare as a fixed-width value. It is not a lease owner token (Base64URL, chosen by + * the lease caller); the transaction {@code OperationId} Javadoc compares the identities. */ public record IdempotencyClaimAttempt(String ownerToken, OperationId operationId) { diff --git a/src/application/core/src/main/java/dev/caskeleton/application/lease/LeaseValidation.java b/src/application/core/src/main/java/dev/caskeleton/application/lease/LeaseValidation.java index 4618c613..6d32894d 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/lease/LeaseValidation.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/lease/LeaseValidation.java @@ -11,6 +11,11 @@ final class LeaseValidation { private LeaseValidation() {} + /** + * A lease's operation id or owner token: 16..128 Base64URL characters, so it fits a header and a + * Redis key without escaping and is long enough to be unguessable. Narrower than the transaction + * {@code OperationId}, whose Javadoc compares the operation-shaped ids and why they differ. + */ static String opaqueToken(String value, String field) { if (value == null || value.length() < 16 diff --git a/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java b/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java index adc509c3..b4958ab6 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java @@ -1,5 +1,9 @@ package dev.caskeleton.application.messaging.event; +import dev.caskeleton.shared.digest.Sha256; +import java.time.Instant; +import java.util.Objects; +import java.util.UUID; import java.util.regex.Pattern; /** @@ -73,6 +77,35 @@ public final class IntegrationIdentity { return value != null && UUID_V7.matcher(value).matches(); } + /** + * A canonical lower-case UUIDv7 derived from {@code value}: the same value and instant always + * give the same id. + * + *

For an event whose id must follow from something that is not itself a UUIDv7 — an operation + * id a client chose, say — so a retry drafts the same event. The time field is {@code at}, so the + * id orders like a minted one; the other 74 bits are the SHA-256 of {@code value}. Kept here, + * with the grammar it must satisfy, rather than in each use case that needs one: the derivation + * is messaging identity, not the use case's own rule. + * + * @param value what the id is derived from + * @param at the instant its time field carries + * @return a canonical lower-case UUIDv7 + */ + public static String uuidV7Derived(String value, Instant at) { + Objects.requireNonNull(value, "value"); + Objects.requireNonNull(at, "at"); + byte[] digest = Sha256.ofUtf8(value).bytes(); + long millis = at.toEpochMilli() & 0xFFFF_FFFF_FFFFL; + long randA = ((digest[0] & 0x0FL) << 8) | (digest[1] & 0xFFL); + long mostSignificant = (millis << 16) | (0x7L << 12) | randA; + long randB = 0; + for (int i = 2; i < 10; i++) { + randB = (randB << 8) | (digest[i] & 0xFFL); + } + long leastSignificant = (randB & 0x3FFF_FFFF_FFFF_FFFFL) | 0x8000_0000_0000_0000L; + return new UUID(mostSignificant, leastSignificant).toString(); + } + /** Refuses a correlation or causation id outside the grammar or the bound. */ static void requireCorrelation(String field, String value) { if (!isCanonical(value, CORRELATION_MAX_LENGTH)) { diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationCommand.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationCommand.java index aa9fc88e..6b4e0a6b 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationCommand.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationCommand.java @@ -1,19 +1,23 @@ package dev.caskeleton.application.operation; import dev.caskeleton.application.command.Command; +import dev.caskeleton.application.tenant.TenantId; import java.util.Objects; +import java.util.Optional; /** * A caller asking to cancel one operation. * * @param operationId the identity to cancel * @param principal the authenticated caller, or null when the caller is not authenticated - * @param tenantId the caller's tenant, or null when the request is not tenant scoped + * @param tenant the caller's tenant, empty when the request is not tenant scoped */ public record CancelDurableOperationCommand( - DurableOperationId operationId, String principal, String tenantId) implements Command { + DurableOperationId operationId, String principal, Optional tenant) + implements Command { public CancelDurableOperationCommand { Objects.requireNonNull(operationId, "operationId"); + Objects.requireNonNull(tenant, "tenant"); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationUseCase.java index 6c3c2449..9aa35d16 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationUseCase.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationUseCase.java @@ -63,7 +63,7 @@ public class CancelDurableOperationUseCase .filter( operation -> DurableOperationAccessPolicy.mayAccess( - operation, command.principal(), command.tenantId())) + operation, command.principal(), command.tenant())) .map(visible -> operations.cancel(command.operationId(), command.principal(), now)); }); } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationAccessPolicy.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationAccessPolicy.java index 6674f707..3e799bf6 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationAccessPolicy.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationAccessPolicy.java @@ -1,6 +1,8 @@ package dev.caskeleton.application.operation; +import dev.caskeleton.application.tenant.TenantId; import java.util.Objects; +import java.util.Optional; /** * Who may read or cancel an operation: the principal that submitted it, in the tenant it was @@ -20,10 +22,21 @@ final class DurableOperationAccessPolicy { * * @param operation the stored operation * @param principal the authenticated caller, or null when the caller is not authenticated - * @param tenantId the caller's tenant, or null when the request is not tenant scoped + * @param tenant the caller's tenant, empty when the request is not tenant scoped */ - static boolean mayAccess(DurableOperation operation, String principal, String tenantId) { + static boolean mayAccess( + DurableOperation operation, String principal, Optional tenant) { return operation.principal().equals(principal) - && Objects.equals(operation.tenantId(), tenantId); + && Objects.equals(operation.tenantId(), storedForm(tenant)); + } + + /** + * The tenant as the operation row holds it: its value, or null when not tenant scoped. + * + * @param tenant the caller's tenant + * @return the stored form + */ + static String storedForm(Optional tenant) { + return tenant.map(TenantId::value).orElse(null); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationId.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationId.java index 573e0135..f8c85981 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationId.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationId.java @@ -10,9 +10,10 @@ import java.util.regex.Pattern; */ public record DurableOperationId(String value) { - // Bounded and free of anything that needs escaping. The identifier travels to whatever transport - // published the operation and is handed back to callers, so a value that one encoding mangles is - // a defect that only appears for some clients. + // Bounded and free of anything that needs escaping: RFC 3986 unreserved characters, because the + // identifier is a URL path segment (/operations/{id}) and is handed back to callers, so a value + // that one encoding mangles is a defect that only appears for some clients. Narrower than the + // transaction OperationId for that reason (its Javadoc compares the operation-shaped ids). private static final Pattern GRAMMAR = Pattern.compile("[A-Za-z0-9._~-]{1,128}"); public DurableOperationId { diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java index 66c718b2..887f5c72 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java @@ -27,8 +27,8 @@ public interface DurableOperationStorePort { * rather than a second one is what makes a resubmission safe while the first is still running. * * @param submission what was accepted - * @return the operation, and whether this call created it; a retry under the same operation id is - * not a creation + * @return the operation, and whether this call created it; a resubmission that finds the + * operation an earlier call created, under its own fresh id, is not a creation */ DurableOperationSubmitResult submit(DurableOperationSubmission submission); diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationSubmitResult.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationSubmitResult.java index 1d43820a..2c0edd5d 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationSubmitResult.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationSubmitResult.java @@ -5,10 +5,10 @@ import java.util.Objects; /** * What a store did with a submission. * - *

{@code created} comes from the store's own write, not from comparing identifiers afterwards. A - * retry under the same operation id finds the operation it created itself, and an identifier - * comparison cannot tell that retry from the first call — which is how a retry appended its outbox - * event a second time. + *

{@code created} comes from the store's own write, not from comparing identifiers afterwards. + * Every call mints a fresh operation id, and a resubmission is found by scope and fingerprint as + * the operation the first call created; only the store's write tells that resubmission from the + * first call, which is what keeps it from appending its outbox event a second time. * * @param operation the operation holding the submission's scope * @param created whether this call inserted it diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationQuery.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationQuery.java index e6ce2ae6..afb85f8f 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationQuery.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationQuery.java @@ -1,19 +1,22 @@ package dev.caskeleton.application.operation; import dev.caskeleton.application.query.Query; +import dev.caskeleton.application.tenant.TenantId; import java.util.Objects; +import java.util.Optional; /** * A caller asking to read one operation. * * @param operationId the identity being polled * @param principal the authenticated caller, or null when the caller is not authenticated - * @param tenantId the caller's tenant, or null when the request is not tenant scoped + * @param tenant the caller's tenant, empty when the request is not tenant scoped */ public record GetDurableOperationQuery( - DurableOperationId operationId, String principal, String tenantId) implements Query { + DurableOperationId operationId, String principal, Optional tenant) implements Query { public GetDurableOperationQuery { Objects.requireNonNull(operationId, "operationId"); + Objects.requireNonNull(tenant, "tenant"); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationUseCase.java index 3fb1bfed..4a66ef81 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationUseCase.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationUseCase.java @@ -15,7 +15,7 @@ import java.util.Optional; transactionMode = TransactionMode.READ_ONLY, idempotency = Idempotency.IDEMPOTENT, repositoryAccess = RepositoryAccess.READ_REPOSITORY) -public final class GetDurableOperationUseCase +public class GetDurableOperationUseCase implements QueryUseCase> { private final DurableOperationStorePort operations; @@ -47,6 +47,6 @@ public final class GetDurableOperationUseCase .filter( operation -> DurableOperationAccessPolicy.mayAccess( - operation, query.principal(), query.tenantId())); + operation, query.principal(), query.tenant())); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationCommand.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationCommand.java index 0d401654..d56f22bb 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationCommand.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationCommand.java @@ -2,34 +2,34 @@ package dev.caskeleton.application.operation; import dev.caskeleton.application.command.Command; import dev.caskeleton.application.idempotency.RequestFingerprint; +import dev.caskeleton.application.tenant.TenantId; import java.time.Duration; import java.util.Objects; +import java.util.Optional; /** * A request to run work asynchronously. * - *

The caller supplies the operation id rather than the store minting one. That is what lets the - * transport hand back a {@code Location} it already knows, and it makes the whole submission one - * value the caller can retry: a generated id would differ on every attempt, so a retry after an - * ambiguous failure would create a second operation. + *

No operation id: {@link SubmitDurableOperationUseCase} mints one with {@link + * DurableOperationIdFactory}, and a resubmission of the same work by the same caller is recognised + * by its scope and payload fingerprint, not by an id the caller kept. The transport used to mint it + * and hand it in (round9 review 2026-11-12, RR9-05-13). * *

There is no event type to choose. Every accepted operation announces itself with the one * {@link DurableOperationSubmittedContract} event, which only wakes a worker; letting a caller name * the type would let it publish any event the broker carries. * - * @param operationId the identity to use * @param operationName what work this is * @param principal who is asking - * @param tenantId the tenant, or null when the operation is not tenant scoped + * @param tenant the tenant, empty when the operation is not tenant scoped * @param fingerprint the semantic fingerprint of the request * @param payload the serialized request a worker will execute * @param timeToLive how long the record and its result stay available */ public record OperationCommand( - DurableOperationId operationId, String operationName, String principal, - String tenantId, + Optional tenant, RequestFingerprint fingerprint, String payload, Duration timeToLive) @@ -39,8 +39,8 @@ public record OperationCommand( public static final Duration DEFAULT_TIME_TO_LIVE = Duration.ofDays(1); public OperationCommand { - Objects.requireNonNull(operationId, "operationId"); Objects.requireNonNull(operationName, "operationName"); + Objects.requireNonNull(tenant, "tenant"); Objects.requireNonNull(fingerprint, "fingerprint"); Objects.requireNonNull(payload, "payload"); Objects.requireNonNull(timeToLive, "timeToLive"); diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java index e5d8719c..45471cef 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java @@ -15,12 +15,10 @@ import dev.caskeleton.application.security.RequiresPermission; import dev.caskeleton.application.transaction.TransactionMode; import dev.caskeleton.application.transaction.TransactionPort; import dev.caskeleton.application.usecase.CommandUseCase; -import dev.caskeleton.shared.digest.Sha256; import java.time.Clock; import java.time.Instant; import java.util.Objects; import java.util.Optional; -import java.util.UUID; /** * Accepts long-running work: one transaction that writes the operation row and the outbox row. @@ -49,8 +47,9 @@ import java.util.UUID; @RequiresPermission("operation:submit") @UseCaseCapability( transactionMode = TransactionMode.WRITE, - // KEYED, not IDEMPOTENT. Re-running this use case is only safe because the caller supplies the - // operation id and the fingerprint scope; without a key it would accept the same work twice. + // KEYED, not IDEMPOTENT. Re-running this use case is only safe because the store recognises the + // submission scope and the payload fingerprint; without that key it would accept the same work + // twice. idempotency = Idempotency.KEYED, repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, // False, and load-bearing. Nothing here reaches the broker: the outbox row is the publish @@ -72,6 +71,7 @@ public class SubmitDurableOperationUseCase private final OutboxAppendPort outbox; private final IntegrationEventEncoderPort encoder; private final TransactionPort transactions; + private final DurableOperationIdFactory operationIds; private final Clock clock; /** @@ -82,6 +82,7 @@ public class SubmitDurableOperationUseCase * @param encoder turns the submission event into the validated event the outbox stores; it must * know {@link DurableOperationSubmittedContract} * @param transactions opens the write transaction both share + * @param operationIds mints the id of a new submission * @param clock the submission clock */ public SubmitDurableOperationUseCase( @@ -89,8 +90,10 @@ public class SubmitDurableOperationUseCase OutboxAppendPort outbox, IntegrationEventEncoderPort encoder, TransactionPort transactions, + DurableOperationIdFactory operationIds, Clock clock) { this.operations = Objects.requireNonNull(operations, "operations"); + this.operationIds = Objects.requireNonNull(operationIds, "operationIds"); this.outbox = Objects.requireNonNull(outbox, "outbox"); this.encoder = Objects.requireNonNull(encoder, "encoder"); this.transactions = Objects.requireNonNull(transactions, "transactions"); @@ -110,18 +113,20 @@ public class SubmitDurableOperationUseCase () -> { DurableOperationSubmission submission = new DurableOperationSubmission( - command.operationId(), + // Minted per call. A resubmission's id is never stored: the store answers with + // the operation the first submission created. + operationIds.newOperationId(), command.operationName(), command.principal(), - command.tenantId(), + DurableOperationAccessPolicy.storedForm(command.tenant()), command.fingerprint(), command.payload(), clock.instant(), command.timeToLive()); DurableOperationSubmitResult submitted = operations.submit(submission); DurableOperation operation = submitted.operation(); - // The store's answer, not an id comparison: a retry under the same operation id finds - // the operation it created itself, and the ids match. + // The store's answer, not an id comparison: every call mints a fresh id, and a retry of + // the same submission is found by the store as the operation the first call created. boolean alreadyAccepted = !submitted.created(); if (!alreadyAccepted) { @@ -147,8 +152,8 @@ public class SubmitDurableOperationUseCase // Single-tenant scope even when the command names a tenant: the event only wakes a worker, the // tenant travels in the stored operation, and mapping a TenantId into the lower-case scope // grammar is the messaging platform's decision, not this use case's. - // No request correlation: the event correlates to the operation itself, so a retry under the - // same operation id produces the same event. + // No request correlation: the event correlates to the operation itself, so a retry that the + // store resolves to the same operation produces the same event. return IntegrationEventDrafts.draft( CONTRACT, new EventId(identity), @@ -160,15 +165,15 @@ public class SubmitDurableOperationUseCase } /** - * The operation id itself when it is a canonical lower-case UUIDv7 — every id the HTTP adapter - * mints is — and otherwise a UUIDv7 derived from it. + * The operation id itself when it is a canonical lower-case UUIDv7 — every id the shipped {@code + * DurableOperationIdFactory} mints is — and otherwise a UUIDv7 derived from it. * *

The event id is the messaging platform's message id, which is a canonical lower-case UUIDv7; - * the draft refuses anything else, so an operation id a client chose ({@code ~report}, a v4, an - * upper-case spelling) cannot be the event id. The derived id keeps the event deterministic for - * the operation — the same id and submission time always give the same event id — and the payload - * carries the operation id unchanged for the worker. Its time field is the submission time, so it - * orders like a minted one; its other 74 bits are the SHA-256 of the operation id. + * the draft refuses anything else, so an id a deployment's own factory mints ({@code ~report}, a + * v4, an upper-case spelling) cannot be the event id. The derived id keeps the event + * deterministic for the operation — the same id and submission time always give the same event id + * — and the payload carries the operation id unchanged for the worker. The derivation is {@link + * IntegrationIdentity#uuidV7Derived}, with the submission time as its time field. * *

It replaced an {@code operation-} digest, which fit the stored identity grammar and * passed the append, and which the bridge then refused, parking the committed event (review @@ -179,15 +184,6 @@ public class SubmitDurableOperationUseCase if (IntegrationIdentity.isCanonicalUuidV7(value)) { return value; } - byte[] digest = Sha256.ofUtf8(value).bytes(); - long millis = submittedAt.toEpochMilli() & 0xFFFF_FFFF_FFFFL; - long randA = ((digest[0] & 0x0FL) << 8) | (digest[1] & 0xFFL); - long mostSignificant = (millis << 16) | (0x7L << 12) | randA; - long randB = 0; - for (int i = 2; i < 10; i++) { - randB = (randB << 8) | (digest[i] & 0xFFL); - } - long leastSignificant = (randB & 0x3FFF_FFFF_FFFF_FFFFL) | 0x8000_0000_0000_0000L; - return new UUID(mostSignificant, leastSignificant).toString(); + return IntegrationIdentity.uuidV7Derived(value, submittedAt); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java index 68ae05cf..0425030a 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java @@ -40,7 +40,7 @@ import java.util.function.Supplier; idempotency = Idempotency.IDEMPOTENT, repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, externalOutboundAllowed = true) -public final class PublishPendingOutboxEventsUseCase +public class PublishPendingOutboxEventsUseCase implements CommandUseCase { /** The failure code of a thrown failure whose type has no simple name. */ diff --git a/src/application/core/src/main/java/dev/caskeleton/application/transaction/OperationId.java b/src/application/core/src/main/java/dev/caskeleton/application/transaction/OperationId.java index f9704f17..6e230d94 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/transaction/OperationId.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/transaction/OperationId.java @@ -7,6 +7,15 @@ import java.util.Objects; * *

The value is intentionally opaque. Adapters may use it for reconciliation, but must not invent * a replacement identity after an uncertain commit. + * + *

The widest of the operation-shaped identities, on purpose: the caller chooses it and it is + * only ever compared, never put in a URL or a key, so any printable ASCII is allowed. The others + * are narrower because of where they travel — a {@link + * dev.caskeleton.application.operation.DurableOperationId} is a URL path segment, a lease's + * operation id and owner token are Base64URL so they fit a header and a Redis key unescaped, and an + * idempotency owner token is the 32 random bytes the store mints, as 64 hex characters. One value + * type for all four would either let a path segment carry {@code /} or refuse caller ids that are + * valid today (round9 review 2026-11-12, RR9-05-11). */ public record OperationId(String value) { diff --git a/src/application/core/src/main/java/dev/caskeleton/application/transaction/ReadConsistency.java b/src/application/core/src/main/java/dev/caskeleton/application/transaction/ReadConsistency.java deleted file mode 100644 index 23912c34..00000000 --- a/src/application/core/src/main/java/dev/caskeleton/application/transaction/ReadConsistency.java +++ /dev/null @@ -1,9 +0,0 @@ -package dev.caskeleton.application.transaction; - -/** Application-owned read consistency vocabulary. */ -public enum ReadConsistency { - STRONG, - READ_YOUR_WRITES, - EVENTUAL, - BOUNDED_STALENESS -} diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java index 8c845cac..69219b0a 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.application.idempotency.RequestFingerprint; import dev.caskeleton.application.operation.testkit.InMemoryDurableOperationStore; +import dev.caskeleton.application.tenant.TenantId; import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; import java.time.Clock; import java.time.Duration; @@ -28,7 +29,8 @@ class CancelDurableOperationUseCaseTest { operations.submit(submission("t-1")); // when - useCase.handle(new CancelDurableOperationCommand(ID, "alice", "t-1")); + useCase.handle( + new CancelDurableOperationCommand(ID, "alice", Optional.of(new TenantId("t-1")))); // then assertThat(operations.find(ID, "alice", NOW).orElseThrow().state()) @@ -45,7 +47,7 @@ class CancelDurableOperationUseCaseTest { // when Optional canceled = - useCase.handle(new CancelDurableOperationCommand(ID, "alice", null)); + useCase.handle(new CancelDurableOperationCommand(ID, "alice", Optional.empty())); // then assertThat(canceled).contains(false); @@ -58,7 +60,8 @@ class CancelDurableOperationUseCaseTest { // when Optional canceled = - useCase.handle(new CancelDurableOperationCommand(ID, "alice", "t-2")); + useCase.handle( + new CancelDurableOperationCommand(ID, "alice", Optional.of(new TenantId("t-2")))); // then assertThat(canceled).isEmpty(); @@ -70,7 +73,8 @@ class CancelDurableOperationUseCaseTest { operations.submit(submission(null)); // when - Optional canceled = useCase.handle(new CancelDurableOperationCommand(ID, null, null)); + Optional canceled = + useCase.handle(new CancelDurableOperationCommand(ID, null, Optional.empty())); // then assertThat(canceled).isEmpty(); diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/DurableOperationAccessPolicyTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/DurableOperationAccessPolicyTest.java index 44c66954..7be18cc6 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/operation/DurableOperationAccessPolicyTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/DurableOperationAccessPolicyTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.application.operation; import static org.assertj.core.api.Assertions.assertThat; +import dev.caskeleton.application.tenant.TenantId; import java.time.Instant; import java.util.Optional; import org.junit.jupiter.params.ParameterizedTest; @@ -33,7 +34,7 @@ class DurableOperationAccessPolicyTest { String owner, String ownerTenant, String caller, String callerTenant, boolean expected) { assertThat( DurableOperationAccessPolicy.mayAccess( - operation(owner, ownerTenant), caller, callerTenant)) + operation(owner, ownerTenant), caller, TenantId.optional(callerTenant))) .isEqualTo(expected); } diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/GetDurableOperationUseCaseTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/GetDurableOperationUseCaseTest.java index 292cce01..3cb19a13 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/operation/GetDurableOperationUseCaseTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/GetDurableOperationUseCaseTest.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.application.idempotency.RequestFingerprint; import dev.caskeleton.application.operation.testkit.InMemoryDurableOperationStore; +import dev.caskeleton.application.tenant.TenantId; import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; import java.time.Clock; import java.time.Duration; @@ -29,7 +30,7 @@ class GetDurableOperationUseCaseTest { // when Optional read = - useCase.handle(new GetDurableOperationQuery(ID, "alice", "t-1")); + useCase.handle(new GetDurableOperationQuery(ID, "alice", Optional.of(new TenantId("t-1")))); // then assertThat(read).contains(submitted); @@ -42,7 +43,7 @@ class GetDurableOperationUseCaseTest { // when Optional read = - useCase.handle(new GetDurableOperationQuery(ID, "alice", "t-2")); + useCase.handle(new GetDurableOperationQuery(ID, "alice", Optional.of(new TenantId("t-2")))); // then assertThat(read).isEmpty(); @@ -54,7 +55,8 @@ class GetDurableOperationUseCaseTest { operations.submit(submission(null)); // when - Optional read = useCase.handle(new GetDurableOperationQuery(ID, null, null)); + Optional read = + useCase.handle(new GetDurableOperationQuery(ID, null, Optional.empty())); // then assertThat(read).isEmpty(); diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java index ac469e2d..9053cfab 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java @@ -16,7 +16,10 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.List; +import java.util.Optional; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -34,7 +37,7 @@ class SubmitDurableOperationUseCaseTest { private static final Instant NOW = Instant.parse("2026-08-25T10:00:00Z"); - /** An id the HTTP adapter mints: a canonical lower-case UUIDv7, used as the event id itself. */ + /** An id the factory mints: a canonical lower-case UUIDv7, used as the event id itself. */ private static final String OPERATION_UUID = "01994e11-4d88-7000-8000-0000000000a1"; private static final DurableOperationId ID = new DurableOperationId(OPERATION_UUID); @@ -48,16 +51,25 @@ class SubmitDurableOperationUseCaseTest { private final FailingOutbox outbox = new FailingOutbox(transactions); private final FakeIntegrationEventEncoder encoder = new FakeIntegrationEventEncoder(new DurableOperationSubmittedContract()); + + /** What the id factory mints, in order: one id per submission, kept or not. */ + private final Deque mintable = + new ArrayDeque<>(List.of(ID, new DurableOperationId("op-2"), new DurableOperationId("op-3"))); + private final CommandUseCase useCase = new SubmitDurableOperationUseCase( - operations, outbox, encoder, transactions, Clock.fixed(NOW, ZoneOffset.UTC)); + operations, + outbox, + encoder, + transactions, + mintable::removeFirst, + Clock.fixed(NOW, ZoneOffset.UTC)); - private static OperationCommand command(DurableOperationId id, String hash) { + private static OperationCommand command(String hash) { return new OperationCommand( - id, "reports.generate", "alice", - null, + Optional.empty(), new RequestFingerprint(hash), "{\"range\":\"2026-08\"}", Duration.ofHours(6)); @@ -66,7 +78,7 @@ class SubmitDurableOperationUseCaseTest { @Test @DisplayName("an accepted submission writes both the operation and the outbox event") void submissionWritesBoth() { - OperationAcceptance acceptance = useCase.handle(command(ID, "a".repeat(64))); + OperationAcceptance acceptance = useCase.handle(command("a".repeat(64))); assertThat(acceptance.operationId()).isEqualTo(ID); assertThat(acceptance.alreadyAccepted()).isFalse(); @@ -74,10 +86,20 @@ class SubmitDurableOperationUseCaseTest { assertThat(outbox.appended()).hasSize(1); } + @Test + @DisplayName("operation id는 호출자가 아니라 use case가 id factory로 만든다") + void theOperationIdIsMintedByTheUseCase() { + // when + OperationAcceptance acceptance = useCase.handle(command("d".repeat(64))); + + // then + assertThat(List.of(acceptance.operationId(), mintable.size())).containsExactly(ID, 2); + } + @Test @DisplayName("both writes happen inside one transaction") void bothWritesShareOneTransaction() { - useCase.handle(command(ID, "a".repeat(64))); + useCase.handle(command("a".repeat(64))); // Not an implementation detail. Outside one transaction the two writes can commit // independently, and every failure mode this use case exists to prevent comes back. The @@ -92,7 +114,7 @@ class SubmitDurableOperationUseCaseTest { void rollbackLeavesNothing() { outbox.failNext = true; - assertThatThrownBy(() -> useCase.handle(command(ID, "a".repeat(64)))) + assertThatThrownBy(() -> useCase.handle(command("a".repeat(64)))) .isInstanceOf(IllegalStateException.class); assertThat(operations.size()).isZero(); @@ -102,10 +124,9 @@ class SubmitDurableOperationUseCaseTest { @Test @DisplayName("an identical resubmission returns the first operation and publishes nothing new") void resubmissionReusesTheFirstOperation() { - useCase.handle(command(ID, "a".repeat(64))); + useCase.handle(command("a".repeat(64))); - OperationAcceptance repeat = - useCase.handle(command(new DurableOperationId("op-2"), "a".repeat(64))); + OperationAcceptance repeat = useCase.handle(command("a".repeat(64))); // The client is pointed at the work that is actually running. Starting a second operation // would run the same report twice and hand back an id for the copy. @@ -116,12 +137,12 @@ class SubmitDurableOperationUseCaseTest { } @Test - void retryUnderTheSameOperationIdAppendsTheOutboxEventOnlyOnce() { + void aRetriedSubmissionAppendsTheOutboxEventOnlyOnce() { // given - useCase.handle(command(ID, "a".repeat(64))); + useCase.handle(command("a".repeat(64))); // when - OperationAcceptance retry = useCase.handle(command(ID, "a".repeat(64))); + OperationAcceptance retry = useCase.handle(command("a".repeat(64))); // then assertThat(List.of(retry.alreadyAccepted(), outbox.appended().size())).containsExactly(true, 1); @@ -130,10 +151,9 @@ class SubmitDurableOperationUseCaseTest { @Test @DisplayName("a different request under the same principal is a different operation") void differentRequestsAreDifferentOperations() { - useCase.handle(command(ID, "a".repeat(64))); + useCase.handle(command("a".repeat(64))); - OperationAcceptance second = - useCase.handle(command(new DurableOperationId("op-2"), "b".repeat(64))); + OperationAcceptance second = useCase.handle(command("b".repeat(64))); assertThat(second.alreadyAccepted()).isFalse(); assertThat(operations.size()).isEqualTo(2); @@ -143,7 +163,7 @@ class SubmitDurableOperationUseCaseTest { @Test void theOutboxEventIsTheSubmittedContractAnchoredToTheOperation() { // given - useCase.handle(command(ID, "a".repeat(64))); + useCase.handle(command("a".repeat(64))); // when ValidatedIntegrationEvent event = outbox.appended().get(0); @@ -167,7 +187,7 @@ class SubmitDurableOperationUseCaseTest { @Test void theSubmittedEventNamesTheOperationAndNotTheRequest() { // when - useCase.handle(command(ID, "a".repeat(64))); + useCase.handle(command("a".repeat(64))); // then // The worker reads what to run from the submission it claims; the request never travels. @@ -200,13 +220,13 @@ class SubmitDurableOperationUseCaseTest { } @Test - @DisplayName("UUIDv7이 아닌 operation id로 받은 작업도 outbox에 발행할 수 있는 event로 남는다") - void aSubmissionUnderAClientChosenIdStillAppendsAPublishableEvent() { + @DisplayName("id factory가 UUIDv7이 아닌 id를 만들어도 outbox에 발행할 수 있는 event로 남는다") + void aSubmissionUnderANonUuidV7IdStillAppendsAPublishableEvent() { // given - DurableOperationId clientChosen = new DurableOperationId("~report"); + mintable.addFirst(new DurableOperationId("~report")); // when - useCase.handle(command(clientChosen, "c".repeat(64))); + useCase.handle(command("c".repeat(64))); // then assertThat(outbox.appended()) diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/FileMetadataPort.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/FileMetadataPort.java index 1b5e7b0d..fa5ffde3 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/FileMetadataPort.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/FileMetadataPort.java @@ -10,14 +10,36 @@ import java.util.Optional; * Authoritative file metadata port. * *

Every transition is conditional on both the expected version and the expected state, so two - * writers racing on the same record produce exactly one winner and one optimistic conflict. + * writers racing on the same record produce exactly one winner and one optimistic conflict. How + * each write fails is part of the contract, because the use cases decide from the failure whether + * to retry, report a conflict or treat the file as gone. A record that does not exist fails as a + * lost precondition, like one that moved on. */ public interface FileMetadataPort { + /** + * Records a new file in {@code CREATED}. + * + * @return the stored record, version 0 + * @throws RuntimeException the store's own refusal when a record already exists under {@code + * draft.fileId()}; file ids are server-minted, so that is a defect, not a race to recover + * from + */ FileRecord insert(FileRecordDraft draft); Optional find(FileId fileId); + /** + * Moves a record from {@code expectedState} at {@code expectedVersion} to {@code targetState}. + * + * @return the record as stored after the move + * @throws IllegalStateException when {@code expectedState} may never move to {@code targetState} + * — a caller defect, refused before anything is written + * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code + * CONCURRENT_MODIFICATION} when the record is no longer at that version and state + * @throws dev.caskeleton.application.fileserver.api.error.FileNotFoundException with {@code + * FILE_NOT_FOUND} when the record disappeared during the move + */ FileRecord transition( FileId fileId, long expectedVersion, @@ -25,6 +47,16 @@ public interface FileMetadataPort { FileState targetState, FileRecordMutation mutation); + /** + * Moves a record at {@code expectedVersion} in a deletable state to {@code DELETING}. + * + * @return the record as stored after the move + * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code + * CONCURRENT_MODIFICATION} when the record is no longer at that version or not in a deletable + * state + * @throws dev.caskeleton.application.fileserver.api.error.FileNotFoundException with {@code + * FILE_NOT_FOUND} when the record disappeared during the move + */ FileRecord markDeleting(FileId fileId, long expectedVersion); /** @@ -33,6 +65,12 @@ public interface FileMetadataPort { *

A namespace is a metadata grouping, so this changes one column and never touches the * immutable physical object. Copying gigabytes to express an ownership change would also break * every strong validator already handed to clients. + * + * @return the record as stored after the move + * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code + * CONCURRENT_MODIFICATION} when the record is no longer {@code READY} at that version + * @throws dev.caskeleton.application.fileserver.api.error.FileNotFoundException with {@code + * FILE_NOT_FOUND} when the record disappeared during the move */ FileRecord relocate(FileId fileId, long expectedVersion, StorageNamespace targetNamespace); diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/UploadSessionPort.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/UploadSessionPort.java index 9116e18f..42062eeb 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/UploadSessionPort.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/UploadSessionPort.java @@ -18,6 +18,14 @@ public interface UploadSessionPort { Optional find(UploadId uploadId); + /** + * Takes the writer lease on an upload that holds none, or whose lease has lapsed. + * + * @return the new lease, with a fresh token and the session version after the claim + * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code + * CONCURRENT_MODIFICATION} when another owner holds an unexpired lease, the session is no + * longer at {@code expectedVersion}, or the upload has expired (then not retryable) + */ WriterLease acquireLease( UploadId uploadId, String owner, Instant now, Duration leaseDuration, long expectedVersion); @@ -34,6 +42,15 @@ public interface UploadSessionPort { */ WriterLease renewLease(WriterLease lease, Instant now, Duration leaseDuration); + /** + * Advances the committed offset from {@code expectedOffset} to {@code committedOffset} for the + * holder of {@code lease}. + * + * @return the session as stored after the commit + * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code + * CONCURRENT_MODIFICATION} when the lease token no longer holds the session or the stored + * offset is not {@code expectedOffset}; nothing is committed + */ UploadSession commitOffset( UploadId uploadId, WriterLease lease, long expectedOffset, long committedOffset); diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/observability/FileserverSpans.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/observability/FileserverSpans.java deleted file mode 100644 index ab6fd5df..00000000 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/observability/FileserverSpans.java +++ /dev/null @@ -1,28 +0,0 @@ -package dev.caskeleton.application.fileserver.observability; - -/** - * The exact span names the design fixes. - * - *

Naming them once means a dashboard or an alert built on one of these keeps working when the - * code that emits it moves. A span name invented at a call site would silently break both. - */ -public final class FileserverSpans { - - public static final String UPLOAD_CREATE = "upload.create"; - public static final String UPLOAD_APPEND = "upload.append"; - public static final String UPLOAD_FINALIZE = "upload.finalize"; - public static final String VERIFY_DIGEST = "verify.digest"; - public static final String VERIFY_MEDIA_TYPE = "verify.media-type"; - public static final String VERIFY_MALWARE = "verify.malware"; - public static final String STORAGE_PUBLISH = "storage.publish"; - public static final String STORAGE_STAT = "storage.stat"; - public static final String METADATA_TRANSITION = "metadata.transition"; - public static final String DOWNLOAD_AUTHORIZE = "download.authorize"; - public static final String DOWNLOAD_RESOLVE_RANGE = "download.resolve-range"; - public static final String DOWNLOAD_OPEN = "download.open"; - public static final String DOWNLOAD_DELEGATE = "download.delegate"; - public static final String CLEANUP_ITEM = "cleanup.item"; - public static final String RECONCILE_FILE = "reconcile.file"; - - private FileserverSpans() {} -} diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/api/security/OriginalFilenamePolicyTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/api/security/OriginalFilenamePolicyTest.java index 05357b59bba9056693e39203e92a7af96a10b53c..0bd7a4c9e3b795bd4d2c1abc1aaa1ffb018b94a3 100644 GIT binary patch delta 19 ZcmeB{o*=!UorNu?)Bp%Jcd#&X0suXj1#bWV delta 14 VcmbOr-7USLorRHMb1w@sCjcUj1Kt1t diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/observability/FileserverObservabilityTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/observability/FileserverObservabilityTest.java index affc9182..6a240a63 100644 --- a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/observability/FileserverObservabilityTest.java +++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/observability/FileserverObservabilityTest.java @@ -76,30 +76,6 @@ class FileserverObservabilityTest { } } - @Test - void everyDesignSpanNameIsDeclaredExactlyOnce() { - List spans = - List.of( - FileserverSpans.UPLOAD_CREATE, - FileserverSpans.UPLOAD_APPEND, - FileserverSpans.UPLOAD_FINALIZE, - FileserverSpans.VERIFY_DIGEST, - FileserverSpans.VERIFY_MEDIA_TYPE, - FileserverSpans.VERIFY_MALWARE, - FileserverSpans.STORAGE_PUBLISH, - FileserverSpans.STORAGE_STAT, - FileserverSpans.METADATA_TRANSITION, - FileserverSpans.DOWNLOAD_AUTHORIZE, - FileserverSpans.DOWNLOAD_RESOLVE_RANGE, - FileserverSpans.DOWNLOAD_OPEN, - FileserverSpans.DOWNLOAD_DELEGATE, - FileserverSpans.CLEANUP_ITEM, - FileserverSpans.RECONCILE_FILE); - - assertThat(spans).doesNotHaveDuplicates().hasSize(15); - assertThat(spans).allSatisfy(name -> assertThat(name).matches("[a-z]+\\.[a-z-]+")); - } - @Test void anAuditEventCarriesFingerprintsRatherThanNames() { SafeFileFingerprint fingerprint = new SafeFileFingerprint(KEY); diff --git a/src/application/notification/AGENTS.md b/src/application/notification/AGENTS.md index 9c163f4d..0dce0cb1 100644 --- a/src/application/notification/AGENTS.md +++ b/src/application/notification/AGENTS.md @@ -12,7 +12,7 @@ ## 책임 - 알림 플랫폼의 업무 흐름: 제출·예약·취소·조회, template 게시, provider callback 수집, 발송 판정과 재시도 정책. -- 들어오는 포트(`platform.port.in`의 `*UseCase`·`*Command`·`*Query`), 나가는 포트(쓰는 흐름의 패키지에 있다: `platform.dispatch`·`platform.callback`·`platform.policy`·`platform.observation`·`platform.admin`·`platform.inbox`·`platform.template`·`platform.provider`·`platform.security`), 오류 어휘(`platform.api.error`). `port.out` 패키지는 따로 없다. 나가는 포트 이름은 대개 `*Port`지만 아닌 것도 있다: `provider.NotificationProviderAdapter`·`BatchNotificationProviderAdapter`·`AttachmentResolver`, `security.SecretMaterialProvider`·`ContactPointProtector`, `template.TemplateRegistry`, `callback.ProviderEventLedger`. 새 포트를 만들기 전에 이 이름들도 찾는다. +- 들어오는 포트(`platform.port.in`의 `*UseCase`·`*Command`·`*Query`), 나가는 포트(쓰는 흐름의 패키지에 있다: `platform.dispatch`·`platform.callback`·`platform.policy`·`platform.observation`·`platform.admin`·`platform.inbox`·`platform.template`·`platform.provider`·`platform.security`), 오류 어휘(`platform.api.error`). `port.out` 패키지는 따로 없다. 나가는 포트 이름은 대개 `*Port`지만 아닌 것도 있다: `provider.NotificationProviderAdapter`·`AttachmentResolver`, `security.SecretMaterialProvider`·`ContactPointProtector`, `template.TemplateRegistry`, `callback.ProviderEventLedger`. 새 포트를 만들기 전에 이 이름들도 찾는다. - testFixtures(흐름 패키지의 `testkit`): 저장 포트의 in-memory fake와 공유 계약(`*Contract`), provider adapter 계약(`ProviderAdapterContract`). 모든 포트에 있지는 않다. adapter의 시험이 같은 계약을 상속한다. ## 규칙 diff --git a/src/application/notification/README.md b/src/application/notification/README.md index 682a0357..b1372fc9 100644 --- a/src/application/notification/README.md +++ b/src/application/notification/README.md @@ -5,7 +5,7 @@ ## 책임과 소유하지 않는 것 - 알림 플랫폼의 업무 흐름: 제출·예약·취소·조회, template 게시, provider callback 수집, 발송 판정과 재시도 정책. -- 들어오는 포트(`platform.port.in`의 `*UseCase`·`*Command`·`*Query`), 나가는 포트(쓰는 흐름의 패키지에 있다: `platform.dispatch`·`platform.callback`·`platform.policy`·`platform.observation`·`platform.admin`·`platform.inbox`·`platform.template`·`platform.provider`·`platform.security`), 오류 어휘(`platform.api.error`). `port.out` 패키지는 따로 없다. 나가는 포트 이름은 대개 `*Port`지만 아닌 것도 있다: `provider.NotificationProviderAdapter`·`BatchNotificationProviderAdapter`·`AttachmentResolver`, `security.SecretMaterialProvider`·`ContactPointProtector`, `template.TemplateRegistry`, `callback.ProviderEventLedger`. 새 포트를 만들기 전에 이 이름들도 찾는다. +- 들어오는 포트(`platform.port.in`의 `*UseCase`·`*Command`·`*Query`), 나가는 포트(쓰는 흐름의 패키지에 있다: `platform.dispatch`·`platform.callback`·`platform.policy`·`platform.observation`·`platform.admin`·`platform.inbox`·`platform.template`·`platform.provider`·`platform.security`), 오류 어휘(`platform.api.error`). `port.out` 패키지는 따로 없다. 나가는 포트 이름은 대개 `*Port`지만 아닌 것도 있다: `provider.NotificationProviderAdapter`·`AttachmentResolver`, `security.SecretMaterialProvider`·`ContactPointProtector`, `template.TemplateRegistry`, `callback.ProviderEventLedger`. 새 포트를 만들기 전에 이 이름들도 찾는다. - testFixtures(흐름 패키지의 `testkit`): 저장 포트의 in-memory fake와 공유 계약(`*Contract`), provider adapter 계약(`ProviderAdapterContract`). 모든 포트에 있지는 않다. adapter의 시험이 같은 계약을 상속한다. - 소유하지 않는 것: HTTP 모양(`adapter:inbound:web-notification`), provider 호출·설정(`adapter:outbound:notification`), 저장(`adapter:outbound:persistence-notification`), 조립(`composition:notification-platform`). diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/ProviderEventId.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/ProviderEventId.java deleted file mode 100644 index 56924a78..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/ProviderEventId.java +++ /dev/null @@ -1,19 +0,0 @@ -package dev.caskeleton.application.notification.platform.api; - -/** - * Provider-assigned event identity. Absent for providers that do not emit one; the ledger then - * falls back to a deterministic fingerprint. - */ -public record ProviderEventId(String value) { - - private static final int MAX_LENGTH = 300; - - public ProviderEventId { - if (value == null || value.isBlank()) { - throw new IllegalArgumentException("value"); - } - if (value.length() > MAX_LENGTH) { - throw new IllegalArgumentException("value exceeds " + MAX_LENGTH + " characters"); - } - } -} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/error/NotificationExpiredException.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/error/NotificationExpiredException.java deleted file mode 100644 index ba926e17..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/error/NotificationExpiredException.java +++ /dev/null @@ -1,11 +0,0 @@ -package dev.caskeleton.application.notification.platform.api.error; - -/** The delivery expiry passed before a new attempt could start. */ -public class NotificationExpiredException extends NotificationException { - - private static final long serialVersionUID = 1L; - - public NotificationExpiredException(NotificationFailureDescriptor descriptor) { - super(descriptor); - } -} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/AcceptNotificationApplicationUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/AcceptNotificationApplicationUseCase.java index d8ed77b3..3107e0ac 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/AcceptNotificationApplicationUseCase.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/AcceptNotificationApplicationUseCase.java @@ -137,6 +137,9 @@ public class AcceptNotificationApplicationUseCase implements AcceptNotificationU command.templateVersion().orElse(NotificationTemplateVersion.FIRST_VERSION), command.locale().orElse(defaultLocale)); CorrelationId correlationId = new CorrelationId(ids.nextId().toString()); + // Not the HTTP Idempotency-Key the executor replays: a business key that makes the + // notification itself one per tenant and lives as long as it does, derived from the plan when + // the caller gave none (manual 05 「executor를 쓰지 않는 중복 판정 둘」). IdempotencyKey idempotencyKey = command .idempotencyKey() diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/CanonicalNotificationPlanEncoder.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/CanonicalNotificationPlanEncoder.java index 9ca2b512219129cc4e67ec0baf1374b2581340fb..0204482ee5ced3dffe8d4ba260aeaade4d17ab2c 100644 GIT binary patch delta 121 zcmZ3NbuDXy7oTEGsR0luDQGJc7o;ZZm1pLqb(j6eHUJ delta 69 zcmcbXwJvLe7at?T8=fP5vk@y;)7io(U+O Styjwg5=u0X+{|I<#{vMEsuAG; diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/BatchNotificationProviderAdapter.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/BatchNotificationProviderAdapter.java deleted file mode 100644 index 43fb9527..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/BatchNotificationProviderAdapter.java +++ /dev/null @@ -1,16 +0,0 @@ -package dev.caskeleton.application.notification.platform.provider; - -import java.util.List; -import java.util.concurrent.CompletionStage; - -/** - * Optional batching extension. - * - *

One transport call may carry many submissions, but the result list is positional: every - * recipient keeps its own attempt and its own outcome. - */ -public interface BatchNotificationProviderAdapter extends NotificationProviderAdapter { - - /** Submit a batch and return one result per input, in input order. */ - CompletionStage> submitBatch(List submissions); -} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/CollapseCapability.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/CollapseCapability.java deleted file mode 100644 index a5ba086c..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/CollapseCapability.java +++ /dev/null @@ -1,10 +0,0 @@ -package dev.caskeleton.application.notification.platform.provider; - -import dev.caskeleton.application.notification.platform.api.CollapseSpec; - -/** Maps a platform collapse hint onto a provider-native key, validating provider limits first. */ -public interface CollapseCapability { - - /** Translate the spec, or fail when the key violates a provider limit. */ - ProviderCollapseMapping map(CollapseSpec spec); -} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/ProviderCollapseMapping.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/ProviderCollapseMapping.java deleted file mode 100644 index 121c940a..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/ProviderCollapseMapping.java +++ /dev/null @@ -1,25 +0,0 @@ -package dev.caskeleton.application.notification.platform.provider; - -import java.util.Objects; - -/** - * Provider-native collapse key. - * - *

{@code guaranteesSingleUserNotification} is always false: collapse replaces an undelivered - * message inside the provider, it cannot recall one the device already showed. - */ -public record ProviderCollapseMapping(String nativeKey, String headerOrFieldName) { - - public ProviderCollapseMapping { - Objects.requireNonNull(nativeKey, "nativeKey"); - Objects.requireNonNull(headerOrFieldName, "headerOrFieldName"); - if (nativeKey.isBlank() || headerOrFieldName.isBlank()) { - throw new IllegalArgumentException("collapse mapping must not be blank"); - } - } - - /** Collapse never promises the user sees exactly one notification. */ - public boolean guaranteesSingleUserNotification() { - return false; - } -} diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/DeduplicationAndCollapseTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/DeduplicationAndCollapseTest.java index ecc03283..904b0a0a 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/DeduplicationAndCollapseTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/DeduplicationAndCollapseTest.java @@ -2,14 +2,10 @@ package dev.caskeleton.application.notification.platform.policy; import static org.assertj.core.api.Assertions.assertThat; -import dev.caskeleton.application.notification.platform.api.CollapseScope; -import dev.caskeleton.application.notification.platform.api.CollapseSpec; import dev.caskeleton.application.notification.platform.api.DeduplicationAction; import dev.caskeleton.application.notification.platform.api.DeduplicationSpec; import dev.caskeleton.application.notification.platform.api.NotificationId; import dev.caskeleton.application.notification.platform.policy.testkit.InMemoryDeduplicationStore; -import dev.caskeleton.application.notification.platform.provider.CollapseCapability; -import dev.caskeleton.application.notification.platform.provider.ProviderCollapseMapping; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -55,14 +51,4 @@ class DeduplicationAndCollapseTest { assertThat(other.duplicate()).isFalse(); } - - @Test - void collapseDoesNotReportExactlyOnceOrDelivered() { - CollapseCapability capability = - spec -> new ProviderCollapseMapping(spec.key(), "apns-collapse-id"); - var mapped = capability.map(new CollapseSpec("feed:user-1", CollapseScope.RECIPIENT)); - - assertThat(mapped.guaranteesSingleUserNotification()).isFalse(); - assertThat(mapped.nativeKey()).isEqualTo("feed:user-1"); - } } diff --git a/src/bootstrap/durable-operation/src/main/java/dev/caskeleton/composition/operation/DurableOperationAutoConfiguration.java b/src/bootstrap/durable-operation/src/main/java/dev/caskeleton/composition/operation/DurableOperationAutoConfiguration.java index cab63f22..c1133358 100644 --- a/src/bootstrap/durable-operation/src/main/java/dev/caskeleton/composition/operation/DurableOperationAutoConfiguration.java +++ b/src/bootstrap/durable-operation/src/main/java/dev/caskeleton/composition/operation/DurableOperationAutoConfiguration.java @@ -71,8 +71,10 @@ public class DurableOperationAutoConfiguration { OutboxAppendPort outbox, IntegrationEventEncoderPort encoder, TransactionPort transactions, + DurableOperationIdFactory operationIds, Clock clock) { - return new SubmitDurableOperationUseCase(operations, outbox, encoder, transactions, clock); + return new SubmitDurableOperationUseCase( + operations, outbox, encoder, transactions, operationIds, clock); } /** Reads one operation for the caller that submitted it. */ From 2d274a6842918ab4db3f64bdf39f710539c08edf Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 13 Nov 2026 16:37:03 +0900 Subject: [PATCH 6/8] =?UTF-8?q?refactor(application)!:=20=EC=8B=9C?= =?UTF-8?q?=EC=8A=A4=ED=85=9C=20=EC=93=B0=EA=B8=B0=EC=99=80=20=EC=9A=B4?= =?UTF-8?q?=EC=98=81=EC=9E=90=20=EC=97=B0=EC=82=B0=EC=9D=84=20=EC=9C=A0?= =?UTF-8?q?=EC=8A=A4=EC=BC=80=EC=9D=B4=EC=8A=A4=20=EA=B3=84=EC=95=BD=20?= =?UTF-8?q?=EC=95=88=EC=9C=BC=EB=A1=9C=20=EC=98=AE=EA=B8=B4=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 시스템이 부르는 쓰기가 유스케이스 계약 밖에 있다. fileserver 정리·맞추기, 알림 claim·발송·lease 회복·맞추기·provider event replay는 capability도 트랜잭션 선언도 없는 서비스를 worker가 바로 부르고, 그 worker와 조립은 outbound adapter 안에 있다. 알림 운영자 연산 넷은 permission 없이 서비스 하나에 모여 있고 재시도 시각은 상수다. web stream bridge는 outbound가 구현한 replay 포트를 바로 읽는다. fileserver 관리 HTTP 기본값은 controller가 정한다. 고른 방법: 시스템 쓰기는 @SystemInvoked 유스케이스로 나누고 composition worker 설정이 new로 만든다(bean 아님). 알림 worker와 fileserver 정리 조립은 composition으로 옮기고, 맞추기의 재시도·포기 시점은 유스케이스가 정한다. 운영자 연산은 platform.admin의 유스케이스 클래스 넷과 notification-admin:* permission이다(port.in 인터페이스는 port.in과 admin의 cycle을 만들어 두지 않는다). replay 읽기는 조회 유스케이스 둘을 거치고 포트는 LiveEventReplayQueryPort로 이름을 바꾼다. 아키텍처 규칙 둘: 사용자 전송은 저장·id 포트를 쥐지 못하고, inbound는 application 서비스·coordinator를 interface로도 쥐지 못한다. 재시도 시각은 설정, 관리 기본값은 command 팩터리다. 재시도 시각의 규칙은 policy의 값 RetryTiming 하나가 갖고 설정은 그 값을 만든다. retry·routing 값들은 서버가 만드는 값이라 잘못된 값을 IllegalStateException으로 거절한다(manual 06). 모듈 기본값이 role notification-admin에 운영자 permission 넷을 준다(fileserver의 ROLE_FILE_ADMIN과 같다). 버린 대안: 서비스에 @UseCaseCapability만 달기(사용자와 시스템이 같은 진입점을 나눠 써서 권한 규칙이 막지 못한다), 운영자 서비스 삭제, 설정에서 policy를 만들어 버리며 검증하기(규칙이 바뀐 policy bean과 상관없이 남는다). 위험·한계: 공개 타입·생성자·bean 이름·아키텍처 규칙이 바뀐다. 파생 서비스가 할 일은 docs/upgrade/2026-11-round9-application.md 1~3·5·6행이다. claim·fence 커밋 위에 쌓았다. 검증: PR 본문의 desktop 결과. Refs: RR9-05-06, RR9-05-07, RR9-05-08, RR9-05-10, RR9-05-19 BREAKING CHANGE: NotificationAdminService·NotificationAdminApplicationService가 없어지고 운영자 연산은 platform.admin의 RedriveDeliveryUseCase 등이다(permission notification-admin:*). 알림 worker가 composition:notification-platform으로 옮기고 ReconciliationJobWorker·ProviderEventReplayWorker는 유스케이스가 됐다. MessagingReplayBridge 생성자가 조회 유스케이스를 받고 LiveEventReplayPort는 LiveEventReplayQueryPort다. 사용자 전송이 저장·id 포트나 application 서비스 interface를 쥐면 architectureTest가 실패한다. 운영자 요청 record 이름이 *Request로 바뀌고 DefaultNotificationRetryPolicy 생성자가 RetryTiming을 받는다. RetryBackoff·RetryBudget·RetryContext·RetryDecision.RetryAfter·RouteCandidate·RoutingContext·RoutingDecision·EligibilityResult가 잘못된 값에 IllegalStateException을 던진다. --- docs/capability-status.md | 2 +- docs/fileserver/design-deviations.md | 13 +- docs/manual/07-authorization.md | 7 +- docs/notification/api-surface-snapshot.txt | 38 +- docs/notification/configuration-reference.md | 12 + docs/notification/operations.md | 25 ++ docs/testing/TESTING_STRATEGY.md | 8 +- docs/upgrade/2026-11-round9-application.md | 33 +- src/adapter/inbound/web-advanced/build.gradle | 2 + .../stream/replay/MessagingReplayBridge.java | 29 +- .../replay/MessagingReplayBridgeTest.java | 20 +- .../admin/FileserverAdminController.java | 44 +-- ...FileserverCapabilityAutoConfiguration.java | 3 +- .../NotificationPlatformDispatchConfig.java | 30 +- .../NotificationPlatformPolicyConfig.java | 13 +- .../NotificationPlatformRuntimeConfig.java | 1 - .../NotificationPlatformSettings.java | 66 ++++ .../NotificationPlatformWorkerConfig.java | 175 --------- .../dispatch/ReconciliationJobWorker.java | 144 ------- ...ficationPlatformCoreConfigurationTest.java | 24 +- ...otificationPlatformImportCoverageTest.java | 7 +- .../NotificationPlatformSettingsTest.java | 99 +++++ .../NotificationWorkerLifecycleTest.java | 99 ----- .../outbound/persistence-jpa/build.gradle | 8 - .../liveevent/JpaLiveEventReplay.java | 6 +- .../JpaLiveEventReplayContractTest.java | 8 +- ...ficationBackgroundPassIntegrationTest.java | 11 +- .../platform/JpaAdminOperationStore.java | 8 +- .../platform/JpaRecipientLeaseStore.java | 7 +- src/application/core/AGENTS.md | 3 +- .../realtime/GetReplayWindowQuery.java | 16 + .../realtime/GetReplayWindowUseCase.java | 41 ++ ...ort.java => LiveEventReplayQueryPort.java} | 13 +- .../realtime/ReplayLiveEventsQuery.java | 22 ++ .../realtime/ReplayLiveEventsUseCase.java | 45 +++ .../InMemoryLiveEventReplayContractTest.java | 6 +- .../testkit/InMemoryLiveEventReplay.java | 14 +- ...java => LiveEventReplayQueryContract.java} | 10 +- .../cleanup/RunScheduledCleanupCommand.java | 18 + .../cleanup/RunScheduledCleanupUseCase.java | 48 +++ .../port/in/CleanupUploadsCommand.java | 22 ++ .../port/in/ListIncompleteUploadsQuery.java | 10 + .../fileserver/port/in/ListOrphansQuery.java | 9 + .../port/in/OrphanReconcileCommand.java | 47 +++ .../ReconcileRecoverableFilesCommand.java | 17 + .../ReconcileRecoverableFilesUseCase.java | 48 +++ .../RunScheduledCleanupUseCaseTest.java | 60 +++ .../port/in/AdminRequestDefaultsTest.java | 116 ++++++ .../ReconcileRecoverableFilesUseCaseTest.java | 68 ++++ src/application/notification/AGENTS.md | 2 + .../admin/AdminOperationExecutor.java | 29 +- ... => AdminOperationRequestFingerprint.java} | 52 +-- .../admin/AdminOperationStorePort.java | 10 +- .../admin/ChangeProviderStateCommand.java | 22 ++ .../admin/ChangeProviderStateUseCase.java | 130 +++++++ .../admin/ChangeSuppressionCommand.java | 22 ++ .../admin/ChangeSuppressionUseCase.java | 149 +++++++ .../NotificationAdminApplicationService.java | 362 ------------------ .../admin/NotificationAdminPermissions.java | 27 ++ .../admin/NotificationAdminService.java | 22 -- ...Command.java => ProviderStateRequest.java} | 4 +- .../admin/ReconcileDeliveriesCommand.java | 22 ++ .../admin/ReconcileDeliveriesUseCase.java | 143 +++++++ ...cileCommand.java => ReconcileRequest.java} | 4 +- .../admin/RedriveDeliveryCommand.java | 21 + .../admin/RedriveDeliveryUseCase.java | 149 +++++++ ...edriveCommand.java => RedriveRequest.java} | 4 +- ...ssCommand.java => SuppressionRequest.java} | 4 +- .../ProviderEventProjectionService.java | 21 + .../callback/ReplayProviderEventsCommand.java | 6 + .../ReplayProviderEventsUseCase.java} | 56 ++- .../dispatch/ClaimDueDeliveriesCommand.java | 24 ++ .../dispatch/ClaimDueDeliveriesUseCase.java | 47 +++ .../DispatchClaimedDeliveryCommand.java | 16 + .../DispatchClaimedDeliveryUseCase.java | 46 +++ .../ReconcileOutstandingAttemptsCommand.java | 6 + .../ReconcileOutstandingAttemptsUseCase.java | 185 +++++++++ .../RecoverAbandonedDeliveriesCommand.java | 6 + .../RecoverAbandonedDeliveriesUseCase.java | 47 +++ .../MeasureDispatchBacklogQuery.java | 6 + .../MeasureDispatchBacklogUseCase.java | 52 +++ .../DefaultNotificationRetryPolicy.java | 15 +- .../platform/policy/EligibilityResult.java | 2 +- .../platform/policy/RetryBackoff.java | 7 +- .../platform/policy/RetryBudget.java | 3 +- .../platform/policy/RetryContext.java | 2 +- .../platform/policy/RetryDecision.java | 2 +- .../platform/policy/RetryTiming.java | 31 ++ .../platform/policy/RouteCandidate.java | 2 +- .../platform/policy/RoutingContext.java | 2 +- .../platform/policy/RoutingDecision.java | 4 +- .../platform/policy/package-info.java | 16 + .../admin/AdminOperationExecutorTest.java | 20 +- ...AdminOperationRequestFingerprintTest.java} | 22 +- ...ava => NotificationAdminUseCasesTest.java} | 263 ++++++++++--- .../ClaimDueDeliveriesUseCaseTest.java | 90 +++++ .../DispatchClaimedDeliveryUseCaseTest.java | 78 ++++ .../NotificationDispatchServiceTest.java | 31 +- .../platform/dispatch/PlatformFakes.java | 33 ++ ...oncileOutstandingAttemptsUseCaseTest.java} | 177 ++++++--- ...RecoverAbandonedDeliveriesUseCaseTest.java | 77 ++++ .../policy/NotificationRetryPolicyTest.java | 2 +- .../platform/policy/PolicyValuesTest.java | 137 +++++++ .../testkit/InMemoryAdminOperationStore.java | 8 +- .../testkit/InMemoryDeliveryAttemptStore.java | 13 + .../AdapterBoundaryArchitectureTest.java | 96 ++++- .../ApplicationBoundaryArchitectureTest.java | 2 +- .../ArchRuleDiscoveryContractTest.java | 1 + .../ArchitectureViolationFixtureTest.java | 200 ++++++++++ ...ileserverUseCaseContractDeviationTest.java | 71 ++-- ...ificationUseCaseContractDeviationTest.java | 81 ++++ ...eNamedServiceHoldingControllerFixture.java | 25 ++ .../UseCaseReadingStreamBridgeFixture.java | 22 ++ .../adapter/inbound/web/package-info.java | 3 + .../application/ReportExportService.java | 28 ++ .../allowed/application/package-info.java | 3 + .../CoordinatorHoldingControllerFixture.java | 24 ++ ...viceInterfaceHoldingControllerFixture.java | 25 ++ .../IdGeneratingReceiptHandlerFixture.java | 21 + .../IdMintingReceiptHandlerFixture.java | 21 + ...dentifierHoldingReceiptHandlerFixture.java | 20 + .../QueryPortHoldingStreamBridgeFixture.java | 21 + .../history/EventHistoryQueryPort.java | 8 + .../application/history/ReceiptIdFactory.java | 8 + .../history/ReceiptIdGeneratorPort.java | 8 + .../history/ReceiptIdentifierPort.java | 11 + .../maintenance/MaintenanceService.java | 11 + .../maintenance/WriterSlotCoordinator.java | 10 + .../fileserver/FileserverCleanupWorker.java | 32 +- ...serverMaintenanceServiceConfiguration.java | 13 +- .../FileserverPlatformAutoConfiguration.java | 1 + src/bootstrap/notification-platform/README.md | 11 +- .../notification-platform/build.gradle | 2 + .../notification-platform/gradle.lockfile | 2 +- .../NotificationAdminConfiguration.java | 92 +++++ ...plicationCompositionAutoConfiguration.java | 11 +- .../NotificationBackgroundWorkers.java | 69 ++-- .../NotificationDeliveryConfiguration.java | 41 +- .../NotificationSchedulerWorker.java | 51 ++- .../NotificationWorkerConfiguration.java | 129 +++++++ .../config/module-defaults/notification.yml | 25 +- .../NotificationModuleDefaultsTest.java | 16 +- .../NotificationWorkerLifecycleTest.java | 111 ++++++ 143 files changed, 4251 insertions(+), 1383 deletions(-) delete mode 100644 src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformWorkerConfig.java delete mode 100644 src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorker.java delete mode 100644 src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationWorkerLifecycleTest.java create mode 100644 src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowQuery.java create mode 100644 src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowUseCase.java rename src/application/core/src/main/java/dev/caskeleton/application/realtime/{LiveEventReplayPort.java => LiveEventReplayQueryPort.java} (59%) create mode 100644 src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsQuery.java create mode 100644 src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsUseCase.java rename src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/{LiveEventReplayContract.java => LiveEventReplayQueryContract.java} (94%) create mode 100644 src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupCommand.java create mode 100644 src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCase.java create mode 100644 src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesCommand.java create mode 100644 src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCase.java create mode 100644 src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCaseTest.java create mode 100644 src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/port/in/AdminRequestDefaultsTest.java create mode 100644 src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCaseTest.java rename src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/{AdminOperationCommandFingerprint.java => AdminOperationRequestFingerprint.java} (50%) create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateCommand.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateUseCase.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionCommand.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionUseCase.java delete mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationService.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminPermissions.java delete mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminService.java rename src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/{SetProviderStateCommand.java => ProviderStateRequest.java} (91%) create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesCommand.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesUseCase.java rename src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/{ReconcileCommand.java => ReconcileRequest.java} (92%) create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryCommand.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryUseCase.java rename src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/{RedriveCommand.java => RedriveRequest.java} (93%) rename src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/{SuppressCommand.java => SuppressionRequest.java} (94%) create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsCommand.java rename src/{adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ProviderEventReplayWorker.java => application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsUseCase.java} (50%) create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesCommand.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCase.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryCommand.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCase.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsCommand.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCase.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesCommand.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCase.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogQuery.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogUseCase.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryTiming.java create mode 100644 src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/package-info.java rename src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/{AdminOperationCommandFingerprintTest.java => AdminOperationRequestFingerprintTest.java} (65%) rename src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/{NotificationAdminApplicationServiceTest.java => NotificationAdminUseCasesTest.java} (56%) create mode 100644 src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCaseTest.java create mode 100644 src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCaseTest.java rename src/{adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorkerTest.java => application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCaseTest.java} (50%) create mode 100644 src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCaseTest.java create mode 100644 src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/PolicyValuesTest.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/NotificationUseCaseContractDeviationTest.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseNamedServiceHoldingControllerFixture.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseReadingStreamBridgeFixture.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/ReportExportService.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/CoordinatorHoldingControllerFixture.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/ServiceInterfaceHoldingControllerFixture.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdGeneratingReceiptHandlerFixture.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdMintingReceiptHandlerFixture.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdentifierHoldingReceiptHandlerFixture.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/QueryPortHoldingStreamBridgeFixture.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/EventHistoryQueryPort.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdFactory.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdGeneratorPort.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdentifierPort.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/MaintenanceService.java create mode 100644 src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/WriterSlotCoordinator.java rename src/{adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/autoconfigure => bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver}/FileserverMaintenanceServiceConfiguration.java (79%) create mode 100644 src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationAdminConfiguration.java rename src/{adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch => bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification}/NotificationBackgroundWorkers.java (55%) rename src/{adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch => bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification}/NotificationSchedulerWorker.java (80%) create mode 100644 src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationWorkerConfiguration.java create mode 100644 src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationWorkerLifecycleTest.java diff --git a/docs/capability-status.md b/docs/capability-status.md index 2302c0a3..e2943d6f 100644 --- a/docs/capability-status.md +++ b/docs/capability-status.md @@ -211,7 +211,7 @@ web 경로의 `W/`는 `src/adapter/inbound/web/src/main/java/dev/caskeleton/adap | cache 정책 카탈로그 (`WebCachePolicyCatalog`, `WebCachePolicy`, `WebVaryPolicy`, `WebCacheHeaders`)와 테스트 | `W/cache/` (W3-2에서 web-advanced로 옮긴 뒤 삭제) | `208077fd` | 한 기능에 한 세대: 운영은 `CacheControlFilter` | | JSON mapper factory (`WebObjectMapperFactory`, `WebJsonProfile`, `BoundedJsonFactory`)와 테스트 | `W/json/` (W3-2에서 web-advanced로 옮긴 뒤 삭제) | `208077fd` | 한 기능에 한 세대: 운영은 Boot `JsonMapper` + `spring.jackson.*`. 스트리밍 encoder는 Boot `JsonMapper` bean을 받도록 변경 | | OpenAPI customizer (`WebOpenApiCustomizer`, `WebOpenApiProfile`, `CursorSchemaContributor`)와 `WebOpenApiSnapshotTest` | `W/openapi/` 의 해당 3개 파일 (W3-2에서 web-advanced로 옮긴 뒤 삭제) | `208077fd` | 한 기능에 한 세대: 운영은 springdoc + `OpenApiContractConfig`. `CursorPage` 스키마는 이미 삭제된 커서 페이지네이션을 설명 | -| realtime 포트 (`ConnectionRegistryPort`, `DurableFanoutPort` 외 7개 타입)와 Redis 연결 레지스트리 (`RedisConnectionRegistryAdapter`) | `src/application-core/src/main/java/dev/caskeleton/application/realtime/`, `src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/realtime/` | `208077fd` | 소비자 0 (websocket 정리 후 호출자 없음, bean 없음). `LiveEventReplayPort` 계열은 유지 | +| realtime 포트 (`ConnectionRegistryPort`, `DurableFanoutPort` 외 7개 타입)와 Redis 연결 레지스트리 (`RedisConnectionRegistryAdapter`) | `src/application-core/src/main/java/dev/caskeleton/application/realtime/`, `src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/realtime/` | `208077fd` | 소비자 0 (websocket 정리 후 호출자 없음, bean 없음). `LiveEventReplayQueryPort` 계열은 유지 | | graphql `advanced/**` 전체 (15개 패키지, 130개, ~5,600줄) + 테스트 | `src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/advanced/` | `208077fd` | 모든 진입점이 `GraphQlAdvancedModuleGuard`를 거치는데 그 플래그(`GraphQlAdvancedFeatureFlags`)를 바인딩하는 코드가 없다. Spring GraphQL/graphql-java SPI를 구현한 타입이 없어 SSE·RSocket·federation·incremental·GET·chaining은 이름만 있는 모델. `GraphQlReplaySource`·`GraphQlPersistedOperationAdminPort`는 구현체 없는 인터페이스, persisted 레지스트리가 기대는 `OperationalRecordStorePort`도 구현체 0. composition/incremental/bootstrap은 gate·등급 코드 | | `backend.graphql.unsupported.*` 6개 키 (`GraphQlPlatformSettings.Unsupported`, `withUnsupportedCapability`, 검증기 분기) | `src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/autoconfigure/GraphQlPlatformSettings.java`, `GraphQlPlatformStartupValidator.java` | `208077fd` | 켜면 부팅 실패만 한다. 구현하지 않은 기능에는 설정 키를 두지 않는다(AGENTS 기준 3) | | `backend.graphql.cursor.key-ids` (`GraphQlPlatformSettings.Cursor`, `withCursorKeyIds`) | `src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/autoconfigure/GraphQlPlatformSettings.java` | `208077fd` | 읽는 곳이 등록되지 않은 actuator 보고서뿐. 커서 서명은 라이브러리(`HmacGraphQlCursorCodec`)를 서비스가 직접 조립한다 | diff --git a/docs/fileserver/design-deviations.md b/docs/fileserver/design-deviations.md index 6e9b23b9..a4a28952 100644 --- a/docs/fileserver/design-deviations.md +++ b/docs/fileserver/design-deviations.md @@ -193,9 +193,16 @@ transport가 닿는 모든 연산은 `application.fileserver.port.in`의 메서 `BlockingTransferExecutor`가 요청의 context를 worker로 옮긴다(`BlockingTransferExecutorSecurityContextTest`). 이것이 없으면 인증한 append가 익명으로 거부됐다. -use case가 아닌 것은 어떤 transport도 부르지 않는 것이다. `CleanupService`와 `FileReconciliationService`는 -호출자 신원이 없는 백그라운드 worker가 돌린다. 상태를 바꾸는 use case는 scheduler가 가질 수 없는 권한을 -선언해야 한다. `FileserverUseCaseContractDeviationTest`가 그 목록을 고정한다. +백그라운드 worker가 하는 쓰기도 use case다. 다만 사용자 것과 따로다. `FileserverCleanupWorker`는 호출자 신원이 없는 +scheduler 스레드에서 돌므로 `@SystemInvoked` use case 둘(`RunScheduledCleanupUseCase`, +`ReconcileRecoverableFilesUseCase`)을 `new`로 만들어 쥔다. bean이 아니어서 transport가 닿지 못한다. 운영자가 같은 +cleanup을 돌리는 길은 `file:admin`을 선언한 `CleanupUploadsUseCase`다(2026-11-13). + +use case가 아닌 것은 그 use case들이 쥔 협력 객체다. `CleanupService`(`DefaultCleanupService`), +`FileReconciliationService`(`DefaultFileReconciliationService`), `WriterLeaseCoordinator`다. 이 서비스들은 +`composition:fileserver-platform`이 bean으로 조립한다. 권한 검사 없이 쓰는 bean이라, inbound adapter가 interface로라도 +쥐면 `INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES`가 빌드를 실패시킨다(2026-11-13부터 interface와 +coordinator도 본다). `FileserverUseCaseContractDeviationTest`가 그 목록을 고정한다. `FileserverErrorCode`의 HTTP 상태 어휘는 애플리케이션 계층에 남는다. 이 절이 예전에 든 이유 그대로다: servlet transport와 Nginx 위임 경로가 같은 실패에 다른 상태로 답하면 안 된다. diff --git a/docs/manual/07-authorization.md b/docs/manual/07-authorization.md index 4d5db056..057d5f55 100644 --- a/docs/manual/07-authorization.md +++ b/docs/manual/07-authorization.md @@ -91,7 +91,7 @@ client가 같은 이름의 역할을 정의해도 이 서비스 권한이 되지 | 경로 | 어디에 두나 | 유스케이스는 누가 만드나 | | --- | --- | --- | -| 조립 모듈의 스케줄러 | `composition:*` leaf. 예: outbox relay(`src/bootstrap/outbox-relay`) | 조립 모듈의 `@Bean` 메서드가 `new`로 만들어 스케줄러에 넘긴다 | +| 조립 모듈의 스케줄러 | `composition:*` leaf. 예: outbox relay(`src/bootstrap/outbox-relay`), 알림 worker(`NotificationWorkerConfiguration`), fileserver cleanup worker(`FileserverCleanupWorker`) | 조립 모듈의 `@Bean` 메서드가 `new`로 만들어 스케줄러에 넘긴다. worker 생성자가 받은 서비스로 만들어도 같다 | | 메시지 handler | `adapter:inbound:messaging` leaf, 또는 서비스 leaf의 `..adapter.inbound.messaging..` 패키지 | handler 생성자가 port를 받아 `new`로 만든다. handler에 `@SystemCaller`. 작업 순서는 09장 「소비」 4단계 | | 예약 작업 | 서비스 자기 leaf의 `..adapter.inbound.scheduler..` 패키지. 등록된 scheduler leaf는 없다 | 작업 클래스 생성자가 port를 받아 `new`로 만든다. 작업 클래스에 `@SystemCaller`. 모양은 메시지 handler와 같고, 깨우는 것이 `@Scheduled` 메서드다. `Runnable`·`ApplicationRunner` 같은 프로젝트 밖 타입을 구현하지 않는다. task scheduler에 직접 등록하면 그 자리에서 `job::run`을 넘긴다 | @@ -105,6 +105,7 @@ client가 같은 이름의 역할을 정의해도 이 서비스 권한이 되지 | bean 금지 | `@SystemInvoked` 유스케이스가 bean이면 기동 실패. 호출자가 `new`로 만들어 혼자 들고 있는 것만 허용한다 | `src/bootstrap/service-runtime/src/main/java/dev/caskeleton/composition/runtime/authorization/PermissionEnforcementGuard.java` | | 시스템 호출자만 | inbound adapter가 그 클래스에 의존하면 빌드 실패. `@SystemCaller(reason = "...")`를 단 클래스와 그 안에 중첩된 클래스만 예외. inbound 밖 사용자 전송과 중간 클래스를 거친 길은 아래 「전송 → 시스템 호출자 금지」가 잡는다 | `ONLY_SYSTEM_CALLERS_REACH_SYSTEM_INVOKED_USE_CASES`(`ApplicationBoundaryArchitectureTest`) | | controller 위치 | Spring controller는 `..adapter.inbound..`에만 둔다. 조립 루트나 조립 모듈에 둔 controller는 빌드 실패 | `CONTROLLERS_LIVE_IN_INBOUND_ADAPTERS`(`AdapterBoundaryArchitectureTest`) | +| 감싼 서비스 금지 | inbound adapter가 application의 `*Service`·`*Orchestrator`·`*Coordinator`에 의존하면 빌드 실패. interface도 든다. 유스케이스를 구현한 타입은 빠진다. `@SystemInvoked` 유스케이스가 감싸는 서비스(fileserver `CleanupService`, 알림 `ReconciliationService` 등)는 bean이고 권한 검사 없이 쓰기 때문이다 | `INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES`(`AdapterBoundaryArchitectureTest`) | | 표시 검사 | `@SystemCaller` 클래스와 그 안에 중첩된 클래스는 `..adapter.inbound.messaging..`·`..adapter.inbound.scheduler..`에만 있고, 사용자 전송 타입(controller·JAX-RS·GraphQL·STOMP mapping과 channel interceptor·gRPC·servlet)에 의존하지 않는다 | `SYSTEM_CALLERS_ARE_SYSTEM_DRIVEN_INBOUND_ADAPTERS`(같은 파일) | | 전송 → 시스템 호출자 금지 | 사용자 전송 클래스가 `@SystemCaller`나, 그 클래스가 구현·상속하는 프로젝트 타입(작업의 인터페이스 등), 또는 `@SystemInvoked` 유스케이스 자신에 의존하면 빌드 실패. 중간 클래스를 거쳐도 잡는다. 사용자 전송 타입에는 Spring Security의 요청 경로 확장점(`org.springframework.security.web`·`.config.annotation.web`·`.oauth2.server.resource`·`.oauth2.jwt` 패키지, `AuthenticationProvider`, `AuthorizationManager`)이 든다. 그래서 web 패키지 밖에 둔 JWT converter나 authorization manager도 사용자 전송이다. 사용자 전송 클래스는 넷이다. ① 사용자 전송 패키지에 있는 클래스. ② 설정(`@Configuration`·`@AutoConfiguration`) 클래스 자신이 요청 처리기 타입(함수형 `RouterFunction`·`HandlerFunction`, `@RestController`·`@RequestMapping`, servlet·`Filter`·`HandlerInterceptor`, `WebSocketHandler`, GraphQL `DataFetcher`·`RuntimeWiringConfigurer`, gRPC `BindableService`·`ServerInterceptor`, STOMP `ChannelInterceptor`·`ExecutorChannelInterceptor` 등)에 의존하는 것. 요청 처리 코드를 설정에 직접 쓴 경우와 등록만 한 경우가 모두 든다. 요청 처리기 타입을 구현·상속한 프로젝트 클래스(예: `ChannelInterceptor`를 구현한 interceptor)를 필드·생성자 인자·`new`로 쥔 설정도 든다. `registration.interceptors(frameCounter)`는 클래스 의존으로 `ChannelRegistration`만 남기므로 그 필드의 타입이 설정을 사용자 전송으로 만든다. channel interceptor를 구현한 설정은 프레임마다 자기 메서드가 돈다. interceptor를 `@Bean`으로 내놓는 설정은 route를 등록하는 설정과 같다. 함수형 route나 data fetcher를 람다로 쓰면 그 코드가 설정 클래스의 메서드로 컴파일되어 설정이 곧 endpoint다. 처리 코드를 다른 클래스에 두고 메서드 참조로 등록만 해도(`route().GET("/orders", orders::list)`) 설정은 route builder에 의존한다. 클래스 의존으로는 등록과 처리 코드를 가를 수 없어서 둘 다 사용자 전송으로 센다. ③ 설정 안에 중첩된 클래스 가운데 사용자 전송 타입에 의존하는 것. 중첩된 JAX-RS 자원·`@ServerEndpoint`, `configureClientInboundChannel`에서 등록하는 STOMP channel interceptor(익명 클래스여도, STOMP 헤더를 읽지 않아도)도 여기서 잡힌다. 설정용 타입(`WebMvcConfigurer`·`WebSocketConfigurer`·STOMP broker 설정과 그 registry, CORS 설정, `RequestMethod`)에만 의존하는 중첩 클래스는 설정의 일부로 본다. inbound channel의 thread 수만 정하는 설정도 설정용이다. ④ 그 밖의 클래스 가운데 사용자 전송 타입에 의존하는 것. 사용자 전송 패키지 밖에 둔 channel interceptor가 여기에 든다. 그래서 handler를 bean으로 등록하는 설정은 같은 클래스에서 `WebMvcConfigurer`(설정 필드를 읽는 익명 클래스여도)·CORS 설정·controller bean을 함께 만들어도 걸리지 않는다. 같은 클래스에 "지금 실행" route·data fetcher·interceptor를 쓰거나, route를 등록하거나, 안에 중첩한 endpoint가 작업을 부르면(설정의 메서드를 거쳐 불러도) 걸린다. route·data fetcher·interceptor 등록은 handler bean을 만드는 설정과 다른 설정에 둔다. 설정이 만드는 controller는 controller 자신이 검사받는다 | `USER_TRANSPORTS_DO_NOT_REACH_SYSTEM_CALLERS`(같은 파일) | | 프로젝트 밖 상위 타입 금지 | `@SystemCaller` 클래스와 그 안에 중첩된 클래스(익명·지역 클래스와 예외 제외)는 프로젝트(`dev.caskeleton..`와 소유자의 별도 repo 라이브러리 `dev.cakit..`) 밖의 클래스·인터페이스를 상속·구현하지 않는다. `Runnable`은 누구나 의존하는 타입이라, endpoint가 작업을 `Runnable`로 주입받아도 위 규칙이 다른 사용과 구별하지 못하기 때문이다. 메시지 handler의 `MessageHandler`(메시징 라이브러리 `dev.cakit.messaging`)는 프로젝트 타입으로 쳐서 된다. record·enum은 언어가 붙이는 상위 타입(`Record`, `Enum`과 `Enum`이 구현하는 `Comparable`·`Serializable`·`Constable`)만은 가져도 된다. 그래서 handler 안의 파싱 결과 record, 결과 enum, 독(poison) 메시지 예외, record로 쓴 handler는 통과하고, `Runnable`을 구현한 record는 걸린다 | `SYSTEM_CALLERS_HAVE_ONLY_PROJECT_SUPERTYPES`(같은 파일) | @@ -165,7 +166,7 @@ web이 빠진 조립(gRPC 전용·배치 전용)에는 `@RequiresPermission`을 - 쓰기 = `@RequiresPermission` + non-final + bean + 역할 매핑. - 시스템만 부르는 쓰기 = `@SystemInvoked` + bean 아님 + 사용자 전송에서 안 닿음. 부르는 쪽은 셋뿐이다: 조립 모듈의 스케줄러, `@SystemCaller` 메시지 handler, `@SystemCaller` 예약 작업. 사용자 전송은 - `@SystemCaller`에도, 그 작업의 인터페이스에도 닿지 못한다. 타입 이름이 남지 않는 길(이름으로 찾기, - 설정이 넘기는 함수)은 빌드가 못 보니 리뷰가 막는다. + `@SystemCaller`에도, 그 작업의 인터페이스에도, 유스케이스가 감싸는 서비스에도 닿지 못한다. 타입 이름이 + 남지 않는 길(이름으로 찾기, 설정이 넘기는 함수)은 빌드가 못 보니 리뷰가 막는다. - 객체 판정은 유스케이스 안, 남의 것은 404. - 역할은 자기 client 것만 읽는다. diff --git a/docs/notification/api-surface-snapshot.txt b/docs/notification/api-surface-snapshot.txt index f5622441..fb1fdbf5 100644 --- a/docs/notification/api-surface-snapshot.txt +++ b/docs/notification/api-surface-snapshot.txt @@ -2,7 +2,7 @@ # Every top-level public type under the platform packages. Growth is a reviewed change. # Update only after review with: # ./gradlew :adapter:outbound:notification:updateNotificationApiSurface -PapproveNotificationApiSurfaceChange -# types: 359 +# types: 375 dev.caskeleton.adapter.outbound.notification.NotificationConfig dev.caskeleton.adapter.outbound.notification.NotificationRoutesSettings dev.caskeleton.adapter.outbound.notification.core.FailOpenNotificationProvider @@ -35,14 +35,10 @@ dev.caskeleton.adapter.outbound.notification.platform.dispatch.CapabilityReconci dev.caskeleton.adapter.outbound.notification.platform.dispatch.ConfiguredProfileCatalog dev.caskeleton.adapter.outbound.notification.platform.dispatch.JacksonRoutingPlanCodec dev.caskeleton.adapter.outbound.notification.platform.dispatch.LoggingInboxSignalPublisher -dev.caskeleton.adapter.outbound.notification.platform.dispatch.NotificationBackgroundWorkers dev.caskeleton.adapter.outbound.notification.platform.dispatch.NotificationDispatchProperties -dev.caskeleton.adapter.outbound.notification.platform.dispatch.NotificationSchedulerWorker dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderAttemptLimiter -dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderEventReplayWorker dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderRuntime dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderRuntimeRegistry -dev.caskeleton.adapter.outbound.notification.platform.dispatch.ReconciliationJobWorker dev.caskeleton.adapter.outbound.notification.platform.dispatch.RegistryProviderDispatchGateway dev.caskeleton.adapter.outbound.notification.platform.dispatch.RegistryProviderRuntimeControl dev.caskeleton.adapter.outbound.notification.platform.dispatch.SingleTenantContext @@ -104,16 +100,23 @@ dev.caskeleton.application.notification.platform.admin.AdminAuthorizationGuard dev.caskeleton.application.notification.platform.admin.AdminOperationClaim dev.caskeleton.application.notification.platform.admin.AdminOperationResult dev.caskeleton.application.notification.platform.admin.AdminOperationStorePort +dev.caskeleton.application.notification.platform.admin.ChangeProviderStateCommand +dev.caskeleton.application.notification.platform.admin.ChangeProviderStateUseCase +dev.caskeleton.application.notification.platform.admin.ChangeSuppressionCommand +dev.caskeleton.application.notification.platform.admin.ChangeSuppressionUseCase dev.caskeleton.application.notification.platform.admin.DuplicateRiskApprovalRequiredException dev.caskeleton.application.notification.platform.admin.DuplicateRiskGuard -dev.caskeleton.application.notification.platform.admin.NotificationAdminApplicationService dev.caskeleton.application.notification.platform.admin.NotificationAdminAuthority -dev.caskeleton.application.notification.platform.admin.NotificationAdminService +dev.caskeleton.application.notification.platform.admin.NotificationAdminPermissions dev.caskeleton.application.notification.platform.admin.ProviderRuntimeControlPort -dev.caskeleton.application.notification.platform.admin.ReconcileCommand -dev.caskeleton.application.notification.platform.admin.RedriveCommand -dev.caskeleton.application.notification.platform.admin.SetProviderStateCommand -dev.caskeleton.application.notification.platform.admin.SuppressCommand +dev.caskeleton.application.notification.platform.admin.ProviderStateRequest +dev.caskeleton.application.notification.platform.admin.ReconcileDeliveriesCommand +dev.caskeleton.application.notification.platform.admin.ReconcileDeliveriesUseCase +dev.caskeleton.application.notification.platform.admin.ReconcileRequest +dev.caskeleton.application.notification.platform.admin.RedriveDeliveryCommand +dev.caskeleton.application.notification.platform.admin.RedriveDeliveryUseCase +dev.caskeleton.application.notification.platform.admin.RedriveRequest +dev.caskeleton.application.notification.platform.admin.SuppressionRequest dev.caskeleton.application.notification.platform.api.CallbackIngestionResult dev.caskeleton.application.notification.platform.api.CancelCommand dev.caskeleton.application.notification.platform.api.CancelResult @@ -198,6 +201,8 @@ dev.caskeleton.application.notification.platform.callback.ProviderEventRecord dev.caskeleton.application.notification.platform.callback.ProviderEventRecordId dev.caskeleton.application.notification.platform.callback.ProviderEventSource dev.caskeleton.application.notification.platform.callback.ProviderRequestIdHash +dev.caskeleton.application.notification.platform.callback.ReplayProviderEventsCommand +dev.caskeleton.application.notification.platform.callback.ReplayProviderEventsUseCase dev.caskeleton.application.notification.platform.callback.StandardDeliveryProjector dev.caskeleton.application.notification.platform.callback.SuppressionFacts dev.caskeleton.application.notification.platform.callback.VerifiedProviderEvent @@ -218,11 +223,15 @@ dev.caskeleton.application.notification.platform.dispatch.AttachmentIntegrityGua dev.caskeleton.application.notification.platform.dispatch.CancelNotificationApplicationUseCase dev.caskeleton.application.notification.platform.dispatch.CanonicalNotificationPlanEncoder dev.caskeleton.application.notification.platform.dispatch.CanonicalNotificationPlanWriter +dev.caskeleton.application.notification.platform.dispatch.ClaimDueDeliveriesCommand +dev.caskeleton.application.notification.platform.dispatch.ClaimDueDeliveriesUseCase dev.caskeleton.application.notification.platform.dispatch.ContactPointRecord dev.caskeleton.application.notification.platform.dispatch.ContactPointStorePort dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptFactory dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptRecord dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort +dev.caskeleton.application.notification.platform.dispatch.DispatchClaimedDeliveryCommand +dev.caskeleton.application.notification.platform.dispatch.DispatchClaimedDeliveryUseCase dev.caskeleton.application.notification.platform.dispatch.DispatchGuardOutcome dev.caskeleton.application.notification.platform.dispatch.DispatchOutcomeRecorder dev.caskeleton.application.notification.platform.dispatch.DispatchPipeline @@ -249,10 +258,14 @@ dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryRecor dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort dev.caskeleton.application.notification.platform.dispatch.RecipientLease dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort +dev.caskeleton.application.notification.platform.dispatch.ReconcileOutstandingAttemptsCommand +dev.caskeleton.application.notification.platform.dispatch.ReconcileOutstandingAttemptsUseCase dev.caskeleton.application.notification.platform.dispatch.ReconciliationGatewayPort dev.caskeleton.application.notification.platform.dispatch.ReconciliationJob dev.caskeleton.application.notification.platform.dispatch.ReconciliationJobStorePort dev.caskeleton.application.notification.platform.dispatch.ReconciliationService +dev.caskeleton.application.notification.platform.dispatch.RecoverAbandonedDeliveriesCommand +dev.caskeleton.application.notification.platform.dispatch.RecoverAbandonedDeliveriesUseCase dev.caskeleton.application.notification.platform.dispatch.ScheduleNotificationApplicationUseCase dev.caskeleton.application.notification.platform.dispatch.SubmitNotificationApplicationUseCase dev.caskeleton.application.notification.platform.dispatch.SuppressionSideEffect @@ -274,6 +287,8 @@ dev.caskeleton.application.notification.platform.inbox.NotificationInbox dev.caskeleton.application.notification.platform.inbox.NotificationInboxSignalPort dev.caskeleton.application.notification.platform.observation.CardinalityGuard dev.caskeleton.application.notification.platform.observation.IllegalMetricTagException +dev.caskeleton.application.notification.platform.observation.MeasureDispatchBacklogQuery +dev.caskeleton.application.notification.platform.observation.MeasureDispatchBacklogUseCase dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent dev.caskeleton.application.notification.platform.observation.NotificationAuditPort dev.caskeleton.application.notification.platform.observation.NotificationMetricName @@ -301,6 +316,7 @@ dev.caskeleton.application.notification.platform.policy.RetryBackoff dev.caskeleton.application.notification.platform.policy.RetryBudget dev.caskeleton.application.notification.platform.policy.RetryContext dev.caskeleton.application.notification.platform.policy.RetryDecision +dev.caskeleton.application.notification.platform.policy.RetryTiming dev.caskeleton.application.notification.platform.policy.RouteCandidate dev.caskeleton.application.notification.platform.policy.RoutingContext dev.caskeleton.application.notification.platform.policy.RoutingDecision diff --git a/docs/notification/configuration-reference.md b/docs/notification/configuration-reference.md index ca79ccd6..38f3a653 100644 --- a/docs/notification/configuration-reference.md +++ b/docs/notification/configuration-reference.md @@ -41,6 +41,18 @@ lease는 provider 호출 한 번이 delivery를 쥐는 시간보다 길어야 끝난 호출도 자기 결과를 기록한다(owner와 fence가 그대로다). 두 번째 worker가 다시 보내지는 않는다. claim은 `DISPATCHING`인 delivery를 가져가지 않는다. +## 재시도 시각 + +| 속성 | 환경 변수 | 기본값 | 범위 | +|---|---|---|---| +| `retry.backoff-base` | 없음 | `1s` | 양수. 첫 재시도의 상한이자 가장 짧은 대기 | +| `retry.backoff-max` | 없음 | `5m` | `backoff-base` 이상. provider의 `Retry-After`도 이 값을 넘지 못한다 | +| `retry.reconcile-delay` | 없음 | `1m` | 0 이상. 결과를 모르는 제출 뒤 provider에 처음 물어보기까지 | + +대기는 `backoff-base`와 지수로 늘어난 상한 사이의 full jitter다. 세 값에는 env 자리표시자가 없다. 배포의 설정 +파일(`application-{profile}.yml`)에서 바꾼다. 값은 provider의 호출 한도에 맞춰 정한다. 템플릿이 정할 일이 아니어서 +상수가 아니라 설정이다(2026-11-13). + ## 콜백 | 속성 | 환경 변수 | 기본값 | 범위 | diff --git a/docs/notification/operations.md b/docs/notification/operations.md index 707d7b55..5199a655 100644 --- a/docs/notification/operations.md +++ b/docs/notification/operations.md @@ -37,6 +37,31 @@ redrive는 `NotificationId`와 `RecipientDeliveryId`를 유지하고, 새 `Deliv 렌더링 digest를 다시 쓴다. 다른 내용을 보내는 것은 redrive가 아니라 새 알림이다. 모호한 attempt를 redrive하려면 중복 위험 승인을 명시해야 한다. 첫 제출이 사용자에게 닿았는지 플랫폼이 정말로 알 수 없기 때문이다. +## 운영자 연산 + +운영자 연산은 넷이고, 연산마다 유스케이스 하나다(`application.notification.platform.admin`의 클래스, command는 그 옆의 +`*Command`). 조립은 `composition:notification-platform`의 `NotificationAdminConfiguration`이 bean으로 한다. + +| 유스케이스 | 하는 일 | permission | 운영자 authority | +|---|---|---|---| +| `RedriveDeliveryUseCase` | delivery 하나를 다시 보낼 차례로 돌린다 | `notification-admin:redrive` | `REDRIVE` | +| `ReconcileDeliveriesUseCase` | attempt 묶음(200개까지)을 provider에 다시 묻는다 | `notification-admin:reconcile` | `RECONCILE` | +| `ChangeSuppressionUseCase` | 억제 항목 하나를 더하거나 푼다 | `notification-admin:suppress` | `SUPPRESS` | +| `ChangeProviderStateUseCase` | provider profile을 켜고 끄거나 drain한다 | `notification-admin:provider-control` | `PROVIDER_CONTROL`, tenant 범위 없는 운영자만 | + +- 확인은 두 겹이다. permission은 전송의 호출자가 이 연산을 부를 수 있는지다(`@RequiresPermission`). authority와 + tenant 범위는 command의 `AdminActor`가 쥐고, 연산이 건드리는 delivery의 tenant와 대조한다. +- 모든 연산은 운영자가 준 operation id로 멱등이다. 같은 id·같은 요청은 저장된 결과를 돌려주고, 같은 id·다른 요청은 + 충돌이다. 대상 확인(tenant·중복 위험)과 claim과 쓰기는 한 쓰기 트랜잭션에서 커밋한다. reconcile만 대상 확인을 읽기 + 트랜잭션에서 하고, provider 호출은 트랜잭션 밖에서 하며, claim과 결과를 따로 커밋한다. +- dry run은 보고만 하고 쓰지 않는다. redrive와 reconcile의 dry run은 대상을 읽기 트랜잭션에서 읽는다. suppression과 provider 상태의 dry run은 아무것도 읽지 않는다. +- 템플릿은 이 연산을 부르는 전송을 싣지 않는다. 운영자 화면이나 endpoint를 여는 서비스가 전송을 만든다. +- 모듈 기본값(`composition:notification-platform`의 `config/module-defaults/notification.yml`)이 role + `notification-admin`에 넷을 모두 준다(`app.authz.role-permissions`). 이 기본값의 정본은 그 파일이다. 운영자 role + 이름이 다르면 배포의 설정 파일에서 `app.authz.role-permissions`에 그 role을 더한다. IdP가 `notification-admin` role을 + 다른 뜻으로 쓰면 배포의 설정 파일에서 `app.authz.role-permissions.notification-admin`에 그 role이 가질 permission 목록을 + 적는다. 출처마다 map 항목이 합쳐지고 같은 키는 배포의 값이 이긴다. + ## actuator 표면 provider 런타임 상태와 세대, 큐 깊이와 나이, 콜백과 맞추기 상태. 주소와 자격 증명은 절대 싣지 않는다. diff --git a/docs/testing/TESTING_STRATEGY.md b/docs/testing/TESTING_STRATEGY.md index c8ff69f5..baf039a8 100644 --- a/docs/testing/TESTING_STRATEGY.md +++ b/docs/testing/TESTING_STRATEGY.md @@ -215,7 +215,7 @@ source set에서 태그 `jpa-contract`로 `jpaPlatformContractTest` lane이 돌 | `IdempotencyStorePort` | `idempotency.testkit.IdempotencyStoreContract` | `InMemoryIdempotencyStore` | `RedisIdempotencyStoreContractTest`(command fake, `test`) · `LiveRedisIdempotencyStoreContractTest`(실 Redis, `redisTopologyTest` standalone·cluster) · `PostgreSqlIdempotencyStoreContractTest`(실 PostgreSQL, `jpaPlatformContractTest`) | | `SecretSourcePort` | `secret.testkit.SecretSourceContract` | `InMemorySecretSource` | `EnvironmentSecretSourceContractTest` — support (`test`) | | `OutboxAppendPort` | `outbox.testkit.OutboxAppendContract` | `InMemoryOutboxStore(DirectTransactionPort)` | `PostgreSqlOutboxAppendTransactionalIntegrationTest` (`postgresqlOutboxAppendTransactionalIntegrationTest`) | -| `LiveEventReplayPort` | `realtime.testkit.LiveEventReplayContract` | `InMemoryLiveEventReplay` | `JpaLiveEventReplayContractTest` (`jpaPlatformContractTest`) | +| `LiveEventReplayQueryPort` | `realtime.testkit.LiveEventReplayQueryContract` | `InMemoryLiveEventReplay` | `JpaLiveEventReplayContractTest` (`jpaPlatformContractTest`) | | `AuthorizationPort` | `security.testkit.AuthorizationContract` | `InMemoryAuthorization` | `AuthorizationAdapterContractTest` — web (`test`) | | `CorrelationIdPort` | `observability.testkit.CorrelationIdContract` | `FixedCorrelationId` | `MdcCorrelationIdPortAdapterContractTest` — web (`test`) | | `CacheRegionPort` | `cache.testkit.CacheRegionContract` | `InMemoryCacheRegion` | `RedisCacheRegionContractTest`(`test`) · `LiveRedisCacheRegionContractTest`(`redisTopologyTest`) | @@ -299,9 +299,9 @@ PostgreSQL 쪽에만 있던 테스트를 계약으로 옮긴 것이다. `Reconci `PostgreSqlAdminOperationStoreContractTest`에만 남겼다. 처음 적용할 때 fake와 실제가 다르게 답한 경우는 없었다. 유스케이스 테스트의 대역은 `PlatformFakes.RequestStore`·deduplication 람다· `AdminOperationExecutorTest.FakeStore`를 이 fake들로 바꿨다. 아직 테스트 안에 남은 대역: admin -`NotificationAdminApplicationServiceTest.RecordingOperations`·`UnusedSuppressionStore`, reconciliation job -`ReconciliationJobWorkerTest`(notification adapter)의 stub — 호출 횟수를 세거나 "부르지 않는다"를 확인하는 -용도라 fake로 바꾸면 단언을 다시 써야 한다. +`NotificationAdminUseCasesTest.RecordingOperations`, reconciliation job +`ReconcileOutstandingAttemptsUseCaseTest`(application-notification)의 stub — 호출 횟수를 세거나 "부르지 않는다"를 +확인하는 용도라 fake로 바꾸면 단언을 다시 써야 한다. notification 계약 여덟 개(recipient lease·recipient delivery·attempt·resolver·projection·ledger·side effect·contact point, 2026-11-03 r3-J)로 9.2의 "다음 단계"가 끝났다. fake는 JPA 구현의 묶음을 따른다: diff --git a/docs/upgrade/2026-11-round9-application.md b/docs/upgrade/2026-11-round9-application.md index e985b88c..91a2d75a 100644 --- a/docs/upgrade/2026-11-round9-application.md +++ b/docs/upgrade/2026-11-round9-application.md @@ -1,22 +1,41 @@ # 업그레이드 안내 — 2026-11 application 구조와 작은 정리 -- 범위: durable operation 제출이 id를 유스케이스에서 만들고 tenant를 타입으로 받게 한 변경, 쓰지 않던 공개 타입 일곱의 - 삭제, 그 밖의 작은 정리. 공개 타입과 생성자가 바뀐 곳이 있다. 아래 표가 그 목록이다. -- 근거: 2026-11-12 전체 리뷰(`docs/reviews/2026-11-12-round9-review/`)의 application 작은 것(B15b) 발견 +- 범위: 시스템이 부르는 쓰기를 `@SystemInvoked` 유스케이스로 나누고, 알림 worker와 fileserver 정리 서비스 조립을 + composition으로 옮기고, 알림 운영자 연산을 유스케이스 넷으로 나누고, replay 읽기와 durable operation 제출을 유스케이스로 + 거치게 한 변경. 쓰지 않던 공개 타입 일곱을 지웠다. 공개 타입과 생성자가 바뀐 곳이 있다. 아래 표가 그 목록이다. +- 근거: 2026-11-12 전체 리뷰(`docs/reviews/2026-11-12-round9-review/`)의 application 구조(B15a)와 작은 것(B15b) 발견 ## 1. 누가 읽나 -이 템플릿을 `upstream` remote로 두고 merge해 받는 파생 서비스가 읽는다. 특히 durable operation을 직접 제출·조회하는 -서비스와 아래 표의 지운 타입을 쓴 서비스다. +이 템플릿을 `upstream` remote로 두고 merge해 받는 파생 서비스가 읽는다. 특히 알림 운영자 연산이나 알림 worker bean을 +직접 쓴 서비스, `MessagingReplayBridge`로 stream을 다시 보내는 서비스, durable operation을 직접 제출하는 서비스, +사용자 전송(web·GraphQL·gRPC·WebSocket) 클래스에서 저장 포트를 직접 쥐는 서비스다. ## 2. 할 일 | # | 할 일 | 안 하면 | 해당 서비스 | | --- | --- | --- | --- | -| 1 | `OperationCommand`에서 operation id 인자를 지우고, tenant를 `Optional`(`application.tenant.TenantId`)로 넘긴다. `GetDurableOperationQuery`·`CancelDurableOperationCommand`의 tenant도 같다. `SubmitDurableOperationUseCase`는 생성자에 `DurableOperationIdFactory`를 받아 id를 직접 만든다. `DurableOperationController`는 id factory를 받지 않는다 | 컴파일이 실패한다 | durable operation을 직접 제출·조회하거나 그 use case를 조립한 서비스 | -| 2 | 쓰는 곳이 없던 공개 타입 일곱을 지웠다: `application.transaction.ReadConsistency`, `FileserverSpans`, 알림의 `ProviderEventId`·`NotificationExpiredException`·`BatchNotificationProviderAdapter`·`CollapseCapability`·`ProviderCollapseMapping` | 그 타입을 쓴 코드는 컴파일이 실패한다. 템플릿 안에는 쓰는 곳이 없었다 | 그 타입을 쓴 서비스 | +| 1 | `NotificationAdminService`·`NotificationAdminApplicationService`가 없어졌다. 운영자 연산은 `platform.admin`의 유스케이스 클래스 `RedriveDeliveryUseCase`·`ReconcileDeliveriesUseCase`·`ChangeSuppressionUseCase`·`ChangeProviderStateUseCase`를 주입받아 `handle(new …Command(request, actor))`로 부른다(command도 `platform.admin`). 운영자 요청 record의 이름이 바뀌었다: `RedriveCommand`→`RedriveRequest`, `ReconcileCommand`→`ReconcileRequest`, `SuppressCommand`→`SuppressionRequest`, `SetProviderStateCommand`→`ProviderStateRequest`. `adminAuthorizationGuard`·`duplicateRiskGuard` bean은 없어졌다(유스케이스 설정이 `new`로 만든다). 모듈 기본값이 role `notification-admin`에 permission 넷(`notification-admin:redrive`·`notification-admin:reconcile`·`notification-admin:suppress`·`notification-admin:provider-control`)을 준다. 운영자 role 이름이 다르면 `app.authz.role-permissions`에 직접 대응시킨다 | 컴파일이 실패한다. 두 guard bean을 주입받던 정의는 기동이 실패한다(`NoSuchBeanDefinitionException`). 운영자가 `notification-admin` role도 대응시킨 role도 없으면 운영자 호출이 403이다 | 알림 운영자 연산을 쓴 서비스 | +| 2 | 알림 worker가 `adapter:outbound:notification`에서 `composition:notification-platform`으로 옮겼다. `NotificationBackgroundWorkers`는 `dev.caskeleton.composition.notification`의 공개 타입이고, 생성자는 서비스 대신 유스케이스를 받는다(lease 회복·replay·맞추기). 발송 scheduler(`NotificationSchedulerWorker`)는 그 패키지 안의 타입이 됐다. claim·발송·backlog 측정 유스케이스로 돌고, 바꿔 끼우는 bean이 아니다. `ReconciliationJobWorker`·`ProviderEventReplayWorker`는 application의 `@SystemInvoked` 유스케이스(`ReconcileOutstandingAttemptsUseCase(jobs, reconciler, tx, clock, deliveryLease, batchSize, maxAdditionalAttempts)`·`ReplayProviderEventsUseCase(projection, batchSize)`)가 됐다. `reconciliationJobWorker`·`notificationProviderEventReplayWorker` bean을 덮어쓰던 서비스는 `notificationBackgroundWorkers` bean을 덮어쓴다 | 옛 import와 생성자가 컴파일되지 않는다. 옛 bean 이름으로 덮어쓴 정의는 더는 쓰이지 않는다 | 알림 worker를 직접 조립하거나 덮어쓴 서비스 | +| 3 | `MessagingReplayBridge`는 생성자에서 조회 유스케이스 둘을 받는다: `new MessagingReplayBridge<>(window, replay, decoder)`. 두 유스케이스는 서비스의 조립 설정(자기 `composition` 설정이나 bootstrap 설정, web 설정이 아니다)에서 bean으로 만든다: `@Bean GetReplayWindowUseCase(LiveEventReplayQueryPort port, TransactionPort tx)`, `@Bean ReplayLiveEventsUseCase(...)`. 템플릿은 bridge를 조립하지 않으므로 이 bean도 만들지 않는다. `LiveEventReplayPort`는 `LiveEventReplayQueryPort`로, testFixtures의 `LiveEventReplayContract`는 `LiveEventReplayQueryContract`로 이름이 바뀌었다 | 컴파일이 실패한다. web 설정에서 port를 쥐고 유스케이스를 만들면 5행 규칙에 걸린다 | stream replay를 쓰는 서비스 | +| 4 | `OperationCommand`에서 operation id 인자를 지우고, tenant를 `Optional`(`application.tenant.TenantId`)로 넘긴다. `GetDurableOperationQuery`·`CancelDurableOperationCommand`의 tenant도 같다. `SubmitDurableOperationUseCase`는 생성자에 `DurableOperationIdFactory`를 받아 id를 직접 만든다. `DurableOperationController`는 id factory를 받지 않는다 | 컴파일이 실패한다 | durable operation을 직접 제출·조회하거나 그 use case를 조립한 서비스 | +| 5 | 사용자 전송 패키지(시스템이 부르는 `..adapter.inbound.messaging..`·`..adapter.inbound.scheduler..`를 뺀 모든 `..adapter.inbound..`)의 클래스가 `*Repository`·`*StorePort`·`*QueryPort`·`*IdFactory`·`*IdGeneratorPort`·`*IdentifierPort`를 직접 쥐면 새 아키텍처 규칙 `USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS`가 빌드를 실패시킨다. 또 inbound 클래스가 application의 `*Service`·`*Orchestrator`·`*Coordinator`를 interface로 쥐어도 `INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES`가 실패시킨다(전에는 구체 클래스만 봤다). 읽기·쓰기는 유스케이스를 거친다 | `architectureTest`가 실패한다 | 사용자 전송에서 포트나 서비스를 직접 쓴 서비스 | +| 6 | `DefaultNotificationRetryPolicy`를 직접 만들던 코드는 `new DefaultNotificationRetryPolicy(timing, jitter)`로 바꾼다. `timing`은 새 값 `RetryTiming(backoff, reconcileDelay)`이고 배포 설정의 값은 `NotificationPlatformSettings.retry().timing()`이다(기본값은 `RetryTiming.DEFAULT`). `RetryBackoff`·`RetryBudget`·`RetryContext`·`RetryDecision.RetryAfter`·`RouteCandidate`·`RoutingContext`·`RoutingDecision`·`EligibilityResult`의 생성자와 `RetryBackoff.delay`(attempt 번호 1 미만)는 잘못된 값에 `IllegalStateException`을 던진다(전에는 `IllegalArgumentException`). 새 값 `RetryTiming`도 같다 | 컴파일이 실패한다. IAE를 잡던 코드는 그 예외를 놓친다 | 알림 retry policy를 직접 만들거나, route planner·plan codec·eligibility policy를 직접 구현하거나, `RoutingDecision`을 직접 만드는 서비스 | +| 7 | 쓰는 곳이 없던 공개 타입 일곱을 지웠다: `application.transaction.ReadConsistency`, `FileserverSpans`, 알림의 `ProviderEventId`·`NotificationExpiredException`·`BatchNotificationProviderAdapter`·`CollapseCapability`·`ProviderCollapseMapping` | 그 타입을 쓴 코드는 컴파일이 실패한다. 템플릿 안에는 쓰는 곳이 없었다 | 그 타입을 쓴 서비스 | ## 3. 바뀐 동작 (따라 할 일 없음) +- 알림 재시도 시각이 설정이다: `app.notification.platform.retry.backoff-base`(1s)·`backoff-max`(5m)·`reconcile-delay`(1m). + 기본값은 전의 상수와 같고 코드(`RetryTiming.DEFAULT`)에만 있다. module-defaults에는 주석 예시만 있다. 값이 틀리면 + (`backoff-base` 0 이하, `backoff-max`가 base보다 짧음, `reconcile-delay` 음수) 기동이 실패한다. +- fileserver cleanup worker와 알림 worker는 `@SystemInvoked` 유스케이스를 거쳐 쓴다(manual 07 「시스템이 부르는 쓰기」). + 알림 worker의 lease claim과 reconciliation job 기록은 이제 쓰기 트랜잭션 안에서, scheduler의 backlog 측정과 replay의 + ledger 읽기는 읽기 트랜잭션 안에서 돈다. provider에는 여전히 트랜잭션 없이 묻는다. +- 운영자 redrive는 대상 attempt를 claim·쓰기와 같은 쓰기 트랜잭션에서 읽는다(dry run은 읽기 트랜잭션). 운영자 + reconcile의 tenant 확인은 읽기 트랜잭션에서 한다. 전에는 트랜잭션 밖에서 읽었다. +- fileserver 정리·맞추기 서비스(`CleanupService`·`FileReconciliationService`)는 `composition:fileserver-platform`이 조립한다. + bean 이름과 타입은 그대로다. +- fileserver 관리 HTTP의 기본값(orphan 정리 dry run과 1 GiB·`ORPHAN_APPLY`, cleanup 100개·1 GiB, 목록 100개)은 application + command의 `requested(...)`가 채운다. 응답은 같다. - `GetDurableOperationUseCase`·`PublishPendingOutboxEventsUseCase`가 final이 아니다. - 알림 받기의 `idempotencyKey`가 `IdempotencyExecutor`와 다른 이유는 manual 05 「executor를 쓰지 않는 중복 판정 둘」에 있다. diff --git a/src/adapter/inbound/web-advanced/build.gradle b/src/adapter/inbound/web-advanced/build.gradle index 87d283a5..56dbe5dd 100644 --- a/src/adapter/inbound/web-advanced/build.gradle +++ b/src/adapter/inbound/web-advanced/build.gradle @@ -20,6 +20,8 @@ dependencies { testImplementation 'org.springframework.boot:spring-boot-starter-webmvc-test' implementation project(':adapter:inbound:web') implementation project(':application-core') + // DirectTransactionPort, the read boundary the replay bridge's query use cases open. + testImplementation testFixtures(project(':application-core')) implementation project(':shared-contract') implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-validation' diff --git a/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/stream/replay/MessagingReplayBridge.java b/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/stream/replay/MessagingReplayBridge.java index 40abdf8b..2400adbd 100644 --- a/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/stream/replay/MessagingReplayBridge.java +++ b/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/stream/replay/MessagingReplayBridge.java @@ -1,8 +1,11 @@ package dev.caskeleton.adapter.inbound.web.advanced.stream.replay; import dev.caskeleton.adapter.inbound.web.advanced.stream.StreamSequence; -import dev.caskeleton.application.realtime.LiveEventReplayPort; +import dev.caskeleton.application.realtime.GetReplayWindowQuery; +import dev.caskeleton.application.realtime.GetReplayWindowUseCase; import dev.caskeleton.application.realtime.ReplayCursorUnavailableException; +import dev.caskeleton.application.realtime.ReplayLiveEventsQuery; +import dev.caskeleton.application.realtime.ReplayLiveEventsUseCase; import dev.caskeleton.application.realtime.ReplayWindow; import java.util.List; import java.util.Objects; @@ -12,9 +15,11 @@ import java.util.function.Function; * {@link WebStreamReplaySource} over the application's durable event log. * *

The web module stores no history, and this is what that decision looks like in code: the - * bridge holds a port and a decoder and nothing else. A copy here would have its own retention, its - * own eviction and its own opinion about ordering, and the two would diverge silently because both - * would look plausible. + * bridge holds the two read use cases over the event log and a decoder, and nothing else. It reads + * through them rather than through {@code LiveEventReplayQueryPort}: every read a transport makes + * goes through a query use case, which declares its capability and opens the read boundary (round9 + * review 2026-11-12, RR9-05-10). A copy here would have its own retention, its own eviction and its + * own opinion about ordering, and the two would diverge silently because both would look plausible. * *

An expired cursor becomes {@link ReplayCursorExpiredException} rather than an empty replay. * Serving from the oldest available position would deliver contiguous positions with a hole in the @@ -28,14 +33,18 @@ public final class MessagingReplayBridge implements WebStreamReplaySource /** How many events one replay call loads. The port caps this again on its own side. */ public static final int PAGE_SIZE = 200; - private final LiveEventReplayPort replay; + private final GetReplayWindowUseCase window; + private final ReplayLiveEventsUseCase replay; private final Function decoder; /** - * @param replay the durable history + * @param window what a stream still holds + * @param replay the events after a position * @param decoder turns a stored payload into the item the stream carries */ - public MessagingReplayBridge(LiveEventReplayPort replay, Function decoder) { + public MessagingReplayBridge( + GetReplayWindowUseCase window, ReplayLiveEventsUseCase replay, Function decoder) { + this.window = Objects.requireNonNull(window, "window"); this.replay = Objects.requireNonNull(replay, "replay"); this.decoder = Objects.requireNonNull(decoder, "decoder"); } @@ -65,7 +74,9 @@ public final class MessagingReplayBridge implements WebStreamReplaySource throw new ReplayCursorExpiredException(cursor); } try { - return replay.replayAfter(position.streamId(), position.value(), PAGE_SIZE).stream() + return replay + .handle(new ReplayLiveEventsQuery(position.streamId(), position.value(), PAGE_SIZE)) + .stream() .map( event -> new Replayed<>( @@ -91,7 +102,7 @@ public final class MessagingReplayBridge implements WebStreamReplaySource private java.util.Optional window(Position position) { try { - return java.util.Optional.of(replay.window(position.streamId())); + return java.util.Optional.of(window.handle(new GetReplayWindowQuery(position.streamId()))); } catch (RuntimeException unavailable) { // Empty means "no answer", not "no history". The caller turns that into a resnapshot, which // is expensive and correct; treating it as a readable empty window would say the opposite. diff --git a/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/stream/replay/MessagingReplayBridgeTest.java b/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/stream/replay/MessagingReplayBridgeTest.java index a5025518..d1eb52e0 100644 --- a/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/stream/replay/MessagingReplayBridgeTest.java +++ b/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/stream/replay/MessagingReplayBridgeTest.java @@ -4,10 +4,14 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import dev.caskeleton.adapter.inbound.web.advanced.stream.StreamSequence; -import dev.caskeleton.application.realtime.LiveEventReplayPort; +import dev.caskeleton.application.realtime.GetReplayWindowUseCase; +import dev.caskeleton.application.realtime.LiveEventReplayQueryPort; import dev.caskeleton.application.realtime.ReplayCursorUnavailableException; +import dev.caskeleton.application.realtime.ReplayLiveEventsUseCase; import dev.caskeleton.application.realtime.ReplayWindow; import dev.caskeleton.application.realtime.ReplayedEvent; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -30,10 +34,16 @@ class MessagingReplayBridgeTest { private static final Instant NOW = Instant.parse("2026-08-25T10:00:00Z"); - private static MessagingReplayBridge bridge(LiveEventReplayPort port) { - return new MessagingReplayBridge<>(port, payload -> payload); + private static MessagingReplayBridge bridge(LiveEventReplayQueryPort port) { + return new MessagingReplayBridge<>( + new GetReplayWindowUseCase(port, PASS_THROUGH), + new ReplayLiveEventsUseCase(port, PASS_THROUGH), + payload -> payload); } + /** The storage and its read boundary are the JPA adapter's to test; this runs the read inline. */ + private static final TransactionPort PASS_THROUGH = new DirectTransactionPort(); + @Test @DisplayName("a cursor carries both the stream and the position") void cursorCarriesStreamAndPosition() { @@ -149,7 +159,7 @@ class MessagingReplayBridgeTest { } /** A replay port over a fixed window and a fixed event list. */ - private static final class StubPort implements LiveEventReplayPort { + private static final class StubPort implements LiveEventReplayQueryPort { private final long earliest; private final long latest; @@ -180,7 +190,7 @@ class MessagingReplayBridgeTest { } /** A replay port that is down. */ - private static final class FailingPort implements LiveEventReplayPort { + private static final class FailingPort implements LiveEventReplayQueryPort { @Override public ReplayWindow window(String streamId) { diff --git a/src/adapter/inbound/web-fileserver/src/main/java/dev/caskeleton/adapter/inbound/web/fileserver/admin/FileserverAdminController.java b/src/adapter/inbound/web-fileserver/src/main/java/dev/caskeleton/adapter/inbound/web/fileserver/admin/FileserverAdminController.java index 4cd3b340..333af414 100644 --- a/src/adapter/inbound/web-fileserver/src/main/java/dev/caskeleton/adapter/inbound/web/fileserver/admin/FileserverAdminController.java +++ b/src/adapter/inbound/web-fileserver/src/main/java/dev/caskeleton/adapter/inbound/web/fileserver/admin/FileserverAdminController.java @@ -29,6 +29,7 @@ import dev.caskeleton.application.fileserver.port.in.ReverifyFileCommand; import dev.caskeleton.application.fileserver.port.in.ReverifyFileUseCase; import jakarta.validation.Valid; import java.util.List; +import java.util.Optional; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -57,8 +58,6 @@ import org.springframework.web.bind.annotation.RestController; havingValue = "true") public class FileserverAdminController { - private static final int DEFAULT_PAGE = 100; - private final GetStorageHealthUseCase storageHealth; private final GetRuntimeCapabilitiesUseCase capabilities; private final ListOrphansUseCase orphans; @@ -105,8 +104,9 @@ public class FileserverAdminController { } @GetMapping(path = "/internal/fileserver/orphans", produces = MediaType.APPLICATION_JSON_VALUE) - public List orphans(@RequestParam(name = "limit", defaultValue = "100") int limit) { - return orphans.handle(new ListOrphansQuery(limit, contextFactory.current())); + public List orphans(@RequestParam(name = "limit", required = false) Integer limit) { + return orphans.handle( + ListOrphansQuery.requested(Optional.ofNullable(limit), contextFactory.current())); } @PostMapping( @@ -141,40 +141,36 @@ public class FileserverAdminController { path = "/internal/fileserver/uploads/incomplete", produces = MediaType.APPLICATION_JSON_VALUE) public List incompleteUploads( - @RequestParam(name = "limit", defaultValue = "100") int limit) { + @RequestParam(name = "limit", required = false) Integer limit) { return incompleteUploads.handle( - new ListIncompleteUploadsQuery(limit, contextFactory.current())); + ListIncompleteUploadsQuery.requested(Optional.ofNullable(limit), contextFactory.current())); } @PostMapping( path = "/internal/fileserver/uploads:cleanup", produces = MediaType.APPLICATION_JSON_VALUE) public CleanupBatchResult cleanupUploads( - @RequestParam(name = "maxItems", defaultValue = "100") int maxItems, - @RequestParam(name = "maxBytes", defaultValue = "1073741824") long maxBytes) { + @RequestParam(name = "maxItems", required = false) Integer maxItems, + @RequestParam(name = "maxBytes", required = false) Long maxBytes) { return cleanupUploads.handle( - new CleanupUploadsCommand(maxItems, maxBytes, contextFactory.current())); + CleanupUploadsCommand.requested( + Optional.ofNullable(maxItems), + Optional.ofNullable(maxBytes), + contextFactory.current())); } /** - * Reads the wire request conservatively. - * - *

Every absent field resolves to the safe value: a dry run, the default page, and no - * fingerprints. Nothing here can be omitted into a destructive default. + * Reads the wire request: absent fields stay absent, and the command applies the use case's safe + * defaults ({@link OrphanReconcileCommand#requested}). */ private static OrphanReconcileCommand toCommand( OrphanReconcileHttpRequest request, RequestContext context) { - boolean dryRun = request.dryRun() == null || request.dryRun(); - int limit = request.limit() == null ? DEFAULT_PAGE : request.limit(); - if (dryRun) { - return OrphanReconcileCommand.dryRun(limit, context); - } - return new OrphanReconcileCommand( - false, - limit, - request.maxBytes() == null ? 1L << 30 : request.maxBytes(), - request.expectedFingerprints() == null ? List.of() : request.expectedFingerprints(), - request.reasonCode() == null ? "ORPHAN_APPLY" : request.reasonCode(), + return OrphanReconcileCommand.requested( + Optional.ofNullable(request.dryRun()), + Optional.ofNullable(request.limit()), + Optional.ofNullable(request.maxBytes()), + Optional.ofNullable(request.expectedFingerprints()), + Optional.ofNullable(request.reasonCode()), context); } } diff --git a/src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/autoconfigure/FileserverCapabilityAutoConfiguration.java b/src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/autoconfigure/FileserverCapabilityAutoConfiguration.java index 9c93d01f..d1a4f934 100644 --- a/src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/autoconfigure/FileserverCapabilityAutoConfiguration.java +++ b/src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/autoconfigure/FileserverCapabilityAutoConfiguration.java @@ -42,8 +42,7 @@ import org.springframework.core.env.Environment; FileserverStorageGatewayConfiguration.class, FileserverStorageOperationsConfiguration.class, FileserverVerificationConfiguration.class, - FileserverSecurityConfiguration.class, - FileserverMaintenanceServiceConfiguration.class + FileserverSecurityConfiguration.class }) public class FileserverCapabilityAutoConfiguration { diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformDispatchConfig.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformDispatchConfig.java index d12aab0c..fe0d1fc4 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformDispatchConfig.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformDispatchConfig.java @@ -2,8 +2,6 @@ package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure; import dev.caskeleton.adapter.outbound.notification.platform.dispatch.NotificationDispatchProperties; import dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderRuntimeRegistry; -import dev.caskeleton.application.notification.platform.admin.AdminAuthorizationGuard; -import dev.caskeleton.application.notification.platform.admin.DuplicateRiskGuard; import dev.caskeleton.application.notification.platform.admin.ProviderRuntimeControlPort; import dev.caskeleton.application.notification.platform.callback.ProviderEventLedger; import dev.caskeleton.application.notification.platform.callback.ProviderEventProjectionService; @@ -41,11 +39,11 @@ import org.springframework.context.annotation.Configuration; matchIfMissing = true) class NotificationPlatformDispatchConfig { /** - * The operator plane, and the narrow port it reaches runtimes through. + * The narrow port the operator plane reaches provider runtimes through. * - *

Neither had a bean. The admin service existed in the outbound adapter, fully written, doing - * authorization and transaction orchestration, and nothing constructed it — so every operator - * capability the platform documented was unreachable. + *

The operator use cases themselves, and the authority and duplicate-risk guards they hold, + * are assembled in {@code composition:notification-platform}: who may perform an operator action + * is a decision about the product, not about speaking a provider protocol. * * @param runtimes the provider runtime registry * @return the control port @@ -56,26 +54,6 @@ class NotificationPlatformDispatchConfig { .RegistryProviderRuntimeControl(runtimes); } - /** - * Operator authority, tenant scope and duplicate-risk approval. - * - * @return the guard - */ - @Bean - public AdminAuthorizationGuard adminAuthorizationGuard() { - return new AdminAuthorizationGuard(); - } - - /** - * The approval an ambiguous redrive requires. - * - * @return the guard - */ - @Bean - public DuplicateRiskGuard duplicateRiskGuard() { - return new DuplicateRiskGuard(); - } - /** Pre-provider guard chain. */ @Bean public DispatchPipeline dispatchPipeline( diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformPolicyConfig.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformPolicyConfig.java index d0fafbf6..365e8d8b 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformPolicyConfig.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformPolicyConfig.java @@ -13,12 +13,10 @@ import dev.caskeleton.application.notification.platform.policy.JitterSource; import dev.caskeleton.application.notification.platform.policy.NotificationEligibilityPolicy; import dev.caskeleton.application.notification.platform.policy.NotificationRetryPolicy; import dev.caskeleton.application.notification.platform.policy.PreferenceStorePort; -import dev.caskeleton.application.notification.platform.policy.RetryBackoff; import dev.caskeleton.application.notification.platform.policy.RoutingDecisionEngine; import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort; import java.security.SecureRandom; import java.time.Clock; -import java.time.Duration; import java.util.Optional; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Qualifier; @@ -83,13 +81,18 @@ class NotificationPlatformPolicyConfig { .orElseGet(NotificationEligibilityPolicy::permitAll)); } - /** Evidence-aware retry policy. */ + /** + * Evidence-aware retry policy, timed by {@code app.notification.platform.retry}. + * + * @param settings the bound platform settings + * @return the policy + */ @Bean @ConditionalOnMissingBean - public NotificationRetryPolicy notificationRetryPolicy() { + public NotificationRetryPolicy notificationRetryPolicy(NotificationPlatformSettings settings) { SecureRandom random = new SecureRandom(); JitterSource jitter = bound -> bound <= 1 ? 0 : Math.floorMod(random.nextLong(), bound); - return new DefaultNotificationRetryPolicy(RetryBackoff.DEFAULT, jitter, Duration.ofMinutes(1)); + return new DefaultNotificationRetryPolicy(settings.retry().timing(), jitter); } /** Stable routing engine. */ diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformRuntimeConfig.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformRuntimeConfig.java index 7c692b3c..46cdef06 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformRuntimeConfig.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformRuntimeConfig.java @@ -32,7 +32,6 @@ import org.springframework.context.annotation.Import; NotificationPlatformProviderConfig.class, NotificationPlatformPolicyConfig.class, NotificationPlatformDispatchConfig.class, - NotificationPlatformWorkerConfig.class, NotificationCallbackProviderConfig.class, NotificationSesProviderConfig.class, // The SMTP provider family. It carries its own narrower condition — a relay must be configured — diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettings.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettings.java index 19a86e65..469f7054 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettings.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettings.java @@ -1,5 +1,7 @@ package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure; +import dev.caskeleton.application.notification.platform.policy.RetryBackoff; +import dev.caskeleton.application.notification.platform.policy.RetryTiming; import java.net.URI; import java.time.Duration; import java.util.List; @@ -22,6 +24,7 @@ public record NotificationPlatformSettings( Dispatch dispatch, Callbacks callbacks, ServingThresholds servingThresholds, + Retry retry, Map providers) { @ConstructorBinding @@ -34,6 +37,7 @@ public record NotificationPlatformSettings( callbacks = callbacks == null ? Callbacks.defaults() : callbacks; servingThresholds = servingThresholds == null ? ServingThresholds.defaults() : servingThresholds; + retry = retry == null ? Retry.defaults() : retry; providers = providers == null ? Map.of() : Map.copyOf(providers); boolean callbacksEnabled = callbacks.enabled(); providers.forEach((id, provider) -> provider.validate(id, callbacksEnabled)); @@ -51,6 +55,68 @@ public record NotificationPlatformSettings( this(enabled, mode, dispatch, callbacks, ServingThresholds.defaults(), providers); } + /** The settings without a retry block: the shipped retry timing. */ + public NotificationPlatformSettings( + boolean enabled, + NotificationPlatformMode mode, + Dispatch dispatch, + Callbacks callbacks, + ServingThresholds servingThresholds, + Map providers) { + this(enabled, mode, dispatch, callbacks, servingThresholds, Retry.defaults(), providers); + } + + /** + * How long a failed delivery waits before its next attempt, and how long an ambiguous one waits + * before the provider is asked what happened. + * + *

Settings rather than constants in the policy's assembly: how fast to retry a provider is a + * deployment's call, made against that provider's own rate limits (round9 review 2026-11-12, + * RR9-05-08). + * + * @param backoffBase the first retry's ceiling and the shortest delay + * @param backoffMax the longest delay, a provider's {@code Retry-After} included + * @param reconcileDelay how long after an ambiguous submission reconciliation first asks + */ + public record Retry(Duration backoffBase, Duration backoffMax, Duration reconcileDelay) { + + /** + * Binds the block; a key left out takes the shipped value. The defaults and the rules are the + * retry policy's own {@link RetryTiming}: this builds one, which refuses what is not a timing, + * so neither the defaults nor the rules have a second home here. + */ + @ConstructorBinding + public Retry { + backoffBase = backoffBase == null ? RetryTiming.DEFAULT.backoff().base() : backoffBase; + backoffMax = backoffMax == null ? RetryTiming.DEFAULT.backoff().max() : backoffMax; + reconcileDelay = + reconcileDelay == null ? RetryTiming.DEFAULT.reconcileDelay() : reconcileDelay; + timing(backoffBase, backoffMax, reconcileDelay); + } + + /** + * The shipped timing. + * + * @return the defaults + */ + public static Retry defaults() { + return new Retry(null, null, null); + } + + /** + * The timing the retry policy uses. + * + * @return the timing of these settings + */ + public RetryTiming timing() { + return timing(backoffBase, backoffMax, reconcileDelay); + } + + private static RetryTiming timing(Duration base, Duration max, Duration reconcileDelay) { + return new RetryTiming(new RetryBackoff(base, max), reconcileDelay); + } + } + /** Dispatch runtime bounds. */ public record Dispatch( int claimBatchSize, diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformWorkerConfig.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformWorkerConfig.java deleted file mode 100644 index ba78083f..00000000 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformWorkerConfig.java +++ /dev/null @@ -1,175 +0,0 @@ -package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure; - -import dev.caskeleton.adapter.outbound.notification.platform.dispatch.NotificationBackgroundWorkers; -import dev.caskeleton.adapter.outbound.notification.platform.dispatch.NotificationDispatchProperties; -import dev.caskeleton.adapter.outbound.notification.platform.dispatch.NotificationSchedulerWorker; -import dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderEventReplayWorker; -import dev.caskeleton.adapter.outbound.notification.platform.dispatch.ReconciliationJobWorker; -import dev.caskeleton.application.notification.platform.callback.ProviderEventLedger; -import dev.caskeleton.application.notification.platform.callback.ProviderEventProjectionService; -import dev.caskeleton.application.notification.platform.dispatch.LeaseRecoveryService; -import dev.caskeleton.application.notification.platform.dispatch.NotificationDispatchService; -import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort; -import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJobStorePort; -import dev.caskeleton.application.notification.platform.dispatch.ReconciliationService; -import dev.caskeleton.application.notification.platform.observation.NotificationMetricsPort; -import dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort; -import java.util.UUID; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * Background workers and their lifecycle. - * - *

Everything that runs on a schedule rather than on a request. They are grouped so that a - * deployment reading its own configuration can see, in one place, what this process starts. - * - *

Split out of one 690-line composition class holding 42 beans. A configuration that assembles - * codecs, security, templates, policy, providers and workers at once has no boundary a reader can - * use: every change to any of them is a change to the same file, and nothing states which beans - * belong together (NTF-022). - */ -@Configuration(proxyBeanMethods = false) -@ConditionalOnProperty( - prefix = "app.notification.platform", - name = "enabled", - havingValue = "true", - matchIfMissing = true) -class NotificationPlatformWorkerConfig { - - /* - * Queue worker. Started by the application lifecycle, stopped on shutdown. - * - *

The owner carries a per-process nonce, not just a configured name. Every instance used to - * call itself {@code notification-worker-1}, so a restarted process and the one it replaced were - * the same owner as far as the database was concerned — and a stale process's release could clear - * the lease its successor was working under. The nonce makes each incarnation distinguishable; - * the lease fence handles the rest. - */ - - /** - * Asks providers what happened to attempts whose outcome is unknown. - * - * @param jobs the outstanding questions - * @param reconciliation the reconciliation service - * @param clock the clock - * @param properties the dispatch bounds - * @return the worker - */ - @Bean - @ConditionalOnMissingBean - public ReconciliationJobWorker reconciliationJobWorker( - ReconciliationJobStorePort jobs, - ReconciliationService reconciliation, - java.time.Clock clock, - NotificationDispatchProperties properties) { - return new ReconciliationJobWorker( - jobs, - reconciliation::reconcile, - clock, - properties.leaseDuration(), - // The claim lease reuses the delivery lease rather than adding a setting. One pass asks - // up to claimBatchSize providers in turn, so a pass slower than the lease lets another - // instance re-ask its remaining jobs; the claim fence keeps those answers recorded once. - properties.leaseDuration(), - properties.claimBatchSize(), - properties.maxAdditionalAttempts() + 1); - } - - /** - * Projects provider events that were stored and never applied. - * - * @param ledger the stored events - * @param projection the projection service - * @param properties the dispatch bounds - * @return the replay worker - */ - @Bean - @ConditionalOnMissingBean - public ProviderEventReplayWorker notificationProviderEventReplayWorker( - ProviderEventLedger ledger, - ProviderEventProjectionService projection, - NotificationDispatchProperties properties) { - return new ProviderEventReplayWorker(ledger, projection, properties.claimBatchSize()); - } - - /** - * Everything that starts a thread, and therefore everything {@code INGEST_ONLY} must not have. - * - *

A nested configuration rather than a condition on each bean. Both beans below used to sit on - * the parent, which carried the capability switch alone, and both call {@code start()} in the - * factory method — so a deployment in {@code INGEST_ONLY}, the mode whose whole purpose is to - * accept and store without delivering, started the dispatch scheduler, the lease recovery pass, - * the provider-event replay worker and the reconciliation job. It accepted requests and then - * tried to deliver them, which is the mode not existing. - * - *

Structural because a per-bean condition is one a future bean can forget, and the cost of - * forgetting here is not a stray bean — it is a process delivering notifications that an operator - * put into a drain. A configuration that is not loaded cannot be forgotten. - * - *

{@code matchIfMissing = true} because {@code SERVING} is the shipped default: an absent mode - * must still deliver, or a missing property becomes a silent drain. - */ - @Configuration(proxyBeanMethods = false) - @ConditionalOnProperty( - prefix = "app.notification.platform", - name = "mode", - havingValue = "SERVING", - matchIfMissing = true) - public static class ServingWorkerConfig { - - /** - * Schedules the recovery and replay passes, and stops them on shutdown. - * - * @param recovery the recovery service - * @param replay the replay worker - * @param reconciliation the reconciliation worker - * @param properties the dispatch bounds - * @return the started background workers - */ - @Bean(destroyMethod = "close") - @ConditionalOnMissingBean - public NotificationBackgroundWorkers notificationBackgroundWorkers( - LeaseRecoveryService recovery, - ProviderEventReplayWorker replay, - ReconciliationJobWorker reconciliation, - NotificationDispatchProperties properties) { - NotificationBackgroundWorkers workers = - new NotificationBackgroundWorkers( - recovery, - replay, - reconciliation, - properties.leaseDuration(), - properties.shutdownGrace()); - workers.start(); - return workers; - } - - /** - * The dispatch scheduler. - * - * @param leases the recipient lease store - * @param dispatcher the dispatch service - * @param metrics the metrics port - * @param properties the dispatch bounds - * @param servingState the serving-state port - * @return the started scheduler worker - */ - @Bean(destroyMethod = "close") - public NotificationSchedulerWorker notificationSchedulerWorker( - RecipientLeaseStorePort leases, - NotificationDispatchService dispatcher, - NotificationMetricsPort metrics, - NotificationDispatchProperties properties, - NotificationServingStatePort servingState) { - String owner = "notification-worker-1:" + UUID.randomUUID(); - NotificationSchedulerWorker worker = - new NotificationSchedulerWorker( - leases, dispatcher, metrics, properties, servingState, owner); - worker.start(); - return worker; - } - } -} diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorker.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorker.java deleted file mode 100644 index 7d6c7490..00000000 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorker.java +++ /dev/null @@ -1,144 +0,0 @@ -package dev.caskeleton.adapter.outbound.notification.platform.dispatch; - -import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJob; -import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJobStorePort; -import dev.caskeleton.application.notification.platform.provider.ReconciliationResult; -import java.time.Clock; -import java.time.Duration; -import java.util.List; -import java.util.Objects; -import java.util.function.Function; - -/** - * Asks the provider what happened to attempts whose outcome is unknown. - * - *

Reconciliation only ever ran when a lease expired and recovery walked past an incomplete - * attempt. An ambiguous submission whose worker exited cleanly — the common case, because a worker - * that records AMBIGUOUS and then finishes its shift has not crashed — was never asked about again. - * The delivery sat {@code RECONCILIATION_REQUIRED} indefinitely, which reads as a queue that - * stopped rather than as an outcome nobody knows. - * - *

An unsupported provider is not retried in a loop. Without a status-query capability the answer - * will not change, so the job is completed and the attempt stays visibly ambiguous for an operator - * — a busy loop against a capability that does not exist is how a background worker burns a - * connection pool while achieving nothing. - * - *

Every instance runs this worker, so the claim is a lease: the jobs a pass takes are this - * instance's until {@code claimLease} has passed, and no transaction is held while the provider is - * asked. A pass that outlives its lease may find a job taken over by another instance; the store - * then ignores this pass's write for it, so the job is asked about twice but recorded once. - */ -public final class ReconciliationJobWorker { - - private final ReconciliationJobStorePort jobs; - private final Function< - dev.caskeleton.application.notification.platform.api.DeliveryAttemptId, - ReconciliationResult> - reconciler; - private final Clock clock; - private final Duration retryBackoff; - private final Duration claimLease; - private final int batchSize; - private final int maxAttempts; - - /** - * Creates the worker. - * - * @param jobs the outstanding questions - * @param reconciler asks one attempt's provider - * @param clock the clock - * @param retryBackoff how long to wait before asking again - * @param claimLease how long one pass owns the jobs it claimed; must cover the whole pass - * @param batchSize how many jobs one pass handles - * @param maxAttempts how many times one job may ask before it is left to an operator - */ - public ReconciliationJobWorker( - ReconciliationJobStorePort jobs, - Function< - dev.caskeleton.application.notification.platform.api.DeliveryAttemptId, - ReconciliationResult> - reconciler, - Clock clock, - Duration retryBackoff, - Duration claimLease, - int batchSize, - int maxAttempts) { - this.jobs = Objects.requireNonNull(jobs, "jobs"); - this.reconciler = Objects.requireNonNull(reconciler, "reconciler"); - this.clock = Objects.requireNonNull(clock, "clock"); - this.retryBackoff = Objects.requireNonNull(retryBackoff, "retryBackoff"); - if (retryBackoff.isNegative() || retryBackoff.isZero()) { - throw new IllegalArgumentException("retryBackoff must be positive and finite"); - } - this.claimLease = Objects.requireNonNull(claimLease, "claimLease"); - if (claimLease.isNegative() || claimLease.isZero()) { - throw new IllegalArgumentException("claimLease must be positive"); - } - if (batchSize < 1) { - throw new IllegalArgumentException("batchSize"); - } - if (maxAttempts < 1) { - throw new IllegalArgumentException("maxAttempts"); - } - this.batchSize = batchSize; - this.maxAttempts = maxAttempts; - } - - /** - * Runs one pass. - * - * @return how many jobs reached a terminal decision - */ - public int reconcileOnce() { - List due = jobs.claimDue(batchSize, clock.instant(), claimLease); - int settled = 0; - for (ReconciliationJob job : due) { - // One job's failure is not the pass's: a provider that is refusing connections would - // otherwise stop every other provider's jobs behind it. - try { - if (handle(job)) { - settled++; - } - } catch (RuntimeException failure) { - jobs.reschedule( - job, failure.getClass().getSimpleName(), clock.instant().plus(retryBackoff)); - } - } - return settled; - } - - private boolean handle(ReconciliationJob job) { - if (job.attempts() >= maxAttempts) { - // Asked enough times. Completing the job leaves the attempt ambiguous and visible rather - // than asking forever; an outcome that has not arrived after this many tries is an - // operator's decision, not a scheduler's. - jobs.reschedule(job, "MAX_ATTEMPTS", clock.instant().plus(retryBackoff)); - jobs.complete(job); - return true; - } - ReconciliationResult result = reconciler.apply(job.attemptId()); - return switch (result) { - case ReconciliationResult.Confirmed ignored -> { - jobs.complete(job); - yield true; - } - case ReconciliationResult.Unsupported ignored -> { - // The capability does not exist; asking again cannot change that. - jobs.complete(job); - yield true; - } - case ReconciliationResult.StillUnknown stillUnknown -> { - jobs.reschedule(job, "STILL_UNKNOWN", stillUnknown.nextCheckAt()); - yield false; - } - case ReconciliationResult.Failed failed -> { - if (failed.retryable()) { - jobs.reschedule(job, failed.code(), clock.instant().plus(retryBackoff)); - yield false; - } - jobs.complete(job); - yield true; - } - }; - } -} diff --git a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformCoreConfigurationTest.java b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformCoreConfigurationTest.java index a95f7e96..efe78f6d 100644 --- a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformCoreConfigurationTest.java +++ b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformCoreConfigurationTest.java @@ -5,13 +5,17 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.adapter.outbound.notification.platform.security.AesGcmContactPointProtector; import dev.caskeleton.adapter.outbound.notification.platform.template.NotificationTemplateEngine; import dev.caskeleton.adapter.outbound.notification.platform.template.PlaceholderTemplateEngine; -import dev.caskeleton.application.notification.platform.admin.NotificationAdminService; +import dev.caskeleton.application.notification.platform.admin.ChangeProviderStateUseCase; +import dev.caskeleton.application.notification.platform.admin.ChangeSuppressionUseCase; +import dev.caskeleton.application.notification.platform.admin.ReconcileDeliveriesUseCase; +import dev.caskeleton.application.notification.platform.admin.RedriveDeliveryUseCase; import dev.caskeleton.application.notification.platform.security.ContactPointProtector; import dev.caskeleton.application.notification.platform.security.SecretKeyMaterial; import dev.caskeleton.application.notification.platform.security.SecretMaterialProvider; import dev.caskeleton.application.notification.platform.security.SecretPurpose; import dev.caskeleton.application.notification.platform.template.TemplateVariableValidator; import java.util.Arrays; +import java.util.stream.Stream; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.runner.ApplicationContextRunner; @@ -112,8 +116,22 @@ class NotificationPlatformCoreConfigurationTest { } @Test - void adminOperationsAreAbsentByDefaultRatherThanPresentAndUnguarded() { - enabled().run(context -> assertThat(context).doesNotHaveBean(NotificationAdminService.class)); + void theAdapterAssemblesNoOperatorUseCase() { + enabled() + .run( + context -> + assertThat( + Stream.of( + RedriveDeliveryUseCase.class, + ReconcileDeliveriesUseCase.class, + ChangeSuppressionUseCase.class, + ChangeProviderStateUseCase.class) + .filter(type -> context.getBeanNamesForType(type).length > 0) + .toList()) + .as( + "who may perform an operator action is the composition's to assemble, " + + "behind each use case's permission") + .isEmpty()); } @Test diff --git a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformImportCoverageTest.java b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformImportCoverageTest.java index f846e38a..1093f3c0 100644 --- a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformImportCoverageTest.java +++ b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformImportCoverageTest.java @@ -66,13 +66,14 @@ class NotificationPlatformImportCoverageTest { assertThat(imports).isNotNull(); assertThat(Arrays.stream(imports.value()).map(Class::getSimpleName).toList()) - .as("codec, provider, policy, dispatch and worker are the capability's working parts") + .as( + "codec, provider, policy and dispatch are the capability's working parts; the workers" + + " are the composition's") .contains( "NotificationPlatformCodecConfig", "NotificationPlatformProviderConfig", "NotificationSesProviderConfig", "NotificationPlatformPolicyConfig", - "NotificationPlatformDispatchConfig", - "NotificationPlatformWorkerConfig"); + "NotificationPlatformDispatchConfig"); } } diff --git a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettingsTest.java b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettingsTest.java index 91a446d6..038d6dbe 100644 --- a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettingsTest.java +++ b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationPlatformSettingsTest.java @@ -172,6 +172,105 @@ class NotificationPlatformSettingsTest { .hasMessageContaining("lease-duration"); } + @Test + @DisplayName("retry 블록이 없으면 backoff 1s~5m, reconcile 1분으로 묶인다") + void anAbsentRetryBlockBindsTheShippedTiming() { + // when + NotificationPlatformSettings settings = + bind(Map.of("app.notification.platform.enabled", "true")); + + // then + assertThat(settings.retry()) + .extracting( + NotificationPlatformSettings.Retry::backoffBase, + NotificationPlatformSettings.Retry::backoffMax, + NotificationPlatformSettings.Retry::reconcileDelay) + .containsExactly(Duration.ofSeconds(1), Duration.ofMinutes(5), Duration.ofMinutes(1)); + } + + @Test + @DisplayName("retry 블록에 reconcile-delay만 있으면 backoff는 1s~5m 기본값으로 묶인다") + void aPartialRetryBlockKeepsTheShippedBackoff() { + // when + NotificationPlatformSettings settings = + bind( + Map.of( + "app.notification.platform.enabled", "true", + "app.notification.platform.retry.reconcile-delay", "30s")); + + // then + assertThat(settings.retry()) + .extracting( + NotificationPlatformSettings.Retry::backoffBase, + NotificationPlatformSettings.Retry::backoffMax, + NotificationPlatformSettings.Retry::reconcileDelay) + .containsExactly(Duration.ofSeconds(1), Duration.ofMinutes(5), Duration.ofSeconds(30)); + } + + @Test + void theRetryTimingBindsFromTheDeploymentsSettings() { + // given + Map properties = + Map.of( + "app.notification.platform.enabled", "true", + "app.notification.platform.retry.backoff-base", "2s", + "app.notification.platform.retry.backoff-max", "10m", + "app.notification.platform.retry.reconcile-delay", "30s"); + + // when + NotificationPlatformSettings settings = bind(properties); + + // then + assertThat(settings.retry()) + .isEqualTo( + new NotificationPlatformSettings.Retry( + Duration.ofSeconds(2), Duration.ofMinutes(10), Duration.ofSeconds(30))); + } + + @Test + @DisplayName("backoff 상한 30s가 하한 1m보다 짧으면 기동이 retry 블록 이름과 그 이유로 실패한다") + void aBackoffCeilingBelowItsBaseFailsStartup() { + // when + Throwable failure = + catchThrowable( + () -> + bind( + Map.of( + "app.notification.platform.enabled", "true", + "app.notification.platform.retry.backoff-base", "1m", + "app.notification.platform.retry.backoff-max", "30s"))); + + // then + assertThat(failure) + .as("the policy's own timing refuses it; the binder names the block a deployer fixes") + .isInstanceOf(BindException.class) + .hasMessageContaining("'app.notification.platform.retry'") + .rootCause() + .isInstanceOf(IllegalStateException.class) + .hasMessage("the backoff max (PT30S) must not be shorter than its base (PT1M)"); + } + + @Test + @DisplayName("결과를 모르는 제출 뒤 기다림이 -1s면 기동이 retry 블록 이름과 그 이유로 실패한다") + void aNegativeReconcileDelayFailsStartup() { + // when + Throwable failure = + catchThrowable( + () -> + bind( + Map.of( + "app.notification.platform.enabled", "true", + "app.notification.platform.retry.reconcile-delay", "-1s"))); + + // then + assertThat(failure) + .isInstanceOf(BindException.class) + .hasMessageContaining("'app.notification.platform.retry'") + .rootCause() + .isInstanceOf(IllegalStateException.class) + .hasMessage("the reconcile delay must not be negative, was PT-1S"); + } + @Test void defaultsAreUsableAndConservative() { var defaults = NotificationPlatformSettings.Dispatch.defaults(); diff --git a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationWorkerLifecycleTest.java b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationWorkerLifecycleTest.java deleted file mode 100644 index 27929bf1..00000000 --- a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationWorkerLifecycleTest.java +++ /dev/null @@ -1,99 +0,0 @@ -package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.lang.reflect.Method; -import java.util.Arrays; -import java.util.List; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; - -/** - * {@code INGEST_ONLY} starts no dispatch, recovery or reconciliation thread (NTF-INT-003). - * - *

The worker configuration carried the master switch and nothing else, and both of its worker - * beans called {@code start()} in the factory method. So a deployment in {@code INGEST_ONLY} — the - * mode whose entire purpose is to accept and store without delivering — started the dispatch - * scheduler, the lease recovery pass, the provider-event replay worker and the reconciliation job. - * It accepted requests and then tried to deliver them, which is the mode not existing. - * - *

The gate is structural rather than a condition repeated per bean: the two started beans live - * in a nested configuration the parent imports only in {@code SERVING}. A condition on each bean is - * one a future bean can forget; a configuration that is not loaded cannot be forgotten. - * - *

Asserted on the bean methods' own conditions rather than by booting the platform, - * deliberately. A full notification context needs a database, provider credentials and a secrets - * resolver, and the capability has no JPA persistence wired at all yet (Section D) — a test that - * could not run would be no gate. What this checks is the structural fact that decides the thread - * behaviour; the live thread count is asserted by the {@code local-notification-ingest} Compose - * lane, which is where a running thread can actually be counted. - */ -class NotificationWorkerLifecycleTest { - - /** The bean methods that start something, and therefore may not exist outside SERVING. */ - private static final List THREAD_STARTING_BEANS = - List.of("notificationBackgroundWorkers", "notificationSchedulerWorker"); - - @Test - @DisplayName("every thread-starting worker bean sits behind the SERVING mode") - void everyThreadStartingBeanIsServingOnly() { - for (String beanMethod : THREAD_STARTING_BEANS) { - Class declaring = declaringConfigurationOf(beanMethod); - - ConditionalOnProperty condition = declaring.getAnnotation(ConditionalOnProperty.class); - - assertThat(condition) - .as( - "%s starts a thread; the configuration declaring it must carry the mode condition, or " - + "INGEST_ONLY delivers", - beanMethod) - .isNotNull(); - assertThat(condition.prefix()).isEqualTo("app.notification.platform"); - assertThat(condition.name()).containsExactly("mode"); - assertThat(condition.havingValue()).isEqualTo("SERVING"); - assertThat(condition.matchIfMissing()) - .as( - "SERVING is the shipped default, so an absent mode must still deliver; making this " - + "false would turn a missing property into a silent drain") - .isTrue(); - } - } - - @Test - @DisplayName("the ingest-side beans are not behind the SERVING mode") - void theIngestSideStaysAvailableInBothModes() { - List servingOnly = - Arrays.stream(NotificationPlatformWorkerConfig.class.getDeclaredMethods()) - .filter(method -> method.isAnnotationPresent(Bean.class)) - .map(Method::getName) - .sorted() - .toList(); - - assertThat(servingOnly) - .as( - "what stays on the parent is what INGEST_ONLY still needs; a bean moved into the " - + "SERVING child by accident would stop an ingest-only deployment from accepting") - .doesNotContainAnyElementsOf(THREAD_STARTING_BEANS) - .isNotEmpty(); - } - - private static Class declaringConfigurationOf(String beanMethod) { - for (Class candidate : NotificationPlatformWorkerConfig.class.getDeclaredClasses()) { - boolean declares = - Arrays.stream(candidate.getDeclaredMethods()) - .anyMatch( - method -> - method.getName().equals(beanMethod) - && method.isAnnotationPresent(Bean.class)); - if (declares) { - return candidate; - } - } - throw new AssertionError( - beanMethod - + " is declared by no nested configuration of NotificationPlatformWorkerConfig; a " - + "thread-starting bean on the parent runs in every mode"); - } -} diff --git a/src/adapter/outbound/persistence-jpa/build.gradle b/src/adapter/outbound/persistence-jpa/build.gradle index 9e6c196d..69b8a16a 100644 --- a/src/adapter/outbound/persistence-jpa/build.gradle +++ b/src/adapter/outbound/persistence-jpa/build.gradle @@ -132,14 +132,6 @@ dependencies { // The application-owned store contracts the notification adapters are held to, the same ones // the in-memory fakes pass (review 04-14). postgresqlIntegrationTestImplementation testFixtures(project(':application-notification')) - // The provider-event replay worker lives in the notification adapter and only fails against a - // real JPA stack with no caller transaction, so its real-DB test runs here beside the - // notification stores. Test scope only, like the Fileserver edge. Non-transitive: the worker - // needs only application-notification, which this lane already has, and the adapter's - // mail/Thymeleaf/actuator graph has no business on a database lane. - postgresqlIntegrationTestImplementation(project(':adapter:outbound:notification')) { - transitive = false - } postgresqlIntegrationTestImplementation testFixtures(project(':application-core')) postgresqlIntegrationTestImplementation 'org.testcontainers:testcontainers-postgresql' postgresqlIntegrationTestImplementation 'org.testcontainers:testcontainers-junit-jupiter' diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplay.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplay.java index d2762a63..617fc23c 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplay.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplay.java @@ -2,7 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.liveevent; import dev.caskeleton.adapter.outbound.persistence.liveevent.entity.LiveEventEntity; import dev.caskeleton.adapter.outbound.persistence.transaction.CallerTransaction; -import dev.caskeleton.application.realtime.LiveEventReplayPort; +import dev.caskeleton.application.realtime.LiveEventReplayQueryPort; import dev.caskeleton.application.realtime.ReplayCursorUnavailableException; import dev.caskeleton.application.realtime.ReplayWindow; import dev.caskeleton.application.realtime.ReplayedEvent; @@ -21,7 +21,7 @@ import org.springframework.jdbc.core.JdbcOperations; *

Durable history lives here rather than in a transport adapter for one reason: a second copy * would have its own retention, its own eviction and its own opinion about ordering, and the two * would diverge silently because both would look plausible. The web and websocket modules reach it - * through {@link LiveEventReplayPort} and hold nothing. + * through {@link LiveEventReplayQueryPort} and hold nothing. * *

Positions are assigned per stream and are never reused, including after a sweep. Reusing a * swept position would give two different events the same address, and a client holding the older @@ -33,7 +33,7 @@ import org.springframework.jdbc.core.JdbcOperations; * The counter update and event insert share this method's transaction, so a failed event insert * also rolls the allocation back. */ -public final class JpaLiveEventReplay implements LiveEventReplayPort { +public final class JpaLiveEventReplay implements LiveEventReplayQueryPort { /** The most one replay call will load, whatever the caller asks for. */ public static final int MAX_REPLAY_LIMIT = 500; diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplayContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplayContractTest.java index aa2e38ed..4964b916 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplayContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/JpaLiveEventReplayContractTest.java @@ -5,8 +5,8 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.adapter.outbound.persistence.liveevent.entity.LiveEventEntity; import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; -import dev.caskeleton.application.realtime.LiveEventReplayPort; -import dev.caskeleton.application.realtime.testkit.LiveEventReplayContract; +import dev.caskeleton.application.realtime.LiveEventReplayQueryPort; +import dev.caskeleton.application.realtime.testkit.LiveEventReplayQueryContract; import dev.caskeleton.application.transaction.TransactionPort; import jakarta.persistence.EntityManagerFactory; import java.io.IOException; @@ -45,7 +45,7 @@ import org.springframework.transaction.PlatformTransactionManager; * allocates through the position counter rather than reusing a swept position. */ @Tag("jpa-contract") -class JpaLiveEventReplayContractTest extends LiveEventReplayContract { +class JpaLiveEventReplayContractTest extends LiveEventReplayQueryContract { private static final Duration RETENTION = Duration.ofHours(1); @@ -88,7 +88,7 @@ class JpaLiveEventReplayContractTest extends LiveEventReplayContract { } @Override - protected LiveEventReplayPort replay() { + protected LiveEventReplayQueryPort replay() { return adapter(); } diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlNotificationBackgroundPassIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlNotificationBackgroundPassIntegrationTest.java index 09606e87..02818b8d 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlNotificationBackgroundPassIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlNotificationBackgroundPassIntegrationTest.java @@ -3,7 +3,6 @@ package dev.caskeleton.adapter.outbound.persistence.readiness; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; -import dev.caskeleton.adapter.outbound.notification.platform.dispatch.ProviderEventReplayWorker; import dev.caskeleton.adapter.outbound.persistence.notification.NotificationTestCodecs; import dev.caskeleton.adapter.outbound.persistence.notification.platform.JpaDeliveryAttemptStore; import dev.caskeleton.adapter.outbound.persistence.notification.platform.JpaNotificationRequestStore; @@ -41,6 +40,8 @@ import dev.caskeleton.application.notification.platform.callback.ProjectionResul import dev.caskeleton.application.notification.platform.callback.ProviderEventProjectionService; import dev.caskeleton.application.notification.platform.callback.ProviderEventProjector; import dev.caskeleton.application.notification.platform.callback.ProviderEventRecord; +import dev.caskeleton.application.notification.platform.callback.ReplayProviderEventsCommand; +import dev.caskeleton.application.notification.platform.callback.ReplayProviderEventsUseCase; import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptFactory; import dev.caskeleton.application.notification.platform.dispatch.LeaseRecoveryService; import dev.caskeleton.application.notification.platform.dispatch.NotificationRequestRecord; @@ -322,7 +323,7 @@ class PostgreSqlNotificationBackgroundPassIntegrationTest { UUID event = insertPendingEvent(null, requestHash); // when - replay().replayOnce(); + replay().handle(new ReplayProviderEventsCommand()); // then assertThat(eventRow(event)).containsExactly(attempt.toString(), "APPLIED", null); @@ -336,7 +337,7 @@ class PostgreSqlNotificationBackgroundPassIntegrationTest { UUID event = insertPendingEvent(attempt, null); // when - replay().replayOnce(); + replay().handle(new ReplayProviderEventsCommand()); // then assertThat(eventRow(event)) @@ -381,8 +382,8 @@ class PostgreSqlNotificationBackgroundPassIntegrationTest { 10); } - private ProviderEventReplayWorker replay() { - return new ProviderEventReplayWorker(ledger, projection, 10); + private ReplayProviderEventsUseCase replay() { + return new ReplayProviderEventsUseCase(projection, 10); } private JpaRecipientDeliveryStore deliveries(RecipientDeliveryJpaRepository recipients) { diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaAdminOperationStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaAdminOperationStore.java index 74728e9e..61b49e2c 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaAdminOperationStore.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaAdminOperationStore.java @@ -33,10 +33,10 @@ public final class JpaAdminOperationStore implements AdminOperationStorePort { @Override public dev.caskeleton.application.notification.platform.admin.AdminOperationClaim claim( - String operationId, String commandFingerprint, AdminActor actor, String action) { + String operationId, String requestFingerprint, AdminActor actor, String action) { CallerTransaction.require("admin operation claim"); Objects.requireNonNull(operationId, "operationId"); - Objects.requireNonNull(commandFingerprint, "commandFingerprint"); + Objects.requireNonNull(requestFingerprint, "requestFingerprint"); Objects.requireNonNull(actor, "actor"); Objects.requireNonNull(action, "action"); @@ -46,7 +46,7 @@ public final class JpaAdminOperationStore implements AdminOperationStorePort { operationId, action, actor.actorRef(), - commandFingerprint, + requestFingerprint, clock.instant()); if (claimed == 1) { return dev.caskeleton.application.notification.platform.admin.AdminOperationClaim.claimed(); @@ -61,7 +61,7 @@ public final class JpaAdminOperationStore implements AdminOperationStorePort { } String phase = String.valueOf(existing.get(0)[0]); String fingerprint = existing.get(0)[1] == null ? null : String.valueOf(existing.get(0)[1]); - if (fingerprint != null && !fingerprint.equals(commandFingerprint)) { + if (fingerprint != null && !fingerprint.equals(requestFingerprint)) { // The same id, a different command. Returning the earlier result would execute neither. return dev.caskeleton.application.notification.platform.admin.AdminOperationClaim.conflict(); } diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java index 98cd4bef..220d4acb 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java @@ -39,9 +39,10 @@ public final class JpaRecipientLeaseStore implements RecipientLeaseStorePort { } /** - * The one write here without {@code CallerTransaction.require}: the scheduler claims before any - * use case runs, and the claim is a single CTE statement that selects, locks and fences its rows - * by itself — there is no second write it has to be atomic with. + * The one write here without {@code CallerTransaction.require}: the claim is a single CTE + * statement that selects, locks and fences its rows by itself — there is no second write it has + * to be atomic with. {@code ClaimDueDeliveriesUseCase} opens a write transaction around it + * anyway, as every use case that writes does. */ @Override public List claim(String workerId, int limit, Duration leaseDuration) { diff --git a/src/application/core/AGENTS.md b/src/application/core/AGENTS.md index 9d5e5859..de330cad 100644 --- a/src/application/core/AGENTS.md +++ b/src/application/core/AGENTS.md @@ -73,11 +73,12 @@ architectureTest)가 강제한다. 나머지는 리뷰 규칙이고, 리뷰가 |---|---|---|---| | 유스케이스(inbound 포트 구현) | `UseCase`, ArchUnit이 강제 | 기능의 `application` 패키지 | `*ApplicationUseCase`(application-fileserver, application-notification), `*Service`(notification, fileserver) | | command·query record | `*Command` / `*Query` | 그 유스케이스 옆 | — | +| 요청 값 record: command가 담는 호출자 요청 내용(호출자가 누구인지는 뺀 값), 또는 포트 인자 묶음 | `*Request` | 그것을 담는 command나 받는 포트 옆 | `CreateUploadRequest`·`FinalizeUploadRequest`(application-fileserver `upload`, command는 `port.in`), `VerificationRequest`(application-fileserver `api.security`, 포트는 `upload`), application-objectstorage의 `request` 패키지(포트는 `port`, leaf가 값 타입 패키지로 정했다) | | aggregate 영속(aggregate 전체를 저장하고 돌려준다) | `Repository` | aggregate의 **domain** 패키지, 도메인이 가진다 | — | | 애플리케이션이 가진 기술 저장소(outbox, 멱등성, operation 행) | `StorePort` | application 패키지 | `*WritePort`(옛 onboarding fixture, 지움) | | aggregate가 아니라 projection을 돌려주는 읽기 모델 | `QueryPort` | application 패키지. projection만 돌려준다(ArchUnit) | — | | 그 밖의 outbound 포트 | `Port` | application 패키지 | — | -| 서버가 정하는 id 생성 | `IdFactory extends IdFactory<Id>` | domain 패키지. id record는 `implements ResourceId` | `DurableOperationIdFactory`(durable operation id는 aggregate의 resource id가 아니라 `IdFactory`를 따르지 않는다) | +| 서버가 정하는 id 생성 | `IdFactory extends IdFactory<Id>` | domain 패키지. id record는 `implements ResourceId` | `DurableOperationIdFactory`(durable operation id는 aggregate의 resource id가 아니라 `IdFactory`를 따르지 않는다), `NotificationIdGeneratorPort`(application-notification), `UploadIdentifierPort`(application-fileserver). 셋 다 사용자 전송이 쥐면 `USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS`가 막는다 | | 설정 바인딩 | `*Settings`(운영) 또는 `*Policy`(인가·도메인 정책), ArchUnit이 강제 | 그것을 바인딩하는 leaf | — | | Throwable | `*Exception`, ArchUnit이 강제 | — | — | | 포트의 계약 시험 키트 | 포트 소유자의 testFixtures에 둔 추상 `Contract`. fake의 시험과 각 어댑터의 시험이 상속한다 | testFixtures | 값 타입 시험의 이름이 한때 `*ContractTest`였다. 지금은 `*ValueTypesTest`다 | diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowQuery.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowQuery.java new file mode 100644 index 00000000..14879d5a --- /dev/null +++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowQuery.java @@ -0,0 +1,16 @@ +package dev.caskeleton.application.realtime; + +import dev.caskeleton.application.query.Query; +import java.util.Objects; + +/** + * A reconnecting client's stream, asked what it still holds. + * + * @param streamId which stream + */ +public record GetReplayWindowQuery(String streamId) implements Query { + + public GetReplayWindowQuery { + Objects.requireNonNull(streamId, "streamId"); + } +} diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowUseCase.java new file mode 100644 index 00000000..6d586a46 --- /dev/null +++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowUseCase.java @@ -0,0 +1,41 @@ +package dev.caskeleton.application.realtime; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.QueryUseCase; +import java.util.Objects; + +/** + * What a stream still holds, so a transport can tell "your cursor is too old" from "nothing new". + * + *

Every read goes through a query use case: a transport adapter that held {@link + * LiveEventReplayQueryPort} itself read the event log with no capability declared and no read + * boundary (round9 review 2026-11-12, RR9-05-10). + */ +@UseCaseCapability( + transactionMode = TransactionMode.READ_ONLY, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.READ_REPOSITORY) +public class GetReplayWindowUseCase implements QueryUseCase { + + private final LiveEventReplayQueryPort replay; + private final TransactionPort transactions; + + /** + * @param replay the retained event history + * @param transactions opens the read transaction + */ + public GetReplayWindowUseCase(LiveEventReplayQueryPort replay, TransactionPort transactions) { + this.replay = Objects.requireNonNull(replay, "replay"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + } + + @Override + public ReplayWindow handle(GetReplayWindowQuery query) { + Objects.requireNonNull(query, "query"); + return transactions.inRead(() -> replay.window(query.streamId())); + } +} diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayPort.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayQueryPort.java similarity index 59% rename from src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayPort.java rename to src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayQueryPort.java index 741b0fa6..8d4cdf87 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayPort.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayQueryPort.java @@ -5,15 +5,18 @@ import java.util.List; /** * Retained event history, for a client that reconnects and wants to continue. * - *

The history lives wherever the messaging platform keeps it, and this port is the only way a - * transport adapter reaches it. That boundary is the point: a web or websocket module that kept its - * own copy would have its own retention, its own eviction and its own opinion about ordering, and - * the two would diverge silently because both would look plausible. + *

The history lives wherever the messaging platform keeps it, and this port is the only way to + * it — through {@code GetReplayWindowUseCase} and {@code ReplayLiveEventsUseCase}, never from a + * transport directly. A read model that returns projections, so a {@code *QueryPort}: named {@code + * LiveEventReplayPort} until 2026-11-13, when a web bridge that held it read the log with no use + * case in between (round9 review 2026-11-12, RR9-05-10). That boundary is the point: a web or + * websocket module that kept its own copy would have its own retention, its own eviction and its + * own opinion about ordering, and the two would diverge silently because both would look plausible. * *

{@link #window} exists so a caller can distinguish "your cursor is too old" from "there is * nothing new", which {@link #replayAfter} returning an empty list cannot. */ -public interface LiveEventReplayPort { +public interface LiveEventReplayQueryPort { /** What the stream still holds. */ ReplayWindow window(String streamId); diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsQuery.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsQuery.java new file mode 100644 index 00000000..23b091b4 --- /dev/null +++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsQuery.java @@ -0,0 +1,22 @@ +package dev.caskeleton.application.realtime; + +import dev.caskeleton.application.query.Query; +import java.util.Objects; + +/** + * A reconnecting client's request for what it missed. + * + * @param streamId which stream + * @param afterPosition the last position the client already has; 0 for the beginning + * @param limit the most events to return + */ +public record ReplayLiveEventsQuery(String streamId, long afterPosition, int limit) + implements Query { + + public ReplayLiveEventsQuery { + Objects.requireNonNull(streamId, "streamId"); + if (afterPosition < 0 || limit < 1) { + throw new IllegalArgumentException("afterPosition must be >= 0 and limit >= 1"); + } + } +} diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsUseCase.java new file mode 100644 index 00000000..2d72a1f0 --- /dev/null +++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsUseCase.java @@ -0,0 +1,45 @@ +package dev.caskeleton.application.realtime; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.QueryUseCase; +import java.util.List; +import java.util.Objects; + +/** + * The events a reconnecting client missed, a bounded page at a time. + * + * @see GetReplayWindowUseCase + */ +@UseCaseCapability( + transactionMode = TransactionMode.READ_ONLY, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.READ_REPOSITORY) +public class ReplayLiveEventsUseCase + implements QueryUseCase> { + + private final LiveEventReplayQueryPort replay; + private final TransactionPort transactions; + + /** + * @param replay the retained event history + * @param transactions opens the read transaction + */ + public ReplayLiveEventsUseCase(LiveEventReplayQueryPort replay, TransactionPort transactions) { + this.replay = Objects.requireNonNull(replay, "replay"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + } + + /** + * @throws ReplayCursorUnavailableException when the position is older than what is retained + */ + @Override + public List handle(ReplayLiveEventsQuery query) { + Objects.requireNonNull(query, "query"); + return transactions.inRead( + () -> replay.replayAfter(query.streamId(), query.afterPosition(), query.limit())); + } +} diff --git a/src/application/core/src/test/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplayContractTest.java b/src/application/core/src/test/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplayContractTest.java index 75963702..b7f98131 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplayContractTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplayContractTest.java @@ -1,17 +1,17 @@ package dev.caskeleton.application.realtime.testkit; -import dev.caskeleton.application.realtime.LiveEventReplayPort; +import dev.caskeleton.application.realtime.LiveEventReplayQueryPort; import java.time.Duration; /** The in-memory history keeps the replay contract the transport tests run over. */ -class InMemoryLiveEventReplayContractTest extends LiveEventReplayContract { +class InMemoryLiveEventReplayContractTest extends LiveEventReplayQueryContract { private static final Duration RETENTION = Duration.ofHours(1); private final InMemoryLiveEventReplay replay = new InMemoryLiveEventReplay(clock(), RETENTION); @Override - protected LiveEventReplayPort replay() { + protected LiveEventReplayQueryPort replay() { return replay; } diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplay.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplay.java index f6dc06d2..462220e4 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplay.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplay.java @@ -1,6 +1,6 @@ package dev.caskeleton.application.realtime.testkit; -import dev.caskeleton.application.realtime.LiveEventReplayPort; +import dev.caskeleton.application.realtime.LiveEventReplayQueryPort; import dev.caskeleton.application.realtime.ReplayCursorUnavailableException; import dev.caskeleton.application.realtime.ReplayWindow; import dev.caskeleton.application.realtime.ReplayedEvent; @@ -15,15 +15,15 @@ import java.util.Objects; import java.util.Optional; /** - * In-memory {@link LiveEventReplayPort} with the rules of the JPA history: positions per stream - * from 1 and never reused, an event replayable until {@code appendedAt + retention} on the given - * clock, a cursor older than what is retained refused with {@link - * ReplayCursorUnavailableException}. {@link LiveEventReplayContract} holds it and the JPA store to - * the same cases. + * In-memory {@link LiveEventReplayQueryPort} with the rules of the JPA history: positions per + * stream from 1 and never reused, an event replayable until {@code appendedAt + retention} on the + * given clock, a cursor older than what is retained refused with {@link + * ReplayCursorUnavailableException}. {@link LiveEventReplayQueryContract} holds it and the JPA + * store to the same cases. * *

Single-threaded: one instance per test. */ -public final class InMemoryLiveEventReplay implements LiveEventReplayPort { +public final class InMemoryLiveEventReplay implements LiveEventReplayQueryPort { private record Stored(ReplayedEvent event, Instant retainedUntil) {} diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayContract.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayQueryContract.java similarity index 94% rename from src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayContract.java rename to src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayQueryContract.java index 4c7e5b1f..81a3c335 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayContract.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayQueryContract.java @@ -3,7 +3,7 @@ package dev.caskeleton.application.realtime.testkit; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; -import dev.caskeleton.application.realtime.LiveEventReplayPort; +import dev.caskeleton.application.realtime.LiveEventReplayQueryPort; import dev.caskeleton.application.realtime.ReplayCursorUnavailableException; import dev.caskeleton.application.realtime.ReplayWindow; import dev.caskeleton.application.realtime.ReplayedEvent; @@ -18,14 +18,14 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; /** - * What every {@link LiveEventReplayPort} promises a reconnecting client, run against the in-memory - * fake and the JPA history on real PostgreSQL. + * What every {@link LiveEventReplayQueryPort} promises a reconnecting client, run against the + * in-memory fake and the JPA history on real PostgreSQL. * *

Writing history is not on the port (a transport reads it and never writes it), so each * implementation supplies its own {@link #append}. Retention is measured on {@link #clock()}, which * the implementation is built with and the contract moves. */ -public abstract class LiveEventReplayContract { +public abstract class LiveEventReplayQueryContract { private final ContractClock clock = new ContractClock(Instant.parse("2026-11-03T00:00:00Z")); @@ -43,7 +43,7 @@ public abstract class LiveEventReplayContract { * * @return the port */ - protected abstract LiveEventReplayPort replay(); + protected abstract LiveEventReplayQueryPort replay(); /** * Appends an event the way the store's writer does. diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupCommand.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupCommand.java new file mode 100644 index 00000000..4b4f8612 --- /dev/null +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupCommand.java @@ -0,0 +1,18 @@ +package dev.caskeleton.application.fileserver.cleanup; + +import dev.caskeleton.application.command.Command; + +/** + * The bounds of one unattended cleanup batch. + * + * @param maxItems most queued items the batch claims + * @param maxBytes most bytes the batch reclaims before it stops starting items + */ +public record RunScheduledCleanupCommand(int maxItems, long maxBytes) implements Command { + + public RunScheduledCleanupCommand { + if (maxItems < 1 || maxBytes < 1) { + throw new IllegalStateException("cleanup batch bounds must be positive"); + } + } +} diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCase.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCase.java new file mode 100644 index 00000000..9eed3d9e --- /dev/null +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCase.java @@ -0,0 +1,48 @@ +package dev.caskeleton.application.fileserver.cleanup; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.util.Objects; + +/** + * One unattended cleanup batch: reclaim what the queue holds, within the batch bounds. + * + *

The same work an operator starts with {@code CleanupUploadsUseCase}, which declares {@code + * file:admin}. A write both a user and the system start is two use cases: this one is the + * scheduler's, on a thread with no {@code Authentication}, so it carries no permission and no user + * transport may reach it. + * + *

{@code RepositoryAccess.NONE}: the batch is {@link CleanupService}'s, which opens a boundary + * per claimed item. One around the whole batch would hold a connection across every deletion. + * + *

Not a bean. The Fileserver composition's cleanup worker constructs it and keeps it. + */ +@SystemInvoked( + reason = + "the Fileserver cleanup worker runs it on the scheduler thread, which has no" + + " Authentication; an operator's run is CleanupUploadsUseCase with file:admin") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.NONE, + // The cleanup service deletes staged and orphaned objects in the object store. + externalOutboundAllowed = true) +public class RunScheduledCleanupUseCase + implements CommandUseCase { + + private final CleanupService cleanup; + + public RunScheduledCleanupUseCase(CleanupService cleanup) { + this.cleanup = Objects.requireNonNull(cleanup, "cleanup"); + } + + @Override + public CleanupBatchResult handle(RunScheduledCleanupCommand command) { + Objects.requireNonNull(command, "command"); + return cleanup.runBatch(command.maxItems(), command.maxBytes()); + } +} diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/CleanupUploadsCommand.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/CleanupUploadsCommand.java index a1f15e75..5e2f0fb3 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/CleanupUploadsCommand.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/CleanupUploadsCommand.java @@ -3,6 +3,7 @@ package dev.caskeleton.application.fileserver.port.in; import dev.caskeleton.application.command.Command; import dev.caskeleton.application.fileserver.api.security.RequestContext; import java.util.Objects; +import java.util.Optional; /** * Run one cleanup batch. @@ -14,7 +15,28 @@ import java.util.Objects; public record CleanupUploadsCommand(int maxItems, long maxBytes, RequestContext context) implements Command { + /** How many queued items a request that names no bound processes. */ + public static final int DEFAULT_MAX_ITEMS = 100; + + /** The byte budget of a request that names none: 1 GiB. */ + public static final long DEFAULT_MAX_BYTES = 1L << 30; + public CleanupUploadsCommand { Objects.requireNonNull(context, "context"); } + + /** + * The batch a request asks for, with absent bounds at the use case's defaults rather than at + * whatever a transport chose (round9 review 2026-11-12, RR9-05-19). + * + * @param maxItems absent means {@link #DEFAULT_MAX_ITEMS} + * @param maxBytes absent means {@link #DEFAULT_MAX_BYTES} + * @param context the operator + * @return the command + */ + public static CleanupUploadsCommand requested( + Optional maxItems, Optional maxBytes, RequestContext context) { + return new CleanupUploadsCommand( + maxItems.orElse(DEFAULT_MAX_ITEMS), maxBytes.orElse(DEFAULT_MAX_BYTES), context); + } } diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListIncompleteUploadsQuery.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListIncompleteUploadsQuery.java index 3a8960e4..f3176cc0 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListIncompleteUploadsQuery.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListIncompleteUploadsQuery.java @@ -3,6 +3,7 @@ package dev.caskeleton.application.fileserver.port.in; import dev.caskeleton.application.fileserver.api.security.RequestContext; import dev.caskeleton.application.query.Query; import java.util.Objects; +import java.util.Optional; /** * List incomplete uploads. @@ -12,7 +13,16 @@ import java.util.Objects; */ public record ListIncompleteUploadsQuery(int limit, RequestContext context) implements Query { + /** How many a request that names no limit lists. */ + public static final int DEFAULT_LIMIT = 100; + public ListIncompleteUploadsQuery { Objects.requireNonNull(context, "context"); } + + /** The query a request asks for; an absent limit is {@link #DEFAULT_LIMIT}. */ + public static ListIncompleteUploadsQuery requested( + Optional limit, RequestContext context) { + return new ListIncompleteUploadsQuery(limit.orElse(DEFAULT_LIMIT), context); + } } diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListOrphansQuery.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListOrphansQuery.java index 6456d0a9..a6d71fae 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListOrphansQuery.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListOrphansQuery.java @@ -3,6 +3,7 @@ package dev.caskeleton.application.fileserver.port.in; import dev.caskeleton.application.fileserver.api.security.RequestContext; import dev.caskeleton.application.query.Query; import java.util.Objects; +import java.util.Optional; /** * List orphaned objects. @@ -12,7 +13,15 @@ import java.util.Objects; */ public record ListOrphansQuery(int limit, RequestContext context) implements Query { + /** How many a request that names no limit lists. */ + public static final int DEFAULT_LIMIT = 100; + public ListOrphansQuery { Objects.requireNonNull(context, "context"); } + + /** The query a request asks for; an absent limit is {@link #DEFAULT_LIMIT}. */ + public static ListOrphansQuery requested(Optional limit, RequestContext context) { + return new ListOrphansQuery(limit.orElse(DEFAULT_LIMIT), context); + } } diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/OrphanReconcileCommand.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/OrphanReconcileCommand.java index 52d826fb..9982be14 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/OrphanReconcileCommand.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/OrphanReconcileCommand.java @@ -4,6 +4,7 @@ import dev.caskeleton.application.command.Command; import dev.caskeleton.application.fileserver.api.security.RequestContext; import java.util.List; import java.util.Objects; +import java.util.Optional; /** * Request to reconcile orphaned physical objects. @@ -45,9 +46,55 @@ public record OrphanReconcileCommand( expectedFingerprints = List.copyOf(expectedFingerprints); } + /** How many candidates a request that names no limit considers. */ + public static final int DEFAULT_LIMIT = 100; + + /** The byte budget of an apply that names none: 1 GiB. */ + public static final long DEFAULT_APPLY_MAX_BYTES = 1L << 30; + + /** The audit reason of an apply that names none. */ + public static final String DEFAULT_APPLY_REASON = "ORPHAN_APPLY"; + /** Bounded dry run, the default and the only shape a caller can reach without opting in. */ public static OrphanReconcileCommand dryRun(int limit, RequestContext context) { return new OrphanReconcileCommand( true, limit, Long.MAX_VALUE, List.of(), "ORPHAN_SCAN", context); } + + /** + * The command a request asks for, with every absent field at its safe value: a dry run, the + * default page, no fingerprints. + * + *

The defaults are the use case's, not the transport's, so every transport that offers the + * operation reaches the same ones; they used to be written into the HTTP controller (round9 + * review 2026-11-12, RR9-05-19). Nothing here can be omitted into a destructive default: an apply + * still has to name its fingerprints. + * + * @param dryRun list only; absent means true + * @param limit the most candidates; absent means {@link #DEFAULT_LIMIT} + * @param maxBytes an apply's byte budget; absent means {@link #DEFAULT_APPLY_MAX_BYTES} + * @param expectedFingerprints the objects an apply may remove; absent means none + * @param reasonCode an apply's audit reason; absent means {@link #DEFAULT_APPLY_REASON} + * @param context the operator + * @return the command + */ + public static OrphanReconcileCommand requested( + Optional dryRun, + Optional limit, + Optional maxBytes, + Optional> expectedFingerprints, + Optional reasonCode, + RequestContext context) { + int page = limit.orElse(DEFAULT_LIMIT); + if (dryRun.orElse(true)) { + return dryRun(page, context); + } + return new OrphanReconcileCommand( + false, + page, + maxBytes.orElse(DEFAULT_APPLY_MAX_BYTES), + expectedFingerprints.orElse(List.of()), + reasonCode.orElse(DEFAULT_APPLY_REASON), + context); + } } diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesCommand.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesCommand.java new file mode 100644 index 00000000..8d611047 --- /dev/null +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesCommand.java @@ -0,0 +1,17 @@ +package dev.caskeleton.application.fileserver.recovery; + +import dev.caskeleton.application.command.Command; + +/** + * The bound of one unattended reconciliation pass. + * + * @param limit most open recovery items the pass settles + */ +public record ReconcileRecoverableFilesCommand(int limit) implements Command { + + public ReconcileRecoverableFilesCommand { + if (limit < 1) { + throw new IllegalStateException("reconciliation limit must be positive"); + } + } +} diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCase.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCase.java new file mode 100644 index 00000000..a0347a2e --- /dev/null +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCase.java @@ -0,0 +1,48 @@ +package dev.caskeleton.application.fileserver.recovery; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.util.List; +import java.util.Objects; + +/** + * One unattended reconciliation pass: settle the files an ambiguous publish or an unconfirmed + * commit left open, a bounded number at a time. + * + *

The scheduler's, on a thread with no {@code Authentication}, so it carries no permission and + * no user transport may reach it. + * + *

{@code RepositoryAccess.NONE}: each file is {@link FileReconciliationService}'s to settle, in + * a boundary of its own around each write. + * + *

Not a bean. The Fileserver composition's cleanup worker constructs it and keeps it. + */ +@SystemInvoked( + reason = + "the Fileserver cleanup worker runs it on the scheduler thread, which has no" + + " Authentication") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.NONE, + // The reconciliation service reads the object store to decide each file. + externalOutboundAllowed = true) +public class ReconcileRecoverableFilesUseCase + implements CommandUseCase> { + + private final FileReconciliationService reconciliation; + + public ReconcileRecoverableFilesUseCase(FileReconciliationService reconciliation) { + this.reconciliation = Objects.requireNonNull(reconciliation, "reconciliation"); + } + + @Override + public List handle(ReconcileRecoverableFilesCommand command) { + Objects.requireNonNull(command, "command"); + return reconciliation.reconcileRecoverable(command.limit()); + } +} diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCaseTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCaseTest.java new file mode 100644 index 00000000..ee95821c --- /dev/null +++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCaseTest.java @@ -0,0 +1,60 @@ +package dev.caskeleton.application.fileserver.cleanup; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.security.SystemInvoked; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The scheduler's cleanup run is a use case of its own, apart from the operator's (round9 review + * 2026-11-12, RR9-05-06). + */ +class RunScheduledCleanupUseCaseTest { + + @Test + @DisplayName("예약 cleanup은 command의 항목·byte 상한으로 batch 하나를 돌린다") + void aScheduledRunReclaimsOneBatchWithinTheCommandsBounds() { + // given + List> batches = new ArrayList<>(); + RunScheduledCleanupUseCase useCase = + new RunScheduledCleanupUseCase( + (maxItems, maxBytes) -> { + batches.add(List.of((long) maxItems, maxBytes)); + return CleanupBatchResult.empty(); + }); + + // when + useCase.handle(new RunScheduledCleanupCommand(25, 4_096)); + + // then + assertThat(batches).containsExactly(List.of(25L, 4_096L)); + } + + @Test + @DisplayName("시스템만 부르는 쓰기라 권한을 선언하지 않고 @SystemInvoked로 그 이유를 적는다") + void itIsMarkedAsTheSystemsAndDeclaresNoPermission() { + assertThat( + List.of( + RunScheduledCleanupUseCase.class.isAnnotationPresent(SystemInvoked.class), + RunScheduledCleanupUseCase.class.isAnnotationPresent(RequiresPermission.class))) + .containsExactly(true, false); + } + + @Test + @DisplayName("상한이 0인 batch는 만들 수 없다") + void aBatchWithNoRoomIsRefused() { + // when + Throwable refused = catchThrowable(() -> new RunScheduledCleanupCommand(0, 4_096)); + + // then + assertThat(refused) + .as("the worker's settings build the batch, so a bad bound is the assembly's fault") + .isInstanceOf(IllegalStateException.class) + .hasMessage("cleanup batch bounds must be positive"); + } +} diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/port/in/AdminRequestDefaultsTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/port/in/AdminRequestDefaultsTest.java new file mode 100644 index 00000000..f90e8e26 --- /dev/null +++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/port/in/AdminRequestDefaultsTest.java @@ -0,0 +1,116 @@ +package dev.caskeleton.application.fileserver.port.in; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.fileserver.api.security.FileAccessSubject; +import dev.caskeleton.application.fileserver.api.security.RequestContext; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * What an operator's admin request means when it leaves a field out. + * + *

The defaults are the use case's, not the transport's: the controller used to fill them in + * itself, so a second transport would have had to copy them (round9 review 2026-11-12, RR9-05-19). + */ +class AdminRequestDefaultsTest { + + private static final RequestContext OPERATOR = + new RequestContext(FileAccessSubject.of("operator-1", Set.of("admin")), "trace-1", "node-a"); + + @Test + @DisplayName("값을 하나도 주지 않은 orphan 정리 요청은 dry run이고 아무것도 지우지 않는다") + void anOrphanRequestWithNothingSetIsADryRun() { + // when + OrphanReconcileCommand command = + OrphanReconcileCommand.requested( + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + OPERATOR); + + // then + assertThat(command) + .extracting( + OrphanReconcileCommand::dryRun, + OrphanReconcileCommand::limit, + OrphanReconcileCommand::expectedFingerprints) + .containsExactly(true, OrphanReconcileCommand.DEFAULT_LIMIT, List.of()); + } + + @Test + @DisplayName("apply 요청은 byte 상한 1GiB와 기본 이유 코드를 받는다") + void anApplyRequestGetsTheDefaultByteCeilingAndReason() { + // when + OrphanReconcileCommand command = + OrphanReconcileCommand.requested( + Optional.of(false), + Optional.empty(), + Optional.empty(), + Optional.of(List.of("f".repeat(64))), + Optional.empty(), + OPERATOR); + + // then + assertThat(command) + .extracting( + OrphanReconcileCommand::dryRun, + OrphanReconcileCommand::maxBytes, + OrphanReconcileCommand::reasonCode) + .containsExactly(false, 1L << 30, "ORPHAN_APPLY"); + } + + @Test + @DisplayName("지울 fingerprint를 대지 않은 apply 요청은 거절한다") + void anApplyRequestThatNamesNoFingerprintIsRefused() { + // when + Throwable refused = + catchThrowable( + () -> + OrphanReconcileCommand.requested( + Optional.of(false), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + OPERATOR)); + + // then + assertThat(refused) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("an apply must name the fingerprints it intends to remove"); + } + + @Test + @DisplayName("cleanup 요청은 항목 100개·1GiB 상한을 기본으로 받는다") + void aCleanupRequestGetsTheDefaultBatchBounds() { + // when + CleanupUploadsCommand command = + CleanupUploadsCommand.requested(Optional.empty(), Optional.empty(), OPERATOR); + + // then + assertThat(command) + .extracting(CleanupUploadsCommand::maxItems, CleanupUploadsCommand::maxBytes) + .containsExactly(100, 1L << 30); + } + + @Test + @DisplayName("목록 조회는 쪽 크기 100을 기본으로 받고, 준 값은 그대로 쓴다") + void listingsDefaultToAPageOfOneHundredAndKeepAGivenLimit() { + // when + List limits = + List.of( + ListOrphansQuery.requested(Optional.empty(), OPERATOR).limit(), + ListIncompleteUploadsQuery.requested(Optional.empty(), OPERATOR).limit(), + ListOrphansQuery.requested(Optional.of(7), OPERATOR).limit()); + + // then + assertThat(limits).containsExactly(100, 100, 7); + } +} diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCaseTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCaseTest.java new file mode 100644 index 00000000..a2e1e2c4 --- /dev/null +++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCaseTest.java @@ -0,0 +1,68 @@ +package dev.caskeleton.application.fileserver.recovery; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.security.SystemInvoked; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The scheduler's reconciliation pass is a {@code @SystemInvoked} use case (round9 review + * 2026-11-12, RR9-05-06). + */ +class ReconcileRecoverableFilesUseCaseTest { + + @Test + @DisplayName("예약 reconcile은 command의 상한만큼 열린 복구 항목을 정리한다") + void aScheduledPassSettlesAsManyOpenItemsAsTheCommandAllows() { + // given + List limits = new ArrayList<>(); + ReconcileRecoverableFilesUseCase useCase = + new ReconcileRecoverableFilesUseCase( + new FileReconciliationService() { + @Override + public ReconciliationResult reconcile(FileId fileId) { + throw new UnsupportedOperationException("the pass reconciles in batches only"); + } + + @Override + public List reconcileRecoverable(int limit) { + limits.add(limit); + return List.of(); + } + }); + + // when + useCase.handle(new ReconcileRecoverableFilesCommand(40)); + + // then + assertThat(limits).containsExactly(40); + } + + @Test + void itIsMarkedAsTheSystemsAndDeclaresNoPermission() { + assertThat( + List.of( + ReconcileRecoverableFilesUseCase.class.isAnnotationPresent(SystemInvoked.class), + ReconcileRecoverableFilesUseCase.class.isAnnotationPresent( + RequiresPermission.class))) + .containsExactly(true, false); + } + + @Test + @DisplayName("상한이 0인 pass는 만들 수 없고, 조립의 잘못으로 실패한다") + void aPassWithNoRoomIsRefused() { + // when + Throwable refused = catchThrowable(() -> new ReconcileRecoverableFilesCommand(0)); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessage("reconciliation limit must be positive"); + } +} diff --git a/src/application/notification/AGENTS.md b/src/application/notification/AGENTS.md index 0dce0cb1..cb807a1a 100644 --- a/src/application/notification/AGENTS.md +++ b/src/application/notification/AGENTS.md @@ -20,6 +20,8 @@ - framework 의존이 없다(`ca.java-library`). Spring·JPA·HTTP 타입을 쓰지 않는다. 허용 의존은 registry의 `application-core`·`shared-contract`뿐이다. - 전송 DTO(HTTP 요청·응답 record)를 여기에 두지 않는다. 전송 모양은 `adapter:inbound:web-notification`의 것이다. - 발송 판정(마케팅 억제, category 필수)은 여기서 한다. 전송이나 설정에서 기본값을 골라 넣지 않는다. +- 시스템이 부르는 쓰기는 `@SystemInvoked` 유스케이스다: 발송 claim·발송(`ClaimDueDeliveriesUseCase`·`DispatchClaimedDeliveryUseCase`), lease 회복, provider 결과 맞추기, provider event 다시 투영. `composition:notification-platform`의 worker가 `new`로 만들어 쥐고, bean으로 두지 않는다. 그 유스케이스가 감싸는 서비스(`LeaseRecoveryService`·`ReconciliationService` 등)는 bean이지만 inbound가 쥐면 빌드가 실패한다(`INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES`). 언제 다시 묻고 언제 포기하는지는 이 유스케이스가 정한다. worker는 스레드와 일정만 가진다. +- 운영자 연산(redrive·reconcile·suppress·provider 상태)은 연산마다 `platform.admin`의 유스케이스 클래스이고(`RedriveDeliveryUseCase` 등, command는 그 옆), `notification-admin:*` permission을 선언한다. `platform.port.in`에 두지 않는다: command가 `AdminActor` 같은 운영자 어휘를 실어 `port.in`과 `admin`이 서로를 보게 된다. 운영자 authority와 tenant 범위는 command의 `AdminActor`로 한 번 더 본다. - 공개 타입이 늘면 `docs/notification/api-surface-snapshot.txt`가 바뀐다(`adapter:outbound:notification`의 `verifyNotificationApiSurface`가 이 leaf까지 본다). ## 테스트 diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutor.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutor.java index cf1349a2..76a77fe6 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutor.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutor.java @@ -18,31 +18,30 @@ final class AdminOperationExecutor { } /** - * Claims an idempotent operator action before any side effect. + * Claims an idempotent operator action before any side effect, inside the caller's write + * transaction. * - *

The claim and the operation run inside one write transaction. PostgreSQL's unique claim row - * serializes a concurrent caller on the operation id; the loser observes the committed result - * instead of reaching the side effect. + *

The use case opens that transaction and calls this in it, so the claim and the operation + * commit or roll back together: the boundary is the use case's to declare, and the architecture + * rule that matches a capability to its {@code TransactionPort} call reads it there. PostgreSQL's + * unique claim row serializes a concurrent caller on the operation id; the loser observes the + * committed result instead of reaching the side effect. */ - AdminOperationResult execute( + AdminOperationResult runClaimed( String operationId, - String commandFingerprint, + String requestFingerprint, AdminActor actor, String action, Supplier operation) { Objects.requireNonNull(operation, "operation"); - return transactions.inWrite( - () -> { - AdminOperationClaim claim = - operations.claim(operationId, commandFingerprint, actor, action); - return claim.mayProceed() ? operation.get() : withoutRunning(claim, operationId); - }); + AdminOperationClaim claim = operations.claim(operationId, requestFingerprint, actor, action); + return claim.mayProceed() ? operation.get() : withoutRunning(claim, operationId); } /** * Claims the operation id in its own transaction and runs the operation with none open. * - *

For an operation that calls a provider over the network. Inside {@link #execute} the call + *

For an operation that calls a provider over the network. Inside {@link #runClaimed} the call * held a write transaction — and a pooled connection — for as long as the provider took, once per * attempt in the batch. The operation opens its own boundaries for what it writes, including * saving its result, which completes the claim. @@ -53,14 +52,14 @@ final class AdminOperationExecutor { */ AdminOperationResult executeOutsideTransaction( String operationId, - String commandFingerprint, + String requestFingerprint, AdminActor actor, String action, Supplier operation) { Objects.requireNonNull(operation, "operation"); AdminOperationClaim claim = transactions.inWrite( - () -> operations.claim(operationId, commandFingerprint, actor, action)); + () -> operations.claim(operationId, requestFingerprint, actor, action)); if (!claim.mayProceed()) { return withoutRunning(claim, operationId); } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprint.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprint.java similarity index 50% rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprint.java rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprint.java index a2f1510d..9239c52f 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprint.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprint.java @@ -4,55 +4,55 @@ import dev.caskeleton.shared.digest.Sha256; import java.util.Objects; /** - * Stable SHA-256 identity of an operator command, excluding the operation id itself. + * Stable SHA-256 identity of an operator request, excluding the operation id itself. * *

The operation id is the idempotency key. The fingerprint answers the separate question: - * whether a repeated key names the same command. Every field is length-framed so user supplied + * whether a repeated key names the same request. Every field is length-framed so user supplied * reason text cannot imitate a field boundary. */ -final class AdminOperationCommandFingerprint { +final class AdminOperationRequestFingerprint { - String of(RedriveCommand command) { - Objects.requireNonNull(command, "command"); + String of(RedriveRequest request) { + Objects.requireNonNull(request, "request"); StringBuilder canonical = action("REDRIVE"); - field(canonical, "attemptId", command.attemptId().value().toString()); - field(canonical, "reason", command.reason()); - field(canonical, "approveDuplicateRisk", Boolean.toString(command.approveDuplicateRisk())); + field(canonical, "attemptId", request.attemptId().value().toString()); + field(canonical, "reason", request.reason()); + field(canonical, "approveDuplicateRisk", Boolean.toString(request.approveDuplicateRisk())); return digest(canonical); } - String of(ReconcileCommand command) { - Objects.requireNonNull(command, "command"); + String of(ReconcileRequest request) { + Objects.requireNonNull(request, "request"); StringBuilder canonical = action("RECONCILE"); - command.attemptIds().stream() + request.attemptIds().stream() .map(id -> id.value().toString()) .sorted() .forEach(value -> field(canonical, "attemptId", value)); - field(canonical, "reason", command.reason()); + field(canonical, "reason", request.reason()); return digest(canonical); } - String of(SuppressCommand command) { - Objects.requireNonNull(command, "command"); - StringBuilder canonical = action(command.remove() ? "SUPPRESS_REMOVE" : "SUPPRESS_ADD"); - field(canonical, "tenantId", command.tenantId().value()); - field(canonical, "scope", command.scope().name()); - field(canonical, "reason", command.reason().name()); - field(canonical, "targetFingerprint", command.targetFingerprint()); + String of(SuppressionRequest request) { + Objects.requireNonNull(request, "request"); + StringBuilder canonical = action(request.remove() ? "SUPPRESS_REMOVE" : "SUPPRESS_ADD"); + field(canonical, "tenantId", request.tenantId().value()); + field(canonical, "scope", request.scope().name()); + field(canonical, "reason", request.reason().name()); + field(canonical, "targetFingerprint", request.targetFingerprint()); field( canonical, "expiresAt", - command.expiresAt().map(value -> Long.toString(value.toEpochMilli())).orElse("-")); - field(canonical, "reasonText", command.reasonText()); + request.expiresAt().map(value -> Long.toString(value.toEpochMilli())).orElse("-")); + field(canonical, "reasonText", request.reasonText()); return digest(canonical); } - String of(SetProviderStateCommand command) { - Objects.requireNonNull(command, "command"); + String of(ProviderStateRequest request) { + Objects.requireNonNull(request, "request"); StringBuilder canonical = action("PROVIDER_STATE"); - field(canonical, "profileId", command.profileId().value()); - field(canonical, "desiredState", command.desiredState().name()); - field(canonical, "reason", command.reason()); + field(canonical, "profileId", request.profileId().value()); + field(canonical, "desiredState", request.desiredState().name()); + field(canonical, "reason", request.reason()); return digest(canonical); } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationStorePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationStorePort.java index 88f9b373..0c526edc 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationStorePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationStorePort.java @@ -19,15 +19,17 @@ public interface AdminOperationStorePort { * was checked but never held, so it prevented a repeat and not a race. A destructive operation * run twice concurrently is the exact failure the id exists to stop. * - *

The fingerprint is part of the claim: the same id with a different command is not a replay, - * it is a conflict, and answering it with the earlier result would silently execute neither. + *

The fingerprint is part of the claim: the same id with a different request is not a replay, + * it is a conflict, and answering it with the earlier result would silently execute neither. It + * digests the operator's request and leaves the actor out; the store keeps it in the column V8 + * named {@code command_fingerprint}. * * @param operationId the caller's idempotency key - * @param commandFingerprint a digest of the command this id was presented with + * @param requestFingerprint a digest of the request this id was presented with * @param actor who is asking * @param action what they are asking for * @return whether this caller may proceed, and what the earlier one did if not */ AdminOperationClaim claim( - String operationId, String commandFingerprint, AdminActor actor, String action); + String operationId, String requestFingerprint, AdminActor actor, String action); } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateCommand.java new file mode 100644 index 00000000..d6626f7f --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateCommand.java @@ -0,0 +1,22 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.command.Command; +import java.util.Objects; + +/** + * An operator's change to one provider profile's runtime state. + * + *

The request and the operator are apart because the operation id's fingerprint covers the + * request only: the same operator retrying the same request replays it. + * + * @param request what to do + * @param actor who asks, with the operator authorities and tenant scope the transport resolved + */ +public record ChangeProviderStateCommand(ProviderStateRequest request, AdminActor actor) + implements Command { + + public ChangeProviderStateCommand { + Objects.requireNonNull(request, "request"); + Objects.requireNonNull(actor, "actor"); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateUseCase.java new file mode 100644 index 00000000..4dc51de9 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateUseCase.java @@ -0,0 +1,130 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.notification.platform.api.error.FailureCategory; +import dev.caskeleton.application.notification.platform.api.error.NotificationFailureCode; +import dev.caskeleton.application.notification.platform.api.error.NotificationFailureDescriptor; +import dev.caskeleton.application.notification.platform.api.error.ProviderConfigurationException; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; +import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.time.Clock; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Enables, disables or drains a provider profile as an operator. + * + *

A provider profile is shared by every tenant, so only an operator with no tenant scope may + * change it. The operation id is claimed before the runtime is asked, in the same transaction. + */ +@RequiresPermission(NotificationAdminPermissions.PROVIDER_CONTROL) +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, + // The provider runtime and the audit port are the notification adapter's. + externalOutboundAllowed = true, + crossTenantAdmin = true) +// Not final: @RequiresPermission is applied through a CGLIB proxy. +public class ChangeProviderStateUseCase + implements CommandUseCase { + + private static final String ACTION = "ADMIN_PROVIDER_STATE"; + + private final AdminAuthorizationGuard authorization; + private final ProviderRuntimeControlPort runtimes; + private final AdminOperationStorePort operations; + private final AdminOperationExecutor operationExecutor; + private final AdminOperationRequestFingerprint requestFingerprints = + new AdminOperationRequestFingerprint(); + private final NotificationAuditPort audit; + private final TransactionPort transactions; + private final Clock clock; + + public ChangeProviderStateUseCase( + AdminAuthorizationGuard authorization, + ProviderRuntimeControlPort runtimes, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + Clock clock) { + this.authorization = Objects.requireNonNull(authorization, "authorization"); + this.runtimes = Objects.requireNonNull(runtimes, "runtimes"); + this.operations = Objects.requireNonNull(operations, "operations"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions); + this.audit = Objects.requireNonNull(audit, "audit"); + this.clock = Objects.requireNonNull(clock, "clock"); + } + + @Override + public AdminOperationResult handle(ChangeProviderStateCommand change) { + Objects.requireNonNull(change, "change"); + ProviderStateRequest request = change.request(); + AdminActor actor = change.actor(); + authorization.require(actor, NotificationAdminAuthority.PROVIDER_CONTROL); + authorization.requireGlobal(actor, NotificationAdminAuthority.PROVIDER_CONTROL); + + if (request.dryRun()) { + return new AdminOperationResult( + request.operationId(), + true, + 1, + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of("DRY_RUN")); + } + + return transactions.inWrite( + () -> + operationExecutor.runClaimed( + request.operationId(), + requestFingerprints.of(request), + actor, + ACTION, + () -> change(request, actor))); + } + + private AdminOperationResult change(ProviderStateRequest request, AdminActor actor) { + boolean applied = + runtimes.setState(request.profileId(), request.desiredState(), request.reason()); + if (!applied) { + // The transition was refused — an authentication failure is not cleared by declaring the + // provider healthy, and a healthy runtime is not degraded on request. Recording success for a + // state change that did not happen tells an operator the provider is serving when it is not. + throw new ProviderConfigurationException( + NotificationFailureDescriptor.preDispatch( + NotificationFailureCode.PROVIDER_CONFIGURATION_INVALID, + FailureCategory.AUTHORIZATION)); + } + AdminOperationResult result = + new AdminOperationResult( + request.operationId(), + false, + 1, + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of(request.reason())); + audit.record( + new NotificationAuditEvent( + ACTION, + actor.actorRef(), + Optional.of(request.reason()), + Optional.of(request.operationId()), + clock.instant(), + Map.of( + "providerProfile", request.profileId().value(), + "status", request.desiredState().name()))); + return operations.save(result, actor, ACTION); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionCommand.java new file mode 100644 index 00000000..8a493bac --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionCommand.java @@ -0,0 +1,22 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.command.Command; +import java.util.Objects; + +/** + * An operator's change to one suppression entry. + * + *

The request and the operator are apart because the operation id's fingerprint covers the + * request only: the same operator retrying the same request replays it. + * + * @param request what to do + * @param actor who asks, with the operator authorities and tenant scope the transport resolved + */ +public record ChangeSuppressionCommand(SuppressionRequest request, AdminActor actor) + implements Command { + + public ChangeSuppressionCommand { + Objects.requireNonNull(request, "request"); + Objects.requireNonNull(actor, "actor"); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionUseCase.java new file mode 100644 index 00000000..4351472c --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionUseCase.java @@ -0,0 +1,149 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.notification.platform.dispatch.NotificationIdGeneratorPort; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; +import dev.caskeleton.application.notification.platform.policy.SuppressionEntry; +import dev.caskeleton.application.notification.platform.policy.SuppressionId; +import dev.caskeleton.application.notification.platform.policy.SuppressionSource; +import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort; +import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.time.Clock; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Adds or lifts a suppression entry as an operator. + * + *

Lifting is by fingerprint match rather than by id, because an operator lifting a suppression + * knows the target, not the row identifier the platform assigned. + */ +@RequiresPermission(NotificationAdminPermissions.SUPPRESS) +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, + // The audit port and the id generator are the notification adapter's. + externalOutboundAllowed = true, + crossTenantAdmin = true) +// Not final: @RequiresPermission is applied through a CGLIB proxy. +public class ChangeSuppressionUseCase + implements CommandUseCase { + + private final AdminAuthorizationGuard authorization; + private final SuppressionStorePort suppressions; + private final AdminOperationStorePort operations; + private final AdminOperationExecutor operationExecutor; + private final AdminOperationRequestFingerprint requestFingerprints = + new AdminOperationRequestFingerprint(); + private final NotificationAuditPort audit; + private final TransactionPort transactions; + private final NotificationIdGeneratorPort ids; + private final Clock clock; + + public ChangeSuppressionUseCase( + AdminAuthorizationGuard authorization, + SuppressionStorePort suppressions, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + NotificationIdGeneratorPort ids, + Clock clock) { + this.authorization = Objects.requireNonNull(authorization, "authorization"); + this.suppressions = Objects.requireNonNull(suppressions, "suppressions"); + this.operations = Objects.requireNonNull(operations, "operations"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions); + this.audit = Objects.requireNonNull(audit, "audit"); + this.ids = Objects.requireNonNull(ids, "ids"); + this.clock = Objects.requireNonNull(clock, "clock"); + } + + @Override + public AdminOperationResult handle(ChangeSuppressionCommand change) { + Objects.requireNonNull(change, "change"); + SuppressionRequest request = change.request(); + AdminActor actor = change.actor(); + authorization.require(actor, NotificationAdminAuthority.SUPPRESS); + authorization.requireTenant(actor, request.tenantId(), NotificationAdminAuthority.SUPPRESS); + + if (request.dryRun()) { + return new AdminOperationResult( + request.operationId(), + true, + 1, + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of("DRY_RUN")); + } + + String action = request.remove() ? "ADMIN_SUPPRESS_REMOVE" : "ADMIN_SUPPRESS_ADD"; + return transactions.inWrite( + () -> + operationExecutor.runClaimed( + request.operationId(), + requestFingerprints.of(request), + actor, + action, + () -> change(request, actor, action))); + } + + private AdminOperationResult change(SuppressionRequest request, AdminActor actor, String action) { + int affected = request.remove() ? lift(request) : add(request); + AdminOperationResult result = + new AdminOperationResult( + request.operationId(), + false, + affected, + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of(request.reasonText())); + audit.record( + new NotificationAuditEvent( + request.remove() ? "ADMIN_SUPPRESSION_REMOVED" : "ADMIN_SUPPRESSION_ADDED", + actor.actorRef(), + Optional.of(request.reason().name()), + Optional.of(request.operationId()), + clock.instant(), + Map.of())); + return operations.save(result, actor, action); + } + + private int lift(SuppressionRequest request) { + long lifted = + suppressions + .activeFor(request.tenantId(), request.targetFingerprint(), clock.instant()) + .stream() + .map(entry -> suppressions.remove(request.tenantId(), entry.id())) + .filter(Optional::isPresent) + .count(); + return lifted > 0 ? 1 : 0; + } + + private int add(SuppressionRequest request) { + suppressions.upsert( + new SuppressionEntry( + // ids.nextId(), not UUID.randomUUID(): a random v4 id is unordered, so the suppression + // table's index degrades the way the platform's UUIDv7 policy exists to prevent. + new SuppressionId(ids.nextId()), + request.tenantId(), + request.scope(), + request.reason(), + request.targetFingerprint(), + Optional.empty(), + clock.instant(), + request.expiresAt(), + SuppressionSource.ADMIN)); + return 1; + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationService.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationService.java deleted file mode 100644 index 684bad6e..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationService.java +++ /dev/null @@ -1,362 +0,0 @@ -package dev.caskeleton.application.notification.platform.admin; - -import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; -import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState; -import dev.caskeleton.application.notification.platform.api.error.FailureCategory; -import dev.caskeleton.application.notification.platform.api.error.NotificationFailureCode; -import dev.caskeleton.application.notification.platform.api.error.NotificationFailureDescriptor; -import dev.caskeleton.application.notification.platform.api.error.ProviderConfigurationException; -import dev.caskeleton.application.notification.platform.callback.DeliveryAttemptSnapshot; -import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort; -import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort; -import dev.caskeleton.application.notification.platform.dispatch.ReconciliationService; -import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent; -import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; -import dev.caskeleton.application.notification.platform.policy.SuppressionEntry; -import dev.caskeleton.application.notification.platform.policy.SuppressionId; -import dev.caskeleton.application.notification.platform.policy.SuppressionSource; -import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort; -import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState; -import dev.caskeleton.application.transaction.TransactionPort; -import java.time.Clock; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - -/** - * N4 operator plane. - * - *

It lived in the outbound notification adapter, where it performed operator authorization, - * tenant scoping, duplicate-risk approval, idempotency and transaction orchestration — every one of - * them a decision about what the product allows, in the layer whose job is speaking provider - * protocols. What remains out there is {@link ProviderRuntimeControlPort}: two operations that turn - * an application request into a runtime transition. - * - *

Four properties hold for every operation: a separate authority, an idempotent operation id, a - * recorded reason, and an audit row. The idempotency matters more than it looks — an operator - * retrying a redrive after a timeout must not send the message twice, which is exactly the failure - * the operation is trying to repair. - * - *

A dry run reads and reports but writes nothing, so an operator can see the blast radius of a - * bulk action before committing to it. - */ -public final class NotificationAdminApplicationService implements NotificationAdminService { - - private final AdminAuthorizationGuard authorization; - private final DuplicateRiskGuard duplicateRiskGuard; - private final DeliveryAttemptStorePort attempts; - private final RecipientDeliveryStorePort recipients; - private final ReconciliationService reconciliation; - private final SuppressionStorePort suppressions; - private final ProviderRuntimeControlPort runtimes; - private final AdminOperationStorePort operations; - private final AdminOperationExecutor operationExecutor; - private final AdminOperationCommandFingerprint commandFingerprints; - private final TransactionPort transactions; - private final NotificationAuditPort audit; - private final dev.caskeleton.application.notification.platform.dispatch - .NotificationIdGeneratorPort - ids; - private final Clock clock; - - public NotificationAdminApplicationService( - AdminAuthorizationGuard authorization, - DuplicateRiskGuard duplicateRiskGuard, - DeliveryAttemptStorePort attempts, - RecipientDeliveryStorePort recipients, - ReconciliationService reconciliation, - SuppressionStorePort suppressions, - ProviderRuntimeControlPort runtimes, - AdminOperationStorePort operations, - NotificationAuditPort audit, - TransactionPort transactions, - dev.caskeleton.application.notification.platform.dispatch.NotificationIdGeneratorPort ids, - Clock clock) { - this.authorization = Objects.requireNonNull(authorization, "authorization"); - this.duplicateRiskGuard = Objects.requireNonNull(duplicateRiskGuard, "duplicateRiskGuard"); - this.attempts = Objects.requireNonNull(attempts, "attempts"); - this.recipients = Objects.requireNonNull(recipients, "recipients"); - this.reconciliation = Objects.requireNonNull(reconciliation, "reconciliation"); - this.suppressions = Objects.requireNonNull(suppressions, "suppressions"); - this.runtimes = Objects.requireNonNull(runtimes, "runtimes"); - this.operations = Objects.requireNonNull(operations, "operations"); - this.transactions = Objects.requireNonNull(transactions, "transactions"); - this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions); - this.commandFingerprints = new AdminOperationCommandFingerprint(); - this.audit = Objects.requireNonNull(audit, "audit"); - this.ids = Objects.requireNonNull(ids, "ids"); - this.clock = Objects.requireNonNull(clock, "clock"); - } - - @Override - public AdminOperationResult redrive(RedriveCommand command, AdminActor actor) { - Objects.requireNonNull(command, "command"); - authorization.require(actor, NotificationAdminAuthority.REDRIVE); - - DeliveryAttemptSnapshot original = - attempts - .snapshot(command.attemptId()) - .orElseThrow(() -> new IllegalStateException("delivery attempt is not available")); - authorization.requireTenant(actor, original.tenantId(), NotificationAdminAuthority.REDRIVE); - duplicateRiskGuard.verify(original, command.approveDuplicateRisk()); - - if (command.dryRun()) { - return new AdminOperationResult( - command.operationId(), - true, - 1, - Optional.of(original.notificationId()), - Optional.of(original.recipientDeliveryId()), - Optional.empty(), - List.of("DRY_RUN")); - } - - return operationExecutor.execute( - command.operationId(), - commandFingerprints.of(command), - actor, - "ADMIN_REDRIVE", - () -> { - // The logical identities are preserved and only the attempt is new, so the history stays - // one story rather than becoming two unrelated notifications. - recipients.transition( - original.recipientDeliveryId(), - RecipientDeliveryState.READY_TO_DISPATCH, - Optional.of(clock.instant())); - - AdminOperationResult result = - new AdminOperationResult( - command.operationId(), - false, - 1, - Optional.of(original.notificationId()), - Optional.of(original.recipientDeliveryId()), - Optional.empty(), - List.of(command.reason())); - audit.record( - new NotificationAuditEvent( - "ADMIN_REDRIVE", - actor.actorRef(), - Optional.of(command.reason()), - Optional.of(command.operationId()), - clock.instant(), - Map.of( - "provider", original.providerId().value(), - "channel", original.channel().name()))); - return operations.save(result, actor, "ADMIN_REDRIVE"); - }); - } - - @Override - public AdminOperationResult reconcile(ReconcileCommand command, AdminActor actor) { - Objects.requireNonNull(command, "command"); - authorization.require(actor, NotificationAdminAuthority.RECONCILE); - // Every attempt is checked before any is touched, so a batch with one foreign id changes none. - // An id with no attempt behind it is left to reconciliation, which does nothing with it. - for (DeliveryAttemptId attemptId : command.attemptIds()) { - attempts - .snapshot(attemptId) - .ifPresent( - attempt -> - authorization.requireTenant( - actor, attempt.tenantId(), NotificationAdminAuthority.RECONCILE)); - } - - if (command.dryRun()) { - return new AdminOperationResult( - command.operationId(), - true, - command.attemptIds().size(), - Optional.empty(), - Optional.empty(), - Optional.empty(), - List.of("DRY_RUN")); - } - - // Outside a transaction: each reconcile asks a provider over the network and opens its own - // boundaries for what it writes. - return operationExecutor.executeOutsideTransaction( - command.operationId(), - commandFingerprints.of(command), - actor, - "ADMIN_RECONCILE", - () -> { - List reasons = new ArrayList<>(); - int reconciled = 0; - for (DeliveryAttemptId attemptId : command.attemptIds()) { - reconciliation.reconcile(attemptId); - reconciled++; - } - reasons.add(command.reason()); - - AdminOperationResult result = - new AdminOperationResult( - command.operationId(), - false, - reconciled, - Optional.empty(), - Optional.empty(), - Optional.empty(), - List.copyOf(reasons)); - return transactions.inWrite( - () -> { - audit.record( - new NotificationAuditEvent( - "ADMIN_RECONCILE", - actor.actorRef(), - Optional.of(command.reason()), - Optional.of(command.operationId()), - clock.instant(), - Map.of())); - return operations.save(result, actor, "ADMIN_RECONCILE"); - }); - }); - } - - @Override - public AdminOperationResult suppress(SuppressCommand command, AdminActor actor) { - Objects.requireNonNull(command, "command"); - authorization.require(actor, NotificationAdminAuthority.SUPPRESS); - authorization.requireTenant(actor, command.tenantId(), NotificationAdminAuthority.SUPPRESS); - - if (command.dryRun()) { - return new AdminOperationResult( - command.operationId(), - true, - 1, - Optional.empty(), - Optional.empty(), - Optional.empty(), - List.of("DRY_RUN")); - } - - String action = command.remove() ? "ADMIN_SUPPRESS_REMOVE" : "ADMIN_SUPPRESS_ADD"; - return operationExecutor.execute( - command.operationId(), - commandFingerprints.of(command), - actor, - action, - () -> { - int affected; - if (command.remove()) { - // Removal is by fingerprint match rather than by id, because an operator lifting a - // suppression knows the target, not the row identifier the platform assigned. - affected = - suppressions - .activeFor( - command.tenantId(), command.targetFingerprint(), clock.instant()) - .stream() - .map(entry -> suppressions.remove(command.tenantId(), entry.id())) - .filter(Optional::isPresent) - .count() - > 0 - ? 1 - : 0; - } else { - suppressions.upsert( - new SuppressionEntry( - // ids.nextId(), not UUID.randomUUID(). The identifier contract requires the - // generator port, and while this class lived in the outbound adapter the rule - // that says so did not apply to it — a random v4 id here is unordered, so the - // suppression table's index degrades the way the platform's own UUIDv7 policy - // exists to prevent. - new SuppressionId(ids.nextId()), - command.tenantId(), - command.scope(), - command.reason(), - command.targetFingerprint(), - Optional.empty(), - clock.instant(), - command.expiresAt(), - SuppressionSource.ADMIN)); - affected = 1; - } - - AdminOperationResult result = - new AdminOperationResult( - command.operationId(), - false, - affected, - Optional.empty(), - Optional.empty(), - Optional.empty(), - List.of(command.reasonText())); - audit.record( - new NotificationAuditEvent( - command.remove() ? "ADMIN_SUPPRESSION_REMOVED" : "ADMIN_SUPPRESSION_ADDED", - actor.actorRef(), - Optional.of(command.reason().name()), - Optional.of(command.operationId()), - clock.instant(), - Map.of())); - return operations.save(result, actor, action); - }); - } - - @Override - public AdminOperationResult setProviderState(SetProviderStateCommand command, AdminActor actor) { - Objects.requireNonNull(command, "command"); - authorization.require(actor, NotificationAdminAuthority.PROVIDER_CONTROL); - authorization.requireGlobal(actor, NotificationAdminAuthority.PROVIDER_CONTROL); - - if (command.dryRun()) { - return new AdminOperationResult( - command.operationId(), - true, - 1, - Optional.empty(), - Optional.empty(), - Optional.empty(), - List.of("DRY_RUN")); - } - - return operationExecutor.execute( - command.operationId(), - commandFingerprints.of(command), - actor, - "ADMIN_PROVIDER_STATE", - () -> { - boolean applied = - runtimes.setState(command.profileId(), command.desiredState(), command.reason()); - if (!applied) { - // The transition was refused — an authentication failure is not cleared by declaring - // the provider healthy, and a healthy runtime is not degraded on request. Recording - // success for a state change that did not happen tells an operator the provider is - // serving when it is not. - throw new ProviderConfigurationException( - NotificationFailureDescriptor.preDispatch( - NotificationFailureCode.PROVIDER_CONFIGURATION_INVALID, - FailureCategory.AUTHORIZATION)); - } - - AdminOperationResult result = - new AdminOperationResult( - command.operationId(), - false, - 1, - Optional.empty(), - Optional.empty(), - Optional.empty(), - List.of(command.reason())); - audit.record( - new NotificationAuditEvent( - "ADMIN_PROVIDER_STATE", - actor.actorRef(), - Optional.of(command.reason()), - Optional.of(command.operationId()), - clock.instant(), - Map.of( - "providerProfile", command.profileId().value(), - "status", command.desiredState().name()))); - return operations.save(result, actor, "ADMIN_PROVIDER_STATE"); - }); - } - - /** Current state of a provider runtime, for the health endpoint. */ - public ProviderRuntimeState providerState( - dev.caskeleton.application.notification.platform.api.ProviderProfileId profileId) { - return runtimes.state(profileId); - } -} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminPermissions.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminPermissions.java new file mode 100644 index 00000000..be4175d1 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminPermissions.java @@ -0,0 +1,27 @@ +package dev.caskeleton.application.notification.platform.admin; + +/** + * The permissions the four operator-plane use cases declare. + * + *

One per operation, like the {@link NotificationAdminAuthority} the operator also has to hold: + * the permission is what the transport's caller may invoke at all, the authority and tenant scope + * on the {@code AdminActor} are what the operation then checks against the delivery it touches. + * Which roles hold them by default is the notification module defaults' to say, not this type's: + * {@code docs/notification/operations.md} 「운영자 연산」. + */ +public final class NotificationAdminPermissions { + + /** Re-run one delivery. */ + public static final String REDRIVE = "notification-admin:redrive"; + + /** Ask providers about a batch of attempts. */ + public static final String RECONCILE = "notification-admin:reconcile"; + + /** Add or lift a suppression entry. */ + public static final String SUPPRESS = "notification-admin:suppress"; + + /** Change a provider profile's runtime state. */ + public static final String PROVIDER_CONTROL = "notification-admin:provider-control"; + + private NotificationAdminPermissions() {} +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminService.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminService.java deleted file mode 100644 index 5b003cc8..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminService.java +++ /dev/null @@ -1,22 +0,0 @@ -package dev.caskeleton.application.notification.platform.admin; - -/** - * N4 operator plane. - * - *

Every operation requires a separate authority, an idempotent operation id, a reason and an - * audit record. It is deliberately not reachable from the application-facing APIs. - */ -public interface NotificationAdminService { - - /** Re-run a delivery while preserving its logical identity. */ - AdminOperationResult redrive(RedriveCommand command, AdminActor actor); - - /** Force reconciliation of a bounded batch. */ - AdminOperationResult reconcile(ReconcileCommand command, AdminActor actor); - - /** Add or remove a suppression entry. */ - AdminOperationResult suppress(SuppressCommand command, AdminActor actor); - - /** Change a provider runtime state. */ - AdminOperationResult setProviderState(SetProviderStateCommand command, AdminActor actor); -} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SetProviderStateCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ProviderStateRequest.java similarity index 91% rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SetProviderStateCommand.java rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ProviderStateRequest.java index c942f21f..033884a4 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SetProviderStateCommand.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ProviderStateRequest.java @@ -5,14 +5,14 @@ import dev.caskeleton.application.notification.platform.provider.ProviderRuntime import java.util.Objects; /** Enable, disable or drain a provider profile. */ -public record SetProviderStateCommand( +public record ProviderStateRequest( String operationId, ProviderProfileId profileId, ProviderRuntimeState desiredState, String reason, boolean dryRun) { - public SetProviderStateCommand { + public ProviderStateRequest { Objects.requireNonNull(operationId, "operationId"); Objects.requireNonNull(profileId, "profileId"); Objects.requireNonNull(desiredState, "desiredState"); diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesCommand.java new file mode 100644 index 00000000..8f23b5aa --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesCommand.java @@ -0,0 +1,22 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.command.Command; +import java.util.Objects; + +/** + * An operator's reconciliation of a bounded batch of attempts. + * + *

The request and the operator are apart because the operation id's fingerprint covers the + * request only: the same operator retrying the same request replays it. + * + * @param request what to do + * @param actor who asks, with the operator authorities and tenant scope the transport resolved + */ +public record ReconcileDeliveriesCommand(ReconcileRequest request, AdminActor actor) + implements Command { + + public ReconcileDeliveriesCommand { + Objects.requireNonNull(request, "request"); + Objects.requireNonNull(actor, "actor"); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesUseCase.java new file mode 100644 index 00000000..aba76aca --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesUseCase.java @@ -0,0 +1,143 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; +import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort; +import dev.caskeleton.application.notification.platform.dispatch.ReconciliationService; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; +import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.time.Clock; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Asks providers, on an operator's request, what happened to a bounded batch of attempts. + * + *

Every attempt is checked against the operator's tenant scope before any is touched, so a batch + * with one foreign id changes none. An id with no attempt behind it is left to reconciliation, + * which does nothing with it. + * + *

The provider calls run with no transaction open: each one asks over the network, and inside a + * write transaction it held a pooled connection for as long as the provider took, once per attempt. + * The operation id is claimed in its own transaction first, and the audit row and result are + * written in one at the end. + */ +@RequiresPermission(NotificationAdminPermissions.RECONCILE) +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, + externalOutboundAllowed = true, + crossTenantAdmin = true) +// Not final: @RequiresPermission is applied through a CGLIB proxy. +public class ReconcileDeliveriesUseCase + implements CommandUseCase { + + private static final String ACTION = "ADMIN_RECONCILE"; + + private final AdminAuthorizationGuard authorization; + private final DeliveryAttemptStorePort attempts; + private final ReconciliationService reconciliation; + private final AdminOperationStorePort operations; + private final AdminOperationExecutor operationExecutor; + private final AdminOperationRequestFingerprint requestFingerprints = + new AdminOperationRequestFingerprint(); + private final NotificationAuditPort audit; + private final TransactionPort transactions; + private final Clock clock; + + public ReconcileDeliveriesUseCase( + AdminAuthorizationGuard authorization, + DeliveryAttemptStorePort attempts, + ReconciliationService reconciliation, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + Clock clock) { + this.authorization = Objects.requireNonNull(authorization, "authorization"); + this.attempts = Objects.requireNonNull(attempts, "attempts"); + this.reconciliation = Objects.requireNonNull(reconciliation, "reconciliation"); + this.operations = Objects.requireNonNull(operations, "operations"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions); + this.audit = Objects.requireNonNull(audit, "audit"); + this.clock = Objects.requireNonNull(clock, "clock"); + } + + @Override + public AdminOperationResult handle(ReconcileDeliveriesCommand batch) { + Objects.requireNonNull(batch, "batch"); + ReconcileRequest request = batch.request(); + AdminActor actor = batch.actor(); + authorization.require(actor, NotificationAdminAuthority.RECONCILE); + // Read in a read transaction of its own: the providers are asked with none open, so the tenant + // check cannot share the boundary the answers are recorded in. + transactions.inRead( + () -> { + for (DeliveryAttemptId attemptId : request.attemptIds()) { + attempts + .snapshot(attemptId) + .ifPresent( + attempt -> + authorization.requireTenant( + actor, attempt.tenantId(), NotificationAdminAuthority.RECONCILE)); + } + return null; + }); + + if (request.dryRun()) { + return new AdminOperationResult( + request.operationId(), + true, + request.attemptIds().size(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of("DRY_RUN")); + } + + return operationExecutor.executeOutsideTransaction( + request.operationId(), + requestFingerprints.of(request), + actor, + ACTION, + () -> reconcile(request, actor)); + } + + private AdminOperationResult reconcile(ReconcileRequest request, AdminActor actor) { + int reconciled = 0; + for (DeliveryAttemptId attemptId : request.attemptIds()) { + reconciliation.reconcile(attemptId); + reconciled++; + } + AdminOperationResult result = + new AdminOperationResult( + request.operationId(), + false, + reconciled, + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of(request.reason())); + return transactions.inWrite( + () -> { + audit.record( + new NotificationAuditEvent( + ACTION, + actor.actorRef(), + Optional.of(request.reason()), + Optional.of(request.operationId()), + clock.instant(), + Map.of())); + return operations.save(result, actor, ACTION); + }); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileRequest.java similarity index 92% rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileCommand.java rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileRequest.java index b62d5b7d..491b8526 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileCommand.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileRequest.java @@ -5,12 +5,12 @@ import java.util.List; import java.util.Objects; /** Operator-triggered reconciliation over a bounded batch of attempts. */ -public record ReconcileCommand( +public record ReconcileRequest( String operationId, List attemptIds, String reason, boolean dryRun) { public static final int MAX_BATCH = 200; - public ReconcileCommand { + public ReconcileRequest { Objects.requireNonNull(operationId, "operationId"); attemptIds = List.copyOf(Objects.requireNonNull(attemptIds, "attemptIds")); Objects.requireNonNull(reason, "reason"); diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryCommand.java new file mode 100644 index 00000000..ab395b61 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryCommand.java @@ -0,0 +1,21 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.command.Command; +import java.util.Objects; + +/** + * An operator's redrive of one delivery attempt. + * + *

The request and the operator are apart because the operation id's fingerprint covers the + * request only: the same operator retrying the same request replays it. + * + * @param request what to do + * @param actor who asks, with the operator authorities and tenant scope the transport resolved + */ +public record RedriveDeliveryCommand(RedriveRequest request, AdminActor actor) implements Command { + + public RedriveDeliveryCommand { + Objects.requireNonNull(request, "request"); + Objects.requireNonNull(actor, "actor"); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryUseCase.java new file mode 100644 index 00000000..3b96e42c --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryUseCase.java @@ -0,0 +1,149 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState; +import dev.caskeleton.application.notification.platform.callback.DeliveryAttemptSnapshot; +import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort; +import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; +import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.time.Clock; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Re-runs one delivery while preserving its logical identity. + * + *

The logical identities are kept and only the next attempt is new, so the history stays one + * story rather than becoming two unrelated notifications. An operator retrying after a timeout must + * not send the message twice — exactly the failure a redrive is trying to repair — so the operation + * id is claimed before the delivery is touched, in the same transaction. + * + *

A dry run reads and reports but writes nothing. + */ +@RequiresPermission(NotificationAdminPermissions.REDRIVE) +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, + // The audit port is the notification adapter's. + externalOutboundAllowed = true, + crossTenantAdmin = true) +// Not final: @RequiresPermission is applied through a CGLIB proxy. +public class RedriveDeliveryUseCase + implements CommandUseCase { + + private static final String ACTION = "ADMIN_REDRIVE"; + + private final AdminAuthorizationGuard authorization; + private final DuplicateRiskGuard duplicateRiskGuard; + private final DeliveryAttemptStorePort attempts; + private final RecipientDeliveryStorePort recipients; + private final AdminOperationStorePort operations; + private final AdminOperationExecutor operationExecutor; + private final AdminOperationRequestFingerprint requestFingerprints = + new AdminOperationRequestFingerprint(); + private final NotificationAuditPort audit; + private final TransactionPort transactions; + private final Clock clock; + + public RedriveDeliveryUseCase( + AdminAuthorizationGuard authorization, + DuplicateRiskGuard duplicateRiskGuard, + DeliveryAttemptStorePort attempts, + RecipientDeliveryStorePort recipients, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + Clock clock) { + this.authorization = Objects.requireNonNull(authorization, "authorization"); + this.duplicateRiskGuard = Objects.requireNonNull(duplicateRiskGuard, "duplicateRiskGuard"); + this.attempts = Objects.requireNonNull(attempts, "attempts"); + this.recipients = Objects.requireNonNull(recipients, "recipients"); + this.operations = Objects.requireNonNull(operations, "operations"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions); + this.audit = Objects.requireNonNull(audit, "audit"); + this.clock = Objects.requireNonNull(clock, "clock"); + } + + @Override + public AdminOperationResult handle(RedriveDeliveryCommand delivery) { + Objects.requireNonNull(delivery, "delivery"); + RedriveRequest request = delivery.request(); + AdminActor actor = delivery.actor(); + authorization.require(actor, NotificationAdminAuthority.REDRIVE); + + if (request.dryRun()) { + DeliveryAttemptSnapshot original = transactions.inRead(() -> checkedOriginal(request, actor)); + return new AdminOperationResult( + request.operationId(), + true, + 1, + Optional.of(original.notificationId()), + Optional.of(original.recipientDeliveryId()), + Optional.empty(), + List.of("DRY_RUN")); + } + + // The checks read the attempt in the transaction that claims the operation and writes, so the + // attempt they approved is the one redriven. + return transactions.inWrite( + () -> { + DeliveryAttemptSnapshot original = checkedOriginal(request, actor); + return operationExecutor.runClaimed( + request.operationId(), + requestFingerprints.of(request), + actor, + ACTION, + () -> redrive(request, actor, original)); + }); + } + + private DeliveryAttemptSnapshot checkedOriginal(RedriveRequest request, AdminActor actor) { + DeliveryAttemptSnapshot original = + attempts + .snapshot(request.attemptId()) + .orElseThrow(() -> new IllegalStateException("delivery attempt is not available")); + authorization.requireTenant(actor, original.tenantId(), NotificationAdminAuthority.REDRIVE); + duplicateRiskGuard.verify(original, request.approveDuplicateRisk()); + return original; + } + + private AdminOperationResult redrive( + RedriveRequest request, AdminActor actor, DeliveryAttemptSnapshot original) { + recipients.transition( + original.recipientDeliveryId(), + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(clock.instant())); + + AdminOperationResult result = + new AdminOperationResult( + request.operationId(), + false, + 1, + Optional.of(original.notificationId()), + Optional.of(original.recipientDeliveryId()), + Optional.empty(), + List.of(request.reason())); + audit.record( + new NotificationAuditEvent( + ACTION, + actor.actorRef(), + Optional.of(request.reason()), + Optional.of(request.operationId()), + clock.instant(), + Map.of( + "provider", original.providerId().value(), + "channel", original.channel().name()))); + return operations.save(result, actor, ACTION); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveRequest.java similarity index 93% rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveCommand.java rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveRequest.java index fb9f031c..fe973b96 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveCommand.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveRequest.java @@ -9,14 +9,14 @@ import java.util.Objects; *

A redrive reuses the pinned template version and rendered digest: it re-executes the original * notification rather than sending a new, possibly different message under an old identity. */ -public record RedriveCommand( +public record RedriveRequest( String operationId, DeliveryAttemptId attemptId, String reason, boolean approveDuplicateRisk, boolean dryRun) { - public RedriveCommand { + public RedriveRequest { Objects.requireNonNull(operationId, "operationId"); Objects.requireNonNull(attemptId, "attemptId"); Objects.requireNonNull(reason, "reason"); diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressionRequest.java similarity index 94% rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressCommand.java rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressionRequest.java index 1c4a3764..891d9698 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressCommand.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressionRequest.java @@ -8,7 +8,7 @@ import java.util.Objects; import java.util.Optional; /** Add or remove a suppression entry as an operator. */ -public record SuppressCommand( +public record SuppressionRequest( String operationId, TenantId tenantId, SuppressionScope scope, @@ -19,7 +19,7 @@ public record SuppressCommand( String reasonText, boolean dryRun) { - public SuppressCommand { + public SuppressionRequest { Objects.requireNonNull(operationId, "operationId"); Objects.requireNonNull(tenantId, "tenantId"); Objects.requireNonNull(scope, "scope"); diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ProviderEventProjectionService.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ProviderEventProjectionService.java index 7db02dfb..e24d819c 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ProviderEventProjectionService.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ProviderEventProjectionService.java @@ -3,6 +3,7 @@ package dev.caskeleton.application.notification.platform.callback; import dev.caskeleton.application.notification.platform.observation.NotificationMetricName; import dev.caskeleton.application.notification.platform.observation.NotificationMetricsPort; import dev.caskeleton.application.transaction.TransactionPort; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -72,6 +73,26 @@ public final class ProviderEventProjectionService { transactions.inWrite(() -> ledger.markFailed(event.id(), reasonCode)); } + /** + * Stored events that are not matched to an attempt yet, read in a read transaction of its own. + * + * @param limit the most events to return + * @return the oldest unmatched events + */ + public List unmatched(int limit) { + return transactions.inRead(() -> ledger.unmatched(limit)); + } + + /** + * Stored events whose projection has not run or failed, read in a read transaction of its own. + * + * @param limit the most events to return + * @return the oldest events awaiting projection + */ + public List pendingProjection(int limit) { + return transactions.inRead(() -> ledger.pendingProjection(limit)); + } + private Optional projectInTransaction(ProviderEventRecord event) { Optional attempt = resolve(event); if (attempt.isEmpty()) { diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsCommand.java new file mode 100644 index 00000000..b9cb86c8 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsCommand.java @@ -0,0 +1,6 @@ +package dev.caskeleton.application.notification.platform.callback; + +import dev.caskeleton.application.command.Command; + +/** One replay pass over the stored, unprojected provider events; its bound is the use case's. */ +public record ReplayProviderEventsCommand() implements Command {} diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ProviderEventReplayWorker.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsUseCase.java similarity index 50% rename from src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ProviderEventReplayWorker.java rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsUseCase.java index 50292159..833e4d2a 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ProviderEventReplayWorker.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsUseCase.java @@ -1,8 +1,11 @@ -package dev.caskeleton.adapter.outbound.notification.platform.dispatch; +package dev.caskeleton.application.notification.platform.callback; -import dev.caskeleton.application.notification.platform.callback.ProviderEventLedger; -import dev.caskeleton.application.notification.platform.callback.ProviderEventProjectionService; -import dev.caskeleton.application.notification.platform.callback.ProviderEventRecord; +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.usecase.CommandUseCase; import java.util.List; import java.util.Objects; @@ -19,26 +22,41 @@ import java.util.Objects; * ingestion path deliberately stores such an event as {@code PENDING} rather than dropping it. With * nothing replaying it, "we will match it later" was true only in the comment: the delivery stayed * unconfirmed forever and the callback that would have confirmed it sat in a table. + * + *

A use case, not a worker of the notification adapter, which it used to be (round9 review + * 2026-11-12, RR9-05-06, RR9-05-08). It holds no transaction and touches no repository itself: + * {@link ProviderEventProjectionService} reads each list in a read transaction and projects, or + * marks failed, each event in a write transaction of its own, so one event's failure does not roll + * back the others. + * + *

Not a bean. The notification composition's background workers construct it and keep it. */ -public final class ProviderEventReplayWorker { +@SystemInvoked( + reason = + "the notification background workers run it on their scheduler thread, which has no" + + " Authentication") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + // Every read and write is the projection service's, each in a boundary of that service's. + repositoryAccess = RepositoryAccess.NONE) +public class ReplayProviderEventsUseCase + implements CommandUseCase { - private final ProviderEventLedger ledger; private final ProviderEventProjectionService projection; private final int batchSize; /** - * Creates the replay worker. + * Creates the use case. * - * @param ledger the stored events - * @param projection the projection the events feed + * @param projection reads the stored events and projects them * @param batchSize how many events one pass handles + * @throws IllegalStateException if the batch size is not positive, which is the assembly's fault */ - public ProviderEventReplayWorker( - ProviderEventLedger ledger, ProviderEventProjectionService projection, int batchSize) { - this.ledger = Objects.requireNonNull(ledger, "ledger"); + public ReplayProviderEventsUseCase(ProviderEventProjectionService projection, int batchSize) { this.projection = Objects.requireNonNull(projection, "projection"); if (batchSize < 1) { - throw new IllegalArgumentException("batchSize"); + throw new IllegalStateException("batchSize must be positive"); } this.batchSize = batchSize; } @@ -52,10 +70,12 @@ public final class ProviderEventReplayWorker { * * @return how many events projected successfully */ - public int replayOnce() { + @Override + public Integer handle(ReplayProviderEventsCommand command) { + Objects.requireNonNull(command, "command"); int projected = 0; - projected += replay(ledger.unmatched(batchSize)); - projected += replay(ledger.pendingProjection(batchSize)); + projected += replay(projection.unmatched(batchSize)); + projected += replay(projection.pendingProjection(batchSize)); return projected; } @@ -69,8 +89,8 @@ public final class ProviderEventReplayWorker { projected++; } } catch (RuntimeException failure) { - // Through the projection service, which opens the transaction: this worker has none, and - // marking the event is a modifying query that throws without one. + // Through the projection service, which opens the transaction: this use case holds none + // here, and marking the event is a modifying query that throws without one. projection.markFailed(event, failure.getClass().getSimpleName()); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesCommand.java new file mode 100644 index 00000000..8a7e0b00 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesCommand.java @@ -0,0 +1,24 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.command.Command; +import java.time.Duration; +import java.util.Objects; + +/** + * One claim of due deliveries for a scheduler worker. + * + * @param workerId the claiming worker, including its process incarnation + * @param limit the most deliveries to claim; only as many as can start now + * @param leaseDuration how long the claimed deliveries are the worker's + */ +public record ClaimDueDeliveriesCommand(String workerId, int limit, Duration leaseDuration) + implements Command { + + public ClaimDueDeliveriesCommand { + Objects.requireNonNull(workerId, "workerId"); + Objects.requireNonNull(leaseDuration, "leaseDuration"); + if (limit < 1) { + throw new IllegalStateException("limit must be positive"); + } + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCase.java new file mode 100644 index 00000000..941fb393 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCase.java @@ -0,0 +1,47 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.util.List; +import java.util.Objects; + +/** + * Claims the deliveries that are due, each under a fresh lease and fence. + * + *

The first of the scheduler's two steps; {@link DispatchClaimedDeliveryUseCase} is the second, + * once per claimed delivery. The worker that drives them owns only threads and concurrency permits: + * what is claimed, and under which lease, is decided here (round9 review 2026-11-12, RR9-05-06). + * + *

Not a bean. The notification composition's scheduler worker constructs it and keeps it. + */ +@SystemInvoked( + reason = + "the notification scheduler worker claims on its polling thread, which has no" + + " Authentication") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.NOT_IDEMPOTENT, + repositoryAccess = RepositoryAccess.WRITE_REPOSITORY) +public class ClaimDueDeliveriesUseCase + implements CommandUseCase> { + + private final RecipientLeaseStorePort leases; + private final TransactionPort transactions; + + public ClaimDueDeliveriesUseCase(RecipientLeaseStorePort leases, TransactionPort transactions) { + this.leases = Objects.requireNonNull(leases, "leases"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + } + + @Override + public List handle(ClaimDueDeliveriesCommand command) { + Objects.requireNonNull(command, "command"); + return transactions.inWrite( + () -> leases.claim(command.workerId(), command.limit(), command.leaseDuration())); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryCommand.java new file mode 100644 index 00000000..ab060270 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryCommand.java @@ -0,0 +1,16 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.command.Command; +import java.util.Objects; + +/** + * One claimed delivery to send. + * + * @param lease the claim the delivery is held under + */ +public record DispatchClaimedDeliveryCommand(RecipientLease lease) implements Command { + + public DispatchClaimedDeliveryCommand { + Objects.requireNonNull(lease, "lease"); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCase.java new file mode 100644 index 00000000..68e33fde --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCase.java @@ -0,0 +1,46 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.util.Objects; + +/** + * Sends one claimed delivery, under its lease. + * + *

{@code RepositoryAccess.NONE}: the delivery is {@link NotificationDispatchService}'s, which + * commits the attempt row before the provider call and records the outcome after it, each in a + * boundary of its own. One around the whole dispatch would hold a connection across the provider + * call. + * + *

Not a bean. The notification composition's scheduler worker constructs it and keeps it. + */ +@SystemInvoked( + reason = + "the notification scheduler worker dispatches on its executor threads, which have no" + + " Authentication") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.NOT_IDEMPOTENT, + repositoryAccess = RepositoryAccess.NONE, + // The dispatch reaches the provider. + externalOutboundAllowed = true) +public class DispatchClaimedDeliveryUseCase + implements CommandUseCase { + + private final NotificationDispatchService dispatcher; + + public DispatchClaimedDeliveryUseCase(NotificationDispatchService dispatcher) { + this.dispatcher = Objects.requireNonNull(dispatcher, "dispatcher"); + } + + @Override + public Void handle(DispatchClaimedDeliveryCommand command) { + Objects.requireNonNull(command, "command"); + dispatcher.dispatch(command.lease()); + return null; + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsCommand.java new file mode 100644 index 00000000..2308ac3f --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsCommand.java @@ -0,0 +1,6 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.command.Command; + +/** One reconciliation pass over the due jobs; its bounds are the use case's. */ +public record ReconcileOutstandingAttemptsCommand() implements Command {} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCase.java new file mode 100644 index 00000000..4edf380d --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCase.java @@ -0,0 +1,185 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; +import dev.caskeleton.application.notification.platform.provider.ReconciliationResult; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; + +/** + * Asks the provider what happened to attempts whose outcome is unknown. + * + *

Reconciliation only ever ran when a lease expired and recovery walked past an incomplete + * attempt. An ambiguous submission whose worker exited cleanly — the common case, because a worker + * that records AMBIGUOUS and then finishes its shift has not crashed — was never asked about again. + * The delivery sat {@code RECONCILIATION_REQUIRED} indefinitely, which reads as a queue that + * stopped rather than as an outcome nobody knows. + * + *

An unsupported provider is not retried in a loop. Without a status-query capability the answer + * will not change, so the job is completed and the attempt stays visibly ambiguous for an operator + * — a busy loop against a capability that does not exist is how a background worker burns a + * connection pool while achieving nothing. + * + *

Every instance runs this worker, so the claim is a lease: the jobs a pass takes are this + * instance's until {@code claimLease} has passed, and no transaction is held while the provider is + * asked. A pass that outlives its lease may find a job taken over by another instance; the store + * then ignores this pass's write for it, so the job is asked about twice but recorded once. + * + *

A use case, not a worker of the notification adapter, which it used to be: when to ask again, + * when to give up and what each answer means are the platform's decisions, and the adapter's job is + * speaking provider protocols. The store writes each run in a write transaction of their own; the + * provider is asked with none open (round9 review 2026-11-12, RR9-05-06, RR9-05-08). + * + *

Not a bean. The notification composition's background workers construct it and keep it. + */ +@SystemInvoked( + reason = + "the notification background workers run it on their scheduler thread, which has no" + + " Authentication; an operator's run is ReconcileDeliveriesUseCase") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, + // Each job asks a provider what happened. + externalOutboundAllowed = true) +public class ReconcileOutstandingAttemptsUseCase + implements CommandUseCase { + + private final ReconciliationJobStorePort jobs; + private final Function reconciler; + private final TransactionPort transactions; + private final Clock clock; + private final Duration retryBackoff; + private final Duration claimLease; + private final int batchSize; + private final int maxAttempts; + + /** + * Creates the use case. + * + *

The timing is derived here rather than in the assembly, because when to ask again and when + * to give up are this use case's decisions. A failed or crashed question waits one delivery lease + * before it is asked again. The claim lease reuses the delivery lease rather than adding a + * setting: one pass asks up to {@code batchSize} providers in turn, so a pass slower than the + * lease lets another instance re-ask its remaining jobs, and the claim fence keeps those answers + * recorded once. A job may ask once plus the delivery's additional attempts, the same budget a + * delivery has for being sent. + * + * @param jobs the outstanding questions + * @param reconciler asks one attempt's provider; it opens its own boundaries for what it writes + * @param transactions opens the write transaction around each job-store write + * @param clock the clock + * @param deliveryLease the delivery lease: the retry delay and how long one pass owns its jobs + * @param batchSize how many jobs one pass handles + * @param maxAdditionalAttempts how many more times a job may ask after its first question + * @throws IllegalStateException if a bound is not positive, which is the assembly's fault + */ + public ReconcileOutstandingAttemptsUseCase( + ReconciliationJobStorePort jobs, + Function reconciler, + TransactionPort transactions, + Clock clock, + Duration deliveryLease, + int batchSize, + int maxAdditionalAttempts) { + this.jobs = Objects.requireNonNull(jobs, "jobs"); + this.reconciler = Objects.requireNonNull(reconciler, "reconciler"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + this.clock = Objects.requireNonNull(clock, "clock"); + Objects.requireNonNull(deliveryLease, "deliveryLease"); + if (deliveryLease.isNegative() || deliveryLease.isZero()) { + throw new IllegalStateException("deliveryLease must be positive"); + } + if (batchSize < 1) { + throw new IllegalStateException("batchSize must be positive"); + } + if (maxAdditionalAttempts < 0) { + throw new IllegalStateException("maxAdditionalAttempts must not be negative"); + } + this.retryBackoff = deliveryLease; + this.claimLease = deliveryLease; + this.batchSize = batchSize; + this.maxAttempts = maxAdditionalAttempts + 1; + } + + /** + * Runs one pass. + * + * @return how many jobs reached a terminal decision + */ + @Override + public Integer handle(ReconcileOutstandingAttemptsCommand command) { + Objects.requireNonNull(command, "command"); + List due = + transactions.inWrite(() -> jobs.claimDue(batchSize, clock.instant(), claimLease)); + int settled = 0; + for (ReconciliationJob job : due) { + // One job's failure is not the pass's: a provider that is refusing connections would + // otherwise stop every other provider's jobs behind it. + try { + if (handle(job)) { + settled++; + } + } catch (RuntimeException failure) { + reschedule(job, failure.getClass().getSimpleName(), clock.instant().plus(retryBackoff)); + } + } + return settled; + } + + private boolean handle(ReconciliationJob job) { + if (job.attempts() >= maxAttempts) { + // Asked enough times. Completing the job leaves the attempt ambiguous and visible rather + // than asking forever; an outcome that has not arrived after this many tries is an + // operator's decision, not a scheduler's. + transactions.inWrite( + () -> { + jobs.reschedule(job, "MAX_ATTEMPTS", clock.instant().plus(retryBackoff)); + jobs.complete(job); + }); + return true; + } + ReconciliationResult result = reconciler.apply(job.attemptId()); + return switch (result) { + case ReconciliationResult.Confirmed ignored -> { + complete(job); + yield true; + } + case ReconciliationResult.Unsupported ignored -> { + // The capability does not exist; asking again cannot change that. + complete(job); + yield true; + } + case ReconciliationResult.StillUnknown stillUnknown -> { + reschedule(job, "STILL_UNKNOWN", stillUnknown.nextCheckAt()); + yield false; + } + case ReconciliationResult.Failed failed -> { + if (failed.retryable()) { + reschedule(job, failed.code(), clock.instant().plus(retryBackoff)); + yield false; + } + complete(job); + yield true; + } + }; + } + + private void complete(ReconciliationJob job) { + transactions.inWrite(() -> jobs.complete(job)); + } + + private void reschedule(ReconciliationJob job, String result, Instant nextCheckAt) { + transactions.inWrite(() -> jobs.reschedule(job, result, nextCheckAt)); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesCommand.java new file mode 100644 index 00000000..6ed19414 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesCommand.java @@ -0,0 +1,6 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.command.Command; + +/** One lease-recovery pass; its bounds are the recovery service's. */ +public record RecoverAbandonedDeliveriesCommand() implements Command {} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCase.java new file mode 100644 index 00000000..5d51446a --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCase.java @@ -0,0 +1,47 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.util.Objects; + +/** + * Returns deliveries a dead worker left dispatching to the queue, or to reconciliation when an + * attempt may have reached the provider. + * + *

{@code RepositoryAccess.NONE}: the pass is {@link LeaseRecoveryService}'s, which opens a + * boundary per delivery it settles and none around the provider question. + * + *

Not a bean. The notification composition's background workers construct it and keep it. + */ +@SystemInvoked( + reason = + "the notification background workers run it on their scheduler thread, which has no" + + " Authentication") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.NONE, + // Recovery may ask a provider what happened to an attempt. + externalOutboundAllowed = true) +public class RecoverAbandonedDeliveriesUseCase + implements CommandUseCase { + + private final LeaseRecoveryService recovery; + + public RecoverAbandonedDeliveriesUseCase(LeaseRecoveryService recovery) { + this.recovery = Objects.requireNonNull(recovery, "recovery"); + } + + /** + * @return how many deliveries the pass settled + */ + @Override + public Integer handle(RecoverAbandonedDeliveriesCommand command) { + Objects.requireNonNull(command, "command"); + return recovery.recoverOnce(); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogQuery.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogQuery.java new file mode 100644 index 00000000..b46a378e --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogQuery.java @@ -0,0 +1,6 @@ +package dev.caskeleton.application.notification.platform.observation; + +import dev.caskeleton.application.query.Query; + +/** Asks how many deliveries are due and not yet sent; it has no parameters. */ +public record MeasureDispatchBacklogQuery() implements Query {} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogUseCase.java new file mode 100644 index 00000000..efce336c --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogUseCase.java @@ -0,0 +1,52 @@ +package dev.caskeleton.application.notification.platform.observation; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.QueryUseCase; +import java.util.Objects; + +/** + * Measures the dispatch backlog the scheduler publishes as its queue-depth gauge on every poll. + * + *

The depth alone, one statement in a read transaction of its own: the full serving snapshot is + * several statements and is the health endpoint's. The scheduler read the serving-state port + * directly, with no use case and no read boundary in between (round9 review 2026-11-12, application + * structure). The notification composition constructs it for the scheduler; nothing else needs it. + */ +@UseCaseCapability( + transactionMode = TransactionMode.READ_ONLY, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.READ_REPOSITORY) +public class MeasureDispatchBacklogUseCase + implements QueryUseCase { + + private final NotificationServingStatePort servingState; + private final TransactionPort transactions; + + /** + * Creates the use case. + * + * @param servingState the serving-state read model + * @param transactions opens the read transaction around the measurement + */ + public MeasureDispatchBacklogUseCase( + NotificationServingStatePort servingState, TransactionPort transactions) { + this.servingState = Objects.requireNonNull(servingState, "servingState"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + } + + /** + * Measures the backlog. + * + * @param query the measurement request + * @return how many deliveries are due and not yet sent + */ + @Override + public Long handle(MeasureDispatchBacklogQuery query) { + Objects.requireNonNull(query, "query"); + return transactions.inRead(servingState::backlogDepth); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/DefaultNotificationRetryPolicy.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/DefaultNotificationRetryPolicy.java index 9953dca2..56088cdf 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/DefaultNotificationRetryPolicy.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/DefaultNotificationRetryPolicy.java @@ -19,14 +19,15 @@ public final class DefaultNotificationRetryPolicy implements NotificationRetryPo private final JitterSource jitter; private final Duration reconcileDelay; - public DefaultNotificationRetryPolicy( - RetryBackoff backoff, JitterSource jitter, Duration reconcileDelay) { - this.backoff = Objects.requireNonNull(backoff, "backoff"); + /** + * @param timing the backoff and the reconcile delay, already checked by {@link RetryTiming} + * @param jitter the randomness the backoff draws from + */ + public DefaultNotificationRetryPolicy(RetryTiming timing, JitterSource jitter) { + Objects.requireNonNull(timing, "timing"); + this.backoff = timing.backoff(); this.jitter = Objects.requireNonNull(jitter, "jitter"); - this.reconcileDelay = Objects.requireNonNull(reconcileDelay, "reconcileDelay"); - if (reconcileDelay.isNegative()) { - throw new IllegalArgumentException("reconcileDelay"); - } + this.reconcileDelay = timing.reconcileDelay(); } @Override diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/EligibilityResult.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/EligibilityResult.java index 2e478427..fb8aa0b3 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/EligibilityResult.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/EligibilityResult.java @@ -9,7 +9,7 @@ public record EligibilityResult(boolean allowed, List reasonCodes) { public EligibilityResult { reasonCodes = List.copyOf(Objects.requireNonNull(reasonCodes, "reasonCodes")); if (!allowed && reasonCodes.isEmpty()) { - throw new IllegalArgumentException("a block must carry at least one reason code"); + throw new IllegalStateException("a block must carry at least one reason code"); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBackoff.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBackoff.java index f6ba8ab3..ab282fbb 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBackoff.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBackoff.java @@ -30,10 +30,11 @@ public record RetryBackoff(Duration base, Duration max) { Objects.requireNonNull(base, "base"); Objects.requireNonNull(max, "max"); if (base.isNegative() || base.isZero()) { - throw new IllegalArgumentException("base"); + throw new IllegalStateException("the backoff base must be positive, was " + base); } if (max.compareTo(base) < 0) { - throw new IllegalArgumentException("max must not be smaller than base"); + throw new IllegalStateException( + "the backoff max (" + max + ") must not be shorter than its base (" + base + ")"); } } @@ -42,7 +43,7 @@ public record RetryBackoff(Duration base, Duration max) { Objects.requireNonNull(retryAfter, "retryAfter"); Objects.requireNonNull(jitter, "jitter"); if (attemptNumber < 1) { - throw new IllegalArgumentException("attemptNumber"); + throw new IllegalStateException("attemptNumber must be >= 1, was " + attemptNumber); } Duration jittered = new Backoff(base, max).fullJitter(attemptNumber, asRandom(jitter)); Duration computed = jittered.compareTo(base) < 0 ? base : jittered; diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBudget.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBudget.java index 796b56da..88f519b6 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBudget.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBudget.java @@ -10,7 +10,8 @@ public record RetryBudget(int maxAdditionalAttempts, int consumed) { public RetryBudget { if (maxAdditionalAttempts < 0 || consumed < 0) { - throw new IllegalArgumentException("budget values must not be negative"); + throw new IllegalStateException( + "budget values must not be negative, were " + maxAdditionalAttempts + " and " + consumed); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryContext.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryContext.java index 500992bf..f2b87992 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryContext.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryContext.java @@ -47,7 +47,7 @@ public record RetryContext( Objects.requireNonNull(estimatedDispatchDuration, "estimatedDispatchDuration"); Objects.requireNonNull(now, "now"); if (attemptNumber < 1) { - throw new IllegalArgumentException("attemptNumber"); + throw new IllegalStateException("attemptNumber must be >= 1, was " + attemptNumber); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryDecision.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryDecision.java index f38b9657..4bd55a18 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryDecision.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryDecision.java @@ -12,7 +12,7 @@ public sealed interface RetryDecision { public RetryAfter { Objects.requireNonNull(delay, "delay"); if (delay.isNegative()) { - throw new IllegalArgumentException("delay"); + throw new IllegalStateException("delay must not be negative, was " + delay); } } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryTiming.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryTiming.java new file mode 100644 index 00000000..4f6769ac --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryTiming.java @@ -0,0 +1,31 @@ +package dev.caskeleton.application.notification.platform.policy; + +import java.time.Duration; +import java.util.Objects; + +/** + * When a failed delivery is tried again: the backoff between attempts, and how long an ambiguous + * submission waits before the provider is asked what happened. + * + *

One value so its rules have one home. The retry settings build it from their keys and the + * policy reads it; neither restates what a valid timing is. + * + * @param backoff the delay between attempts + * @param reconcileDelay how long after an ambiguous submission reconciliation first asks; zero asks + * at once + */ +public record RetryTiming(RetryBackoff backoff, Duration reconcileDelay) { + + /** The shipped timing: the default backoff, reconciliation after a minute. */ + public static final RetryTiming DEFAULT = + new RetryTiming(RetryBackoff.DEFAULT, Duration.ofMinutes(1)); + + public RetryTiming { + Objects.requireNonNull(backoff, "backoff"); + Objects.requireNonNull(reconcileDelay, "reconcileDelay"); + if (reconcileDelay.isNegative()) { + throw new IllegalStateException( + "the reconcile delay must not be negative, was " + reconcileDelay); + } + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RouteCandidate.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RouteCandidate.java index a55a3ee6..8773a4c3 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RouteCandidate.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RouteCandidate.java @@ -19,7 +19,7 @@ public record RouteCandidate( Objects.requireNonNull(contactPointId, "contactPointId"); Objects.requireNonNull(providerProfileId, "providerProfileId"); if (routeIndex < 0) { - throw new IllegalArgumentException("routeIndex"); + throw new IllegalStateException("routeIndex must be >= 0, was " + routeIndex); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingContext.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingContext.java index 3185d341..7b5b98f1 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingContext.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingContext.java @@ -27,7 +27,7 @@ public record RoutingContext( Objects.requireNonNull(expiresAt, "expiresAt"); Objects.requireNonNull(now, "now"); if (routeCursor < 0) { - throw new IllegalArgumentException("routeCursor"); + throw new IllegalStateException("routeCursor must be >= 0, was " + routeCursor); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingDecision.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingDecision.java index 218f6cd4..bbee04ea 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingDecision.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingDecision.java @@ -16,10 +16,10 @@ public record RoutingDecision( Objects.requireNonNull(selected, "selected"); Objects.requireNonNull(reasonCode, "reasonCode"); if (reasonCode.isBlank()) { - throw new IllegalArgumentException("reasonCode"); + throw new IllegalStateException("a routing decision needs a reason code"); } if (reconciliationRequired && (retryAllowed || fallbackAllowed)) { - throw new IllegalArgumentException( + throw new IllegalStateException( "an attempt awaiting reconciliation may not also be retried or fallen back"); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/package-info.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/package-info.java new file mode 100644 index 00000000..a21e000c --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/package-info.java @@ -0,0 +1,16 @@ +/** + * The notification platform's delivery policy: who may be sent to (eligibility: consent, + * preference, suppression), what counts as the same request twice (deduplication), and which route + * to try and when to retry, reroute or stop. + * + *

The values the platform, or a policy it calls, assembles ({@link RetryTiming}, {@link + * RetryBackoff}, {@link RetryBudget}, {@link RetryContext}, {@link RetryDecision.RetryAfter}, + * {@link RouteCandidate}, {@link RoutingContext}, {@link RoutingDecision}, {@link + * EligibilityResult}) are built from settings, the shipped defaults, its own counters and stored + * delivery state: what they refuse never comes from a request. A refusal is a server fault, {@link + * IllegalStateException}, not the {@code IllegalArgumentException} the web layer would answer with + * a 400 (manual 06). The records that carry what a request may send ({@link ConsentRecord}, {@link + * PreferenceRecord}, {@link RecipientIdentity}, {@link SuppressionEntry}) refuse with {@code + * IllegalArgumentException}. + */ +package dev.caskeleton.application.notification.platform.policy; diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutorTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutorTest.java index 80b5960d..fec63e1a 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutorTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutorTest.java @@ -22,7 +22,7 @@ class AdminOperationExecutorTest { private final InMemoryAdminOperationStore store = new InMemoryAdminOperationStore(); @Test - @DisplayName("처음 claim한 operation만 claim을 쥔 채 한 쓰기 트랜잭션 안에서 부수 효과를 실행한다") + @DisplayName("처음 claim한 operation만 claim을 쥔 채 부수 효과를 실행하고, 트랜잭션은 열지 않는다") void claimedOperationIsTheOnlyOutcomeThatRunsTheSideEffect() { // given DirectTransactionPort transactions = new DirectTransactionPort(); @@ -31,7 +31,7 @@ class AdminOperationExecutorTest { // when AdminOperationResult result = - executor.execute( + executor.runClaimed( "op-1", FINGERPRINT, ACTOR, "ADMIN_PROVIDER_STATE", sideEffect(sideEffects)); // then @@ -41,12 +41,10 @@ class AdminOperationExecutorTest { sideEffects.get(), store.claim("op-1", FINGERPRINT, ACTOR, "ADMIN_PROVIDER_STATE"), transactions.committed())) - .as("the operation saves its own result; until then the id answers in-progress") - .containsExactly( - "op-1", - 1, - AdminOperationClaim.inProgress(), - List.of(DirectTransactionPort.Boundary.WRITE)); + .as( + "the operation saves its own result; until then the id answers in-progress. The" + + " use case owns the write transaction the claim runs in") + .containsExactly("op-1", 1, AdminOperationClaim.inProgress(), List.of()); } @Test @@ -62,7 +60,7 @@ class AdminOperationExecutorTest { // when AdminOperationResult replayed = - executor.execute( + executor.runClaimed( "op-1", FINGERPRINT, ACTOR, "ADMIN_PROVIDER_STATE", sideEffect(sideEffects)); // then @@ -82,7 +80,7 @@ class AdminOperationExecutorTest { Throwable thrown = catchThrowable( () -> - executor.execute( + executor.runClaimed( "op-1", FINGERPRINT, ACTOR, "ADMIN_REDRIVE", sideEffect(sideEffects))); // then @@ -103,7 +101,7 @@ class AdminOperationExecutorTest { Throwable thrown = catchThrowable( () -> - executor.execute( + executor.runClaimed( "op-1", FINGERPRINT, ACTOR, "ADMIN_REDRIVE", sideEffect(sideEffects))); // then diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprintTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprintTest.java similarity index 65% rename from src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprintTest.java rename to src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprintTest.java index d5f48645..1d33fa71 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprintTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprintTest.java @@ -7,17 +7,17 @@ import java.util.List; import java.util.UUID; import org.junit.jupiter.api.Test; -class AdminOperationCommandFingerprintTest { +class AdminOperationRequestFingerprintTest { - private final AdminOperationCommandFingerprint fingerprints = - new AdminOperationCommandFingerprint(); + private final AdminOperationRequestFingerprint fingerprints = + new AdminOperationRequestFingerprint(); @Test - void fingerprintIsStableSha256AndChangesWithCommandMeaning() { + void fingerprintIsStableSha256AndChangesWithRequestMeaning() { DeliveryAttemptId attempt = new DeliveryAttemptId(UUID.fromString("00000000-0000-0000-0000-1")); - RedriveCommand first = new RedriveCommand("op-1", attempt, "retry", true, false); - RedriveCommand same = new RedriveCommand("another-id", attempt, "retry", true, false); - RedriveCommand changed = new RedriveCommand("op-1", attempt, "different", true, false); + RedriveRequest first = new RedriveRequest("op-1", attempt, "retry", true, false); + RedriveRequest same = new RedriveRequest("another-id", attempt, "retry", true, false); + RedriveRequest changed = new RedriveRequest("op-1", attempt, "different", true, false); assertThat(fingerprints.of(first)).matches("[0-9a-f]{64}"); assertThat(fingerprints.of(same)).isEqualTo(fingerprints.of(first)); @@ -31,10 +31,10 @@ class AdminOperationCommandFingerprintTest { DeliveryAttemptId second = new DeliveryAttemptId(UUID.fromString("00000000-0000-0000-0000-000000000002")); - ReconcileCommand ordered = - new ReconcileCommand("op-1", List.of(first, second), "reconcile", false); - ReconcileCommand reversed = - new ReconcileCommand("op-2", List.of(second, first), "reconcile", false); + ReconcileRequest ordered = + new ReconcileRequest("op-1", List.of(first, second), "reconcile", false); + ReconcileRequest reversed = + new ReconcileRequest("op-2", List.of(second, first), "reconcile", false); assertThat(fingerprints.of(ordered)).isEqualTo(fingerprints.of(reversed)); } diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationServiceTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminUseCasesTest.java similarity index 56% rename from src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationServiceTest.java rename to src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminUseCasesTest.java index 2a7e002b..23681873 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationServiceTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminUseCasesTest.java @@ -27,9 +27,9 @@ import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemor import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryRecipientQueue; import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent; import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; -import dev.caskeleton.application.notification.platform.policy.SuppressionEntry; -import dev.caskeleton.application.notification.platform.policy.SuppressionId; -import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort; +import dev.caskeleton.application.notification.platform.policy.SuppressionReason; +import dev.caskeleton.application.notification.platform.policy.SuppressionScope; +import dev.caskeleton.application.notification.platform.policy.testkit.InMemorySuppressionStore; import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState; import dev.caskeleton.application.notification.platform.provider.ReconciliationResult; import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; @@ -42,10 +42,14 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; -import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -class NotificationAdminApplicationServiceTest { +/** + * The four operator-plane use cases, each its own entrypoint with its own permission (round9 review + * 2026-11-12, RR9-05-07: one service implemented all four behind an interface no use-case rule + * saw). + */ +class NotificationAdminUseCasesTest { private static final Clock CLOCK = Clock.fixed(Instant.parse("2026-09-20T00:00:00Z"), ZoneOffset.UTC); @@ -56,15 +60,16 @@ class NotificationAdminApplicationServiceTest { @Test void providerStateMutationClaimsBeforeCallingTheRuntime() { Fixture fixture = new Fixture(); - SetProviderStateCommand command = - new SetProviderStateCommand( + ProviderStateRequest request = + new ProviderStateRequest( "op-1", new ProviderProfileId("ses-primary"), ProviderRuntimeState.DISABLED, "provider maintenance", false); - AdminOperationResult result = fixture.service.setProviderState(command, ACTOR); + AdminOperationResult result = + fixture.providerState.handle(new ChangeProviderStateCommand(request, ACTOR)); assertThat(result.operationId()).isEqualTo("op-1"); assertThat(fixture.operations.claims).isEqualTo(1); @@ -77,18 +82,40 @@ class NotificationAdminApplicationServiceTest { .containsExactly("ADMIN_PROVIDER_STATE"); } + @Test + void providerStateClaimAndChangeCommitInTheUseCasesOneWriteTransaction() { + // given + Fixture fixture = new Fixture(); + ProviderStateRequest request = + new ProviderStateRequest( + "op-tx", + new ProviderProfileId("ses-primary"), + ProviderRuntimeState.DRAINING, + "provider maintenance", + false); + + // when + fixture.providerState.handle(new ChangeProviderStateCommand(request, ACTOR)); + + // then + assertThat(List.of(fixture.transactions.committed(), fixture.operations.claimsInTransaction)) + .as("a claim outside the transaction would commit while the change rolled back") + .containsExactly(List.of(DirectTransactionPort.Boundary.WRITE), List.of(true)); + } + @Test void providerStateDryRunDoesNotClaimOrMutateRuntime() { Fixture fixture = new Fixture(); - SetProviderStateCommand command = - new SetProviderStateCommand( + ProviderStateRequest request = + new ProviderStateRequest( "op-dry", new ProviderProfileId("ses-primary"), ProviderRuntimeState.DRAINING, "inspect only", true); - AdminOperationResult result = fixture.service.setProviderState(command, ACTOR); + AdminOperationResult result = + fixture.providerState.handle(new ChangeProviderStateCommand(request, ACTOR)); assertThat(result.dryRun()).isTrue(); assertThat(fixture.operations.claims).isZero(); @@ -97,7 +124,6 @@ class NotificationAdminApplicationServiceTest { } @Test - @DisplayName("tenant 범위 운영자는 다른 tenant의 attempt를 reconcile할 수 없다") void aTenantScopedOperatorCannotReconcileAnotherTenantsAttempt() { // given Fixture fixture = new Fixture(); @@ -109,11 +135,13 @@ class NotificationAdminApplicationServiceTest { "operator-b", Set.of(NotificationAdminAuthority.RECONCILE), Optional.of(new TenantId("tenant-b"))); - ReconcileCommand command = - new ReconcileCommand("op-r", List.of(foreign.attemptId()), "stuck attempt", false); + ReconcileRequest request = + new ReconcileRequest("op-r", List.of(foreign.attemptId()), "stuck attempt", false); // when - Throwable failure = catchThrowable(() -> fixture.service.reconcile(command, tenantB)); + Throwable failure = + catchThrowable( + () -> fixture.reconcile.handle(new ReconcileDeliveriesCommand(request, tenantB))); // then assertThat(failure) @@ -124,7 +152,6 @@ class NotificationAdminApplicationServiceTest { } @Test - @DisplayName("tenant 범위 운영자는 모든 tenant가 공유하는 provider 상태를 바꿀 수 없다") void aTenantScopedOperatorCannotChangeAGlobalProviderState() { // given Fixture fixture = new Fixture(); @@ -133,8 +160,8 @@ class NotificationAdminApplicationServiceTest { "operator-b", Set.of(NotificationAdminAuthority.PROVIDER_CONTROL), Optional.of(new TenantId("tenant-b"))); - SetProviderStateCommand command = - new SetProviderStateCommand( + ProviderStateRequest request = + new ProviderStateRequest( "op-p", new ProviderProfileId("ses-primary"), ProviderRuntimeState.DISABLED, @@ -142,7 +169,9 @@ class NotificationAdminApplicationServiceTest { false); // when - Throwable failure = catchThrowable(() -> fixture.service.setProviderState(command, tenantB)); + Throwable failure = + catchThrowable( + () -> fixture.providerState.handle(new ChangeProviderStateCommand(request, tenantB))); // then assertThat(failure) @@ -154,7 +183,6 @@ class NotificationAdminApplicationServiceTest { } @Test - @DisplayName("운영자 reconcile은 DB 트랜잭션을 연 채로 provider에 묻지 않는다") void reconcileAsksTheProviderWithNoTransactionOpen() { // given List transactionOpenDuringProviderCall = new ArrayList<>(); @@ -182,9 +210,10 @@ class NotificationAdminApplicationServiceTest { "operator-1", Set.of(NotificationAdminAuthority.RECONCILE), Optional.empty()); // when - fixture.service.reconcile( - new ReconcileCommand("op-rc", List.of(attempt.attemptId()), "stuck attempt", false), - operator); + fixture.reconcile.handle( + new ReconcileDeliveriesCommand( + new ReconcileRequest("op-rc", List.of(attempt.attemptId()), "stuck attempt", false), + operator)); // then assertThat(transactionOpenDuringProviderCall) @@ -192,6 +221,132 @@ class NotificationAdminApplicationServiceTest { .containsExactly(false); } + @Test + void redriveReturnsTheDeliveryToTheQueueUnderItsClaim() { + // given + Fixture fixture = new Fixture(); + DeliveryAttemptSnapshot attempt = attemptOf(new TenantId("tenant-a")); + AttemptSnapshots.store( + fixture.requests, fixture.attempts, attempt, RecipientDeliveryState.FAILED); + AdminActor operator = + new AdminActor("operator-1", Set.of(NotificationAdminAuthority.REDRIVE), Optional.empty()); + RedriveRequest request = + new RedriveRequest("op-rd", attempt.attemptId(), "provider outage", true, false); + + // when + fixture.redrive.handle(new RedriveDeliveryCommand(request, operator)); + + // then + assertThat( + List.of( + fixture.recipients.find(attempt.recipientDeliveryId()).orElseThrow().state(), + fixture.operations.claimsInTransaction, + fixture.audit.events.stream().map(NotificationAuditEvent::action).toList())) + .containsExactly( + RecipientDeliveryState.READY_TO_DISPATCH, List.of(true), List.of("ADMIN_REDRIVE")); + } + + @Test + void aRedriveReadsItsAttemptInsideTheWriteTransaction() { + // given + Fixture fixture = new Fixture(); + DeliveryAttemptSnapshot attempt = attemptOf(new TenantId("tenant-a")); + AttemptSnapshots.store( + fixture.requests, fixture.attempts, attempt, RecipientDeliveryState.FAILED); + List openWhileReading = new ArrayList<>(); + fixture.attempts.beforeSnapshot(id -> openWhileReading.add(fixture.transactions.active())); + AdminActor operator = + new AdminActor("operator-1", Set.of(NotificationAdminAuthority.REDRIVE), Optional.empty()); + + // when + fixture.redrive.handle( + new RedriveDeliveryCommand( + new RedriveRequest("op-rd-tx", attempt.attemptId(), "provider outage", true, false), + operator)); + + // then + assertThat(List.of(openWhileReading, fixture.transactions.committed())) + .as("the checks read the attempt the claim then redrives, in one transaction") + .containsExactly(List.of(true), List.of(DirectTransactionPort.Boundary.WRITE)); + } + + @Test + void aReconcileDryRunChecksItsTenantsInsideAReadTransaction() { + // given + Fixture fixture = new Fixture(); + DeliveryAttemptSnapshot attempt = attemptOf(new TenantId("tenant-a")); + AttemptSnapshots.store( + fixture.requests, fixture.attempts, attempt, RecipientDeliveryState.DISPATCHING); + List openWhileReading = new ArrayList<>(); + fixture.attempts.beforeSnapshot(id -> openWhileReading.add(fixture.transactions.active())); + AdminActor operator = + new AdminActor( + "operator-1", Set.of(NotificationAdminAuthority.RECONCILE), Optional.empty()); + + // when + fixture.reconcile.handle( + new ReconcileDeliveriesCommand( + new ReconcileRequest("op-rc-tx", List.of(attempt.attemptId()), "stuck attempt", true), + operator)); + + // then + assertThat(List.of(openWhileReading, fixture.transactions.committed())) + .containsExactly(List.of(true), List.of(DirectTransactionPort.Boundary.READ)); + } + + @Test + void aRedriveDryRunReadsInOneReadTransactionAndClaimsNothing() { + // given + Fixture fixture = new Fixture(); + DeliveryAttemptSnapshot attempt = attemptOf(new TenantId("tenant-a")); + AttemptSnapshots.store( + fixture.requests, fixture.attempts, attempt, RecipientDeliveryState.FAILED); + AdminActor operator = + new AdminActor("operator-1", Set.of(NotificationAdminAuthority.REDRIVE), Optional.empty()); + RedriveRequest request = + new RedriveRequest("op-rd-dry", attempt.attemptId(), "provider outage", true, true); + + // when + fixture.redrive.handle(new RedriveDeliveryCommand(request, operator)); + + // then + assertThat(List.of(fixture.transactions.committed(), fixture.operations.claimsInTransaction)) + .containsExactly(List.of(DirectTransactionPort.Boundary.READ), List.of()); + } + + @Test + void anOperatorSuppressionIsStoredUnderItsClaim() { + // given + Fixture fixture = new Fixture(); + TenantId tenant = new TenantId("tenant-a"); + AdminActor operator = + new AdminActor( + "operator-1", Set.of(NotificationAdminAuthority.SUPPRESS), Optional.of(tenant)); + SuppressionRequest request = + new SuppressionRequest( + "op-s", + tenant, + SuppressionScope.CONTACT_POINT, + SuppressionReason.ADMIN_BLOCK, + "c".repeat(64), + Optional.empty(), + false, + "reported abuse", + false); + + // when + AdminOperationResult result = + fixture.suppression.handle(new ChangeSuppressionCommand(request, operator)); + + // then + assertThat( + List.of( + result.affected(), + fixture.suppressions.activeFor(tenant, "c".repeat(64), CLOCK.instant()).size(), + fixture.operations.claimsInTransaction)) + .containsExactly(1, 1, List.of(true)); + } + private static DeliveryAttemptSnapshot attemptOf(TenantId tenantId) { return new DeliveryAttemptSnapshot( new DeliveryAttemptId(UUID.fromString("00000000-0000-7000-8000-000000000011")), @@ -225,7 +380,11 @@ class NotificationAdminApplicationServiceTest { private final RecordingOperations operations = new RecordingOperations(); private final RecordingAudit audit = new RecordingAudit(); private final DirectTransactionPort transactions = new DirectTransactionPort(); - private final NotificationAdminApplicationService service; + private final InMemorySuppressionStore suppressions = new InMemorySuppressionStore(); + private final ChangeProviderStateUseCase providerState; + private final ReconcileDeliveriesUseCase reconcile; + private final RedriveDeliveryUseCase redrive; + private final ChangeSuppressionUseCase suppression; private Fixture() { this(null); @@ -235,24 +394,42 @@ class NotificationAdminApplicationServiceTest { * @param gateway the provider the reconciliation asks, or null for one that supports nothing */ private Fixture(ReconciliationGatewayPort gateway) { + operations.transactions = transactions; ReconciliationService reconciliation = gateway == null ? PlatformCollaborators.reconciliation(attempts, recipients, CLOCK) : PlatformCollaborators.reconciliation(attempts, recipients, gateway, CLOCK); - service = - new NotificationAdminApplicationService( + providerState = + new ChangeProviderStateUseCase( + new AdminAuthorizationGuard(), runtime, operations, audit, transactions, CLOCK); + redrive = + new RedriveDeliveryUseCase( new AdminAuthorizationGuard(), new DuplicateRiskGuard(), attempts, recipients, - reconciliation, - new UnusedSuppressionStore(), - runtime, + operations, + audit, + transactions, + CLOCK); + suppression = + new ChangeSuppressionUseCase( + new AdminAuthorizationGuard(), + suppressions, operations, audit, transactions, new PlatformFakes.SequentialIds(), CLOCK); + reconcile = + new ReconcileDeliveriesUseCase( + new AdminAuthorizationGuard(), + attempts, + reconciliation, + operations, + audit, + transactions, + CLOCK); } } @@ -277,6 +454,8 @@ class NotificationAdminApplicationServiceTest { } private static final class RecordingOperations implements AdminOperationStorePort { + private final List claimsInTransaction = new ArrayList<>(); + private DirectTransactionPort transactions; private int claims; private int saved; @@ -293,9 +472,10 @@ class NotificationAdminApplicationServiceTest { @Override public AdminOperationClaim claim( - String operationId, String commandFingerprint, AdminActor actor, String action) { + String operationId, String requestFingerprint, AdminActor actor, String action) { claims++; - assertThat(commandFingerprint).matches("[0-9a-f]{64}"); + claimsInTransaction.add(transactions.active()); + assertThat(requestFingerprint).matches("[0-9a-f]{64}"); return AdminOperationClaim.claimed(); } } @@ -308,25 +488,4 @@ class NotificationAdminApplicationServiceTest { events.add(event); } } - - private static final class UnusedSuppressionStore implements SuppressionStorePort { - @Override - public List activeFor( - dev.caskeleton.application.notification.platform.api.TenantId tenantId, - String targetFingerprint, - Instant at) { - return List.of(); - } - - @Override - public SuppressionEntry upsert(SuppressionEntry entry) { - throw new UnsupportedOperationException(); - } - - @Override - public Optional remove( - dev.caskeleton.application.notification.platform.api.TenantId tenantId, SuppressionId id) { - return Optional.empty(); - } - } } diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCaseTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCaseTest.java new file mode 100644 index 00000000..9014a229 --- /dev/null +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCaseTest.java @@ -0,0 +1,90 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; +import dev.caskeleton.application.notification.platform.api.TenantId; +import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryNotificationRequestStore; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryRecipientQueue; +import dev.caskeleton.application.notification.platform.dispatch.testkit.NotificationRows; +import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The scheduler's claim, run through its {@code @SystemInvoked} use case. + * + *

The claim used to run with no transaction of the scheduler's own: the store's statement was + * its whole boundary. It now commits in the use case's write transaction. + */ +class ClaimDueDeliveriesUseCaseTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + private static final Duration LEASE = Duration.ofMinutes(2); + + private final InMemoryNotificationRequestStore requests = new InMemoryNotificationRequestStore(); + private final InMemoryRecipientQueue queue = + new InMemoryRecipientQueue(requests, Clock.fixed(NOW, ZoneOffset.UTC)); + private final DirectTransactionPort transactions = new DirectTransactionPort(); + private final ClaimDueDeliveriesUseCase useCase = + new ClaimDueDeliveriesUseCase(queue, transactions); + + @Test + @DisplayName("due 발송은 쓰기 트랜잭션 하나에서 이 worker의 lease로 claim된다") + void aDueDeliveryIsClaimedForTheWorkerInOneWriteTransaction() { + // given + RecipientDeliveryId due = dueJob(); + + // when + List claimed = + useCase.handle(new ClaimDueDeliveriesCommand("worker-1", 10, LEASE)); + + // then + assertThat( + List.of( + claimed.stream().map(RecipientLease::recipientDeliveryId).toList(), + claimed.stream().map(RecipientLease::owner).toList(), + claimed.stream().map(RecipientLease::leaseUntil).toList(), + transactions.committed())) + .containsExactly( + List.of(due), + List.of("worker-1"), + List.of(NOW.plus(LEASE)), + List.of(DirectTransactionPort.Boundary.WRITE)); + } + + @Test + @DisplayName("한 번에 0개를 claim하라는 command는 조립의 잘못으로 실패한다") + void aClaimOfNothingIsTheAssemblysFault() { + // when + Throwable refused = catchThrowable(() -> new ClaimDueDeliveriesCommand("worker-1", 0, LEASE)); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessage("limit must be positive"); + } + + private RecipientDeliveryId dueJob() { + var request = + NotificationRows.request(new TenantId("tenant-a"), "idem-" + UUID.randomUUID(), NOW); + var job = + NotificationRows.job( + request, + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(NOW.minusSeconds(1)), + Optional.empty(), + NOW); + requests.insert(request, List.of(job)); + return job.id(); + } +} diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCaseTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCaseTest.java new file mode 100644 index 00000000..47d87614 --- /dev/null +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCaseTest.java @@ -0,0 +1,78 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryContactPointStore; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryDeliveryAttemptStore; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryNotificationRequestStore; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryRecipientQueue; +import dev.caskeleton.application.notification.platform.policy.NotificationEligibilityPolicy; +import dev.caskeleton.application.notification.platform.policy.RetryDecision; +import dev.caskeleton.application.notification.platform.policy.RoutingDecisionEngine; +import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.UUID; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The scheduler's dispatch of one claimed delivery, run through its {@code @SystemInvoked} use + * case. The use case holds no boundary of its own; the dispatch service opens each one. + */ +class DispatchClaimedDeliveryUseCaseTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + private static final Clock CLOCK = Clock.fixed(NOW, ZoneOffset.UTC); + + private final InMemoryNotificationRequestStore requests = new InMemoryNotificationRequestStore(); + private final InMemoryRecipientQueue recipients = new InMemoryRecipientQueue(requests, CLOCK); + private final InMemoryDeliveryAttemptStore attempts = new InMemoryDeliveryAttemptStore(requests); + private final DirectTransactionPort transactions = new DirectTransactionPort(); + private final PlatformFakes.UnreachableGateway gateway = new PlatformFakes.UnreachableGateway(); + + private final DispatchClaimedDeliveryUseCase useCase = + new DispatchClaimedDeliveryUseCase( + new NotificationDispatchService( + requests, + recipients, + attempts, + new DispatchPipeline( + new InMemoryContactPointStore(CLOCK), + NotificationEligibilityPolicy.permitAll(), + new PlatformFakes.RecordingMetrics()), + new RoutingDecisionEngine(), + context -> new RetryDecision.Stop("unused"), + new DeliveryAttemptFactory(new PlatformFakes.SequentialIds()), + PlatformCollaborators.outcomeRecorder(attempts, recipients), + gateway, + command -> { + throw new AssertionError("a lease whose delivery is gone renders nothing"); + }, + recipients, + transactions, + CLOCK, + 3, + Duration.ofSeconds(5))); + + @Test + @DisplayName("delivery가 사라진 lease는 dispatch 서비스의 읽기·쓰기 경계에서 풀리고 provider에 닿지 않는다") + void aLeaseWhoseDeliveryIsGoneIsReleasedWithoutReachingTheProvider() { + // given + RecipientLease gone = + new RecipientLease( + new RecipientDeliveryId(UUID.randomUUID()), "worker-a", 1L, NOW.plusSeconds(30)); + + // when + useCase.handle(new DispatchClaimedDeliveryCommand(gone)); + + // then + assertThat(List.of(transactions.committed(), gateway.calls())) + .containsExactly( + List.of(DirectTransactionPort.Boundary.READ, DirectTransactionPort.Boundary.WRITE), 0); + } +} diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/NotificationDispatchServiceTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/NotificationDispatchServiceTest.java index e4e13f18..b5ab61a3 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/NotificationDispatchServiceTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/NotificationDispatchServiceTest.java @@ -32,8 +32,8 @@ import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemor import dev.caskeleton.application.notification.platform.policy.DefaultNotificationRetryPolicy; import dev.caskeleton.application.notification.platform.policy.JitterSource; import dev.caskeleton.application.notification.platform.policy.NotificationEligibilityPolicy; -import dev.caskeleton.application.notification.platform.policy.RetryBackoff; import dev.caskeleton.application.notification.platform.policy.RetryDecision; +import dev.caskeleton.application.notification.platform.policy.RetryTiming; import dev.caskeleton.application.notification.platform.policy.RoutingDecisionEngine; import dev.caskeleton.application.notification.platform.port.in.SubmitNotificationCommand; import dev.caskeleton.application.notification.platform.provider.ProviderCapabilities; @@ -70,7 +70,7 @@ class NotificationDispatchServiceTest { new InMemoryRecipientQueue(requests, Clock.fixed(NOW, ZoneOffset.UTC)); InMemoryDeliveryAttemptStore attempts = new InMemoryDeliveryAttemptStore(requests); DirectTransactionPort transactions = new DirectTransactionPort(); - FailingGateway gateway = new FailingGateway(); + PlatformFakes.UnreachableGateway gateway = new PlatformFakes.UnreachableGateway(); NotificationDispatchService service = new NotificationDispatchService( requests, @@ -104,7 +104,7 @@ class NotificationDispatchServiceTest { // The WRITE boundary after the read is the release; the queue has no row to show it on. assertThat(transactions.committed()).containsExactly(Boundary.READ, Boundary.WRITE); - assertThat(gateway.calls).isZero(); + assertThat(gateway.calls()).isZero(); } @Test @@ -500,8 +500,7 @@ class NotificationDispatchServiceTest { NotificationEligibilityPolicy.permitAll(), new PlatformFakes.RecordingMetrics()), new RoutingDecisionEngine(), - new DefaultNotificationRetryPolicy( - RetryBackoff.DEFAULT, JitterSource.none(), Duration.ofMinutes(1)), + new DefaultNotificationRetryPolicy(RetryTiming.DEFAULT, JitterSource.none()), new DeliveryAttemptFactory(new PlatformFakes.SequentialIds()), PlatformCollaborators.outcomeRecorder(attempts, recipients), gateway, @@ -631,26 +630,4 @@ class NotificationDispatchServiceTest { throw new AssertionError("nothing may be submitted through a profile that is gone"); } } - - private static final class FailingGateway implements ProviderDispatchGatewayPort { - private int calls; - - @Override - public ProviderProfileSnapshot profile(ProviderProfileId profileId) { - calls++; - throw new AssertionError("gateway profile must not be reached"); - } - - @Override - public ProviderRuntimeState state(ProviderProfileId profileId) { - calls++; - throw new AssertionError("gateway state must not be reached"); - } - - @Override - public ProviderSubmissionResult submit(ProviderSubmission submission) { - calls++; - throw new AssertionError("provider submit must not be reached"); - } - } } diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/PlatformFakes.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/PlatformFakes.java index a04d1631..7fc4750f 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/PlatformFakes.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/PlatformFakes.java @@ -14,6 +14,10 @@ import dev.caskeleton.application.notification.platform.api.routing.DeliveryStra import dev.caskeleton.application.notification.platform.api.routing.ExplicitChannel; import dev.caskeleton.application.notification.platform.observation.NotificationMetricsPort; import dev.caskeleton.application.notification.platform.policy.RouteCandidate; +import dev.caskeleton.application.notification.platform.provider.ProviderProfileSnapshot; +import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState; +import dev.caskeleton.application.notification.platform.provider.ProviderSubmission; +import dev.caskeleton.application.notification.platform.provider.ProviderSubmissionResult; import java.time.Duration; import java.util.ArrayList; import java.util.List; @@ -35,6 +39,35 @@ public final class PlatformFakes { private PlatformFakes() {} + /** A provider gateway the dispatch under test must not reach; it counts what reached it. */ + public static final class UnreachableGateway implements ProviderDispatchGatewayPort { + + private int calls; + + /** How many calls reached the gateway. */ + public int calls() { + return calls; + } + + @Override + public ProviderProfileSnapshot profile(ProviderProfileId profileId) { + calls++; + throw new AssertionError("gateway profile must not be reached"); + } + + @Override + public ProviderRuntimeState state(ProviderProfileId profileId) { + calls++; + throw new AssertionError("gateway state must not be reached"); + } + + @Override + public ProviderSubmissionResult submit(ProviderSubmission submission) { + calls++; + throw new AssertionError("provider submit must not be reached"); + } + } + /** Metrics sink that keeps what it was told, so tag discipline stays assertable. */ public static final class RecordingMetrics implements NotificationMetricsPort { diff --git a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorkerTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCaseTest.java similarity index 50% rename from src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorkerTest.java rename to src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCaseTest.java index ad25e7bd..2cbe1abb 100644 --- a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorkerTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCaseTest.java @@ -1,4 +1,4 @@ -package dev.caskeleton.adapter.outbound.notification.platform.dispatch; +package dev.caskeleton.application.notification.platform.dispatch; import static org.assertj.core.api.Assertions.assertThat; @@ -6,14 +6,14 @@ import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; import dev.caskeleton.application.notification.platform.api.NormalizedEventType; import dev.caskeleton.application.notification.platform.api.NormalizedProviderEvent; import dev.caskeleton.application.notification.platform.api.ProviderProfileId; -import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJob; -import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJobStorePort; import dev.caskeleton.application.notification.platform.provider.ReconciliationResult; +import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -29,117 +29,190 @@ import org.junit.jupiter.api.Test; * in {@code RECONCILIATION_REQUIRED} permanently. The job table for exactly this has existed since * V3 with no entity, no store and no worker. */ -class ReconciliationJobWorkerTest { +class ReconcileOutstandingAttemptsUseCaseTest { private static final Instant NOW = Instant.parse("2026-08-15T09:00:00Z"); private static final Clock CLOCK = Clock.fixed(NOW, ZoneOffset.UTC); - private static final Duration CLAIM_LEASE = Duration.ofMinutes(2); + private static final Duration DELIVERY_LEASE = Duration.ofMinutes(2); + private static final int MAX_ADDITIONAL_ATTEMPTS = 2; private final RecordingJobs jobs = new RecordingJobs(); - private final Map answers = new java.util.HashMap<>(); + private final Map answers = new HashMap<>(); + private final List asked = new ArrayList<>(); + private final DirectTransactionPort transactions = new DirectTransactionPort(); - private final ReconciliationJobWorker worker = - new ReconciliationJobWorker( - jobs, answers::get, CLOCK, Duration.ofMinutes(5), CLAIM_LEASE, 10, 3); + private final ReconcileOutstandingAttemptsUseCase useCase = + new ReconcileOutstandingAttemptsUseCase( + jobs, + attemptId -> { + asked.add(attemptId); + return answers.get(attemptId); + }, + transactions, + CLOCK, + DELIVERY_LEASE, + 10, + MAX_ADDITIONAL_ATTEMPTS); @Test - @DisplayName("a confirmed outcome completes the job") void aConfirmedOutcomeCompletesTheJob() { + // given ReconciliationJob job = due(0); answers.put(job.attemptId(), new ReconciliationResult.Confirmed(event())); - assertThat(worker.reconcileOnce()).isEqualTo(1); - assertThat(jobs.completed).containsExactly(job.id()); + // when + int settled = useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(List.of(settled, jobs.completed)).containsExactly(1, List.of(job.id())); } @Test - @DisplayName("an unsupported provider is not asked in a loop") - void anUnsupportedProviderIsNotAskedInALoop() { + @DisplayName("상태 조회가 없는 provider는 다시 묻지 않고 job을 끝낸다. 답이 바뀌지 않는다") + void anUnsupportedProviderIsCompletedRatherThanAskedAgain() { + // given ReconciliationJob job = due(0); answers.put(job.attemptId(), new ReconciliationResult.Unsupported()); - assertThat(worker.reconcileOnce()).isEqualTo(1); - assertThat(jobs.completed) - .as( - "without a status-query capability the answer will not change; the attempt stays" - + " visibly ambiguous for an operator instead") - .containsExactly(job.id()); - assertThat(jobs.rescheduled).isEmpty(); + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(List.of(jobs.completed, jobs.rescheduled)) + .as("the attempt stays visibly ambiguous for an operator instead") + .containsExactly(List.of(job.id()), List.of()); } @Test - @DisplayName("a still-unknown outcome is asked again at the provider's own next time") - void aStillUnknownOutcomeIsAskedAgain() { + void aStillUnknownOutcomeIsAskedAgainAtTheProvidersOwnNextTime() { + // given ReconciliationJob job = due(0); Instant providerSuggested = NOW.plus(Duration.ofHours(2)); answers.put(job.attemptId(), new ReconciliationResult.StillUnknown(providerSuggested)); - assertThat(worker.reconcileOnce()).isZero(); - assertThat(jobs.rescheduled).containsExactly(Map.entry(job.id(), providerSuggested)); - assertThat(jobs.completed).isEmpty(); + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(List.of(jobs.rescheduled, jobs.completed)) + .containsExactly(List.of(Map.entry(job.id(), providerSuggested)), List.of()); } @Test - @DisplayName("a retryable failure is asked again after the backoff") - void aRetryableFailureIsAskedAgain() { + void aRetryableFailureIsAskedAgainOneDeliveryLeaseLater() { + // given ReconciliationJob job = due(0); answers.put(job.attemptId(), new ReconciliationResult.Failed("PROVIDER_TIMEOUT", true)); - assertThat(worker.reconcileOnce()).isZero(); - assertThat(jobs.rescheduled) - .containsExactly(Map.entry(job.id(), NOW.plus(Duration.ofMinutes(5)))); + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(jobs.rescheduled).containsExactly(Map.entry(job.id(), NOW.plus(DELIVERY_LEASE))); } @Test - @DisplayName("a non-retryable failure completes the job") void aNonRetryableFailureCompletesTheJob() { + // given ReconciliationJob job = due(0); answers.put(job.attemptId(), new ReconciliationResult.Failed("ATTEMPT_UNKNOWN", false)); - assertThat(worker.reconcileOnce()).isEqualTo(1); + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then assertThat(jobs.completed).containsExactly(job.id()); } @Test - @DisplayName("a job that has asked enough times stops asking") - void aJobThatHasAskedEnoughTimesStopsAsking() { - ReconciliationJob job = due(3); + @DisplayName("첫 질문과 추가 시도 2번, 모두 3번을 물은 job은 더 묻지 않고 끝난다") + void aJobThatAskedOnceMoreThanItsAdditionalAttemptsStopsAsking() { + // given + ReconciliationJob job = due(MAX_ADDITIONAL_ATTEMPTS + 1); - assertThat(worker.reconcileOnce()).isEqualTo(1); - assertThat(jobs.completed) + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(List.of(asked, jobs.completed)) .as("an outcome that has not arrived after this many tries is an operator's decision") - .containsExactly(job.id()); - assertThat(answers).isEmpty(); + .containsExactly(List.of(), List.of(job.id())); } @Test - @DisplayName("one job throwing does not stop the rest of the pass") - void oneJobThrowingDoesNotStopTheRest() { + @DisplayName("2번 물은 job은 추가 시도가 남아 한 번 더 묻는다") + void aJobWithAnAdditionalAttemptLeftIsAskedAgain() { + // given + ReconciliationJob job = due(MAX_ADDITIONAL_ATTEMPTS); + answers.put(job.attemptId(), new ReconciliationResult.Unsupported()); + + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(asked).containsExactly(job.attemptId()); + } + + @Test + void oneJobThrowingDoesNotStopTheRestOfThePass() { + // given + // The failing job has no answer registered, so the reconciler returns null and the switch + // throws — which is exactly the shape of a provider refusing connections. ReconciliationJob failing = due(0); ReconciliationJob healthy = due(0); answers.put(healthy.attemptId(), new ReconciliationResult.Confirmed(event())); - // The failing job has no answer registered, so the reconciler returns null and the switch - // throws — which is exactly the shape of a provider refusing connections. - assertThat(worker.reconcileOnce()).isEqualTo(1); - assertThat(jobs.completed).containsExactly(healthy.id()); - assertThat(jobs.rescheduled).hasSize(1); - assertThat(jobs.rescheduled.get(0).getKey()).isEqualTo(failing.id()); + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(List.of(jobs.completed, jobs.rescheduled.stream().map(Map.Entry::getKey).toList())) + .containsExactly(List.of(healthy.id()), List.of(failing.id())); } @Test - @DisplayName("한 번의 pass는 설정된 lease 동안 소유하도록 job을 claim한다") - void aPassClaimsItsJobsForTheConfiguredLease() { + @DisplayName("한 번의 pass는 delivery lease 동안 소유하도록 job을 claim한다") + void aPassClaimsItsJobsForOneDeliveryLease() { // given due(0); // when - worker.reconcileOnce(); + useCase.handle(new ReconcileOutstandingAttemptsCommand()); // then assertThat(jobs.claimLeases) .as("without a lease the claim ends with the statement and every instance takes the job") - .containsExactly(CLAIM_LEASE); + .containsExactly(DELIVERY_LEASE); + } + + @Test + @DisplayName("provider에는 트랜잭션 없이 묻고, job 기록은 그 앞뒤의 쓰기 트랜잭션에서 한다") + void theProviderIsAskedWithNoTransactionOpenAndTheJobIsRecordedInOne() { + // given + due(0); + List openWhileAsking = new ArrayList<>(); + ReconcileOutstandingAttemptsUseCase observing = + new ReconcileOutstandingAttemptsUseCase( + jobs, + attemptId -> { + openWhileAsking.add(transactions.active()); + return new ReconciliationResult.Unsupported(); + }, + transactions, + CLOCK, + DELIVERY_LEASE, + 10, + MAX_ADDITIONAL_ATTEMPTS); + + // when + observing.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(List.of(openWhileAsking, transactions.committed())) + .as("the claim and the completion each commit; the provider call holds no connection") + .containsExactly( + List.of(false), + List.of(DirectTransactionPort.Boundary.WRITE, DirectTransactionPort.Boundary.WRITE)); } private ReconciliationJob due(int attempts) { diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCaseTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCaseTest.java new file mode 100644 index 00000000..0065ffec --- /dev/null +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCaseTest.java @@ -0,0 +1,77 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; +import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; +import dev.caskeleton.application.notification.platform.api.TenantId; +import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState; +import dev.caskeleton.application.notification.platform.dispatch.testkit.ContractClock; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryDeliveryAttemptStore; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryNotificationRequestStore; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryRecipientQueue; +import dev.caskeleton.application.notification.platform.dispatch.testkit.NotificationRows; +import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** The background workers' lease recovery, run through its {@code @SystemInvoked} use case. */ +class RecoverAbandonedDeliveriesUseCaseTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + + private final InMemoryNotificationRequestStore requests = new InMemoryNotificationRequestStore(); + private final ContractClock queueClock = new ContractClock(NOW); + private final InMemoryRecipientQueue queue = new InMemoryRecipientQueue(requests, queueClock); + private final List reconciled = new ArrayList<>(); + + private final RecoverAbandonedDeliveriesUseCase useCase = + new RecoverAbandonedDeliveriesUseCase( + new LeaseRecoveryService( + queue, + new InMemoryDeliveryAttemptStore(requests), + queue, + reconciled::add, + new DirectTransactionPort(), + Clock.fixed(NOW, ZoneOffset.UTC), + Duration.ofMinutes(1), + 10)); + + @Test + @DisplayName("worker가 attempt를 남기기 전에 죽은 delivery는 한 pass에서 다시 보낼 차례로 돌아간다") + void aDeliveryAbandonedBeforeItsAttemptIsRequeuedInOnePass() { + // given + RecipientDeliveryId abandoned = abandonedJob(); + queueClock.moveTo(NOW.plus(Duration.ofMinutes(5))); + + // when + int recovered = useCase.handle(new RecoverAbandonedDeliveriesCommand()); + + // then + assertThat(List.of(recovered, queue.find(abandoned).orElseThrow().state(), reconciled)) + .containsExactly(1, RecipientDeliveryState.READY_TO_DISPATCH, List.of()); + } + + private RecipientDeliveryId abandonedJob() { + var request = + NotificationRows.request(new TenantId("tenant-a"), "idem-" + UUID.randomUUID(), NOW); + var job = + NotificationRows.job( + request, + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(NOW), + Optional.empty(), + NOW); + requests.insert(request, List.of(job)); + queue.claim("worker-a", 10, Duration.ofSeconds(30)); + return job.id(); + } +} diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/NotificationRetryPolicyTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/NotificationRetryPolicyTest.java index d757f954..d774edc0 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/NotificationRetryPolicyTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/NotificationRetryPolicyTest.java @@ -17,7 +17,7 @@ class NotificationRetryPolicyTest { private static final Instant NOW = Instant.parse("2026-08-10T00:00:00Z"); private final NotificationRetryPolicy policy = - new DefaultNotificationRetryPolicy(RetryBackoff.DEFAULT, bound -> 0L, Duration.ofMinutes(1)); + new DefaultNotificationRetryPolicy(RetryTiming.DEFAULT, bound -> 0L); @Test void authenticationFailureStopsInsteadOfRetryingEveryMessage() { diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/PolicyValuesTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/PolicyValuesTest.java new file mode 100644 index 00000000..dffa456b --- /dev/null +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/PolicyValuesTest.java @@ -0,0 +1,137 @@ +package dev.caskeleton.application.notification.platform.policy; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; +import dev.caskeleton.application.notification.platform.api.delivery.AttemptConfirmation; +import dev.caskeleton.application.notification.platform.api.delivery.SubmissionOutcome; +import dev.caskeleton.application.notification.platform.api.error.FailureCategory; +import dev.caskeleton.application.notification.platform.api.routing.Channel; +import dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence; +import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * The retry and routing values refuse what no settings or stored state could hold, as the package + * says. + */ +class PolicyValuesTest { + + static Stream refusedValues() { + return Stream.of( + Arguments.of( + "backoff 하한 0", + (ThrowingCallable) + () -> + new RetryTiming( + new RetryBackoff(Duration.ZERO, Duration.ofMinutes(5)), + Duration.ofMinutes(1)), + "the backoff base must be positive, was PT0S"), + Arguments.of( + "backoff 상한이 하한보다 짧음", + (ThrowingCallable) + () -> + new RetryTiming( + new RetryBackoff(Duration.ofMinutes(1), Duration.ofSeconds(30)), + Duration.ofMinutes(1)), + "the backoff max (PT30S) must not be shorter than its base (PT1M)"), + Arguments.of( + "음수 reconcile 지연", + (ThrowingCallable) () -> new RetryTiming(RetryBackoff.DEFAULT, Duration.ofSeconds(-1)), + "the reconcile delay must not be negative, was PT-1S"), + Arguments.of( + "attempt 0의 delay", + (ThrowingCallable) + () -> RetryBackoff.DEFAULT.delay(0, Optional.empty(), JitterSource.none()), + "attemptNumber must be >= 1, was 0"), + Arguments.of( + "attempt 0의 retry context", + (ThrowingCallable) () -> contextForAttempt(0), + "attemptNumber must be >= 1, was 0"), + Arguments.of( + "소비가 음수인 budget", + (ThrowingCallable) () -> new RetryBudget(2, -1), + "budget values must not be negative, were 2 and -1"), + Arguments.of( + "음수 delay의 재시도 결정", + (ThrowingCallable) () -> new RetryDecision.RetryAfter(Duration.ofSeconds(-1)), + "delay must not be negative, was PT-1S"), + Arguments.of( + "음수 위치의 route 후보", + (ThrowingCallable) () -> PolicyFixtures.route(-1, Channel.PUSH, true), + "routeIndex must be >= 0, was -1"), + Arguments.of( + "음수 cursor의 routing context", + (ThrowingCallable) + () -> + new RoutingContext( + new RecipientDeliveryId(UUID.randomUUID()), + PolicyFixtures.pushThenSms(), + -1, + SubmissionOutcome.NOT_SUBMITTED, + Optional.empty(), + false, + Optional.empty(), + Instant.parse("2026-11-13T09:00:00Z")), + "routeCursor must be >= 0, was -1"), + Arguments.of( + "사유 코드 없는 routing 결정", + (ThrowingCallable) () -> RoutingDecision.reconcile(" ", false), + "a routing decision needs a reason code"), + Arguments.of( + "맞추기와 재시도를 함께 고른 routing 결정", + (ThrowingCallable) + () -> new RoutingDecision(Optional.empty(), true, false, true, false, "BOTH"), + "an attempt awaiting reconciliation may not also be retried or fallen back"), + Arguments.of( + "맞추기와 fallback을 함께 고른 routing 결정", + (ThrowingCallable) + () -> new RoutingDecision(Optional.empty(), false, true, true, false, "BOTH"), + "an attempt awaiting reconciliation may not also be retried or fallen back"), + Arguments.of( + "사유 없는 수신 자격 차단", + (ThrowingCallable) () -> EligibilityResult.blocked(), + "a block must carry at least one reason code")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("refusedValues") + void aValueNoSettingsOrStoredStateCouldHoldIsAServerFault( + String name, ThrowingCallable build, String message) { + // when + Throwable refused = catchThrowable(build); + + // then + assertThat(refused).isInstanceOf(IllegalStateException.class).hasMessage(message); + } + + private static RetryContext contextForAttempt(int attemptNumber) { + Instant now = Instant.parse("2026-11-13T09:00:00Z"); + return new RetryContext( + FailureCategory.TRANSIENT_PROVIDER, + AttemptConfirmation.REJECTED, + SubmissionOutcome.NOT_SUBMITTED, + ProviderExecutionEvidence.notStarted(), + false, + false, + true, + false, + false, + Optional.of(now.plus(Duration.ofHours(1))), + Optional.empty(), + attemptNumber, + new RetryBudget(2, 0), + ProviderRuntimeState.HEALTHY, + Duration.ofSeconds(1), + now); + } +} diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/admin/testkit/InMemoryAdminOperationStore.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/admin/testkit/InMemoryAdminOperationStore.java index 8c222dcf..a2a6e80d 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/admin/testkit/InMemoryAdminOperationStore.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/admin/testkit/InMemoryAdminOperationStore.java @@ -49,17 +49,17 @@ public final class InMemoryAdminOperationStore implements AdminOperationStorePor @Override public synchronized AdminOperationClaim claim( - String operationId, String commandFingerprint, AdminActor actor, String action) { + String operationId, String requestFingerprint, AdminActor actor, String action) { Objects.requireNonNull(operationId, "operationId"); - Objects.requireNonNull(commandFingerprint, "commandFingerprint"); + Objects.requireNonNull(requestFingerprint, "requestFingerprint"); Objects.requireNonNull(actor, "actor"); Objects.requireNonNull(action, "action"); Row existing = rows.get(operationId); if (existing == null) { - rows.put(operationId, new Row(commandFingerprint, actor.actorRef(), action)); + rows.put(operationId, new Row(requestFingerprint, actor.actorRef(), action)); return AdminOperationClaim.claimed(); } - if (!existing.fingerprint.equals(commandFingerprint)) { + if (!existing.fingerprint.equals(requestFingerprint)) { return AdminOperationClaim.conflict(); } return existing diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java index 65bc4d4d..748f00fb 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java @@ -50,6 +50,8 @@ public final class InMemoryDeliveryAttemptStore private volatile Consumer beforeAttemptsOf = id -> {}; + private volatile Consumer beforeSnapshot = id -> {}; + private final Map requestIdHashes = new LinkedHashMap<>(); private final Map engagement = new LinkedHashMap<>(); private final Map suppression = new LinkedHashMap<>(); @@ -169,9 +171,20 @@ public final class InMemoryDeliveryAttemptStore @Override public synchronized Optional snapshot(DeliveryAttemptId id) { + beforeSnapshot.accept(id); return find(id).flatMap(this::toSnapshot); } + /** + * Runs {@code hook} before every {@link #snapshot} read, for a test that needs to see what was + * open when a caller read an attempt. + * + * @param hook what to run, given the attempt being read + */ + public void beforeSnapshot(Consumer hook) { + beforeSnapshot = Objects.requireNonNull(hook, "hook"); + } + @Override public synchronized int nextAttemptNo(RecipientDeliveryId recipientDeliveryId) { return attemptsFor(recipientDeliveryId).stream() diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/AdapterBoundaryArchitectureTest.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/AdapterBoundaryArchitectureTest.java index 17516da9..613d3910 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/AdapterBoundaryArchitectureTest.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/AdapterBoundaryArchitectureTest.java @@ -16,6 +16,7 @@ import com.tngtech.archunit.lang.ArchCondition; import com.tngtech.archunit.lang.ArchRule; import com.tngtech.archunit.lang.ConditionEvents; import com.tngtech.archunit.lang.SimpleConditionEvent; +import dev.caskeleton.application.usecase.UseCase; import java.util.Map; import java.util.Set; @@ -36,6 +37,27 @@ import java.util.Set; @AnalyzeClasses(packages = "dev.caskeleton", importOptions = ProductionClassImportOption.class) class AdapterBoundaryArchitectureTest { + /** + * The application types behind a use case: services, orchestrators and coordinators that are not + * use cases themselves, interfaces included. Interfaces were left out until 2026-11-13, so a + * controller injecting the fileserver's {@code CleanupService} — the service the cleanup worker's + * {@code @SystemInvoked} use case wraps — passed (round9 review 2026-11-12, application + * structure). + */ + static final DescribedPredicate ARE_APPLICATION_SERVICES = + JavaClass.Predicates.resideInAPackage("..application..") + .and(DescribedPredicate.not(JavaClass.Predicates.resideInAPackage("..adapter.."))) + .and( + DescribedPredicate.describe( + "named *Service, *Orchestrator or *Coordinator and not a use case", + type -> + type.getName().startsWith("dev.caskeleton.") + && !type.isAssignableTo(UseCase.class) + && (type.getSimpleName().endsWith("Service") + || type.getSimpleName().endsWith("Orchestrator") + || type.getSimpleName().endsWith("Coordinator")))) + .as("application services, orchestrators and coordinators that are not use cases"); + /** * A transport must depend on an inbound port, never on the class behind it. * @@ -50,26 +72,15 @@ class AdapterBoundaryArchitectureTest { static final ArchRule INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES = noClasses() .that() - .resideInAPackage("dev.caskeleton.adapter.inbound..") + .resideInAPackage("..adapter.inbound..") .should() - .dependOnClassesThat( - new com.tngtech.archunit.base.DescribedPredicate<>( - "are concrete application orchestration services") { - @Override - public boolean test(com.tngtech.archunit.core.domain.JavaClass candidate) { - return candidate.getPackageName().startsWith("dev.caskeleton.application.") - && !candidate.isInterface() - && !candidate - .getModifiers() - .contains(com.tngtech.archunit.core.domain.JavaModifier.ABSTRACT) - && (candidate.getSimpleName().endsWith("Service") - || candidate.getSimpleName().endsWith("Orchestrator")); - } - }) + .dependOnClassesThat(ARE_APPLICATION_SERVICES) .because( "INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES: an entrypoint reached " - + "through a concrete class bypasses the use-case fitness gate, which is what " - + "declares transaction mode, repository access and outbound permission") + + "through a service, orchestrator or coordinator — class or interface — bypasses " + + "the use-case fitness gate, which is what declares transaction mode, repository " + + "access, outbound permission and the required permission. A service a " + + "@SystemInvoked use case wraps writes with no permission check at all") .allowEmptyShould(true); /** @@ -451,6 +462,57 @@ class AdapterBoundaryArchitectureTest { + " or a query port (AGENTS.md HARD-STOP 2)") .allowEmptyShould(true); + /** + * The server-assigned id ports: minting an id is part of a use case, so the transport that hands + * a submission to one does not mint the id the use case will store. {@code *IdFactory} is the + * name new code uses; {@code *IdGeneratorPort} and {@code *IdentifierPort} are the names the + * notification and fileserver platforms already had (application-core AGENTS 이름 규칙). + */ + static final DescribedPredicate ARE_ID_FACTORIES = + JavaClass.Predicates.resideInAnyPackage("..domain..", "..application..") + .and( + DescribedPredicate.describe( + "named *IdFactory, *IdGeneratorPort or *IdentifierPort", + type -> + type.getName().startsWith("dev.caskeleton.") + && (type.getSimpleName().endsWith("IdFactory") + || type.getSimpleName().endsWith("IdGeneratorPort") + || type.getSimpleName().endsWith("IdentifierPort")))) + .as("id factories"); + + /** + * The packages a user request comes in through: every inbound package but the system-driven ones + * ({@code ApplicationBoundaryArchitectureTest.SYSTEM_DRIVEN_INBOUND_PACKAGES}), so a transport + * added later is covered without being listed here. + */ + static final DescribedPredicate ARE_IN_USER_TRANSPORT_PACKAGES = + JavaClass.Predicates.resideInAPackage("..adapter.inbound..") + .and( + DescribedPredicate.not( + JavaClass.Predicates.resideInAnyPackage( + ApplicationBoundaryArchitectureTest.SYSTEM_DRIVEN_INBOUND_PACKAGES))) + .as("classes in a user-transport package (an inbound package that is not system-driven)"); + + /** + * The rule above for every class of a user transport, not only its endpoints: a stream bridge, a + * data fetcher helper or a filter that holds a store, query or id port reaches persistence with + * no use case, no capability and no read boundary in between. A web streaming bridge read the + * live event log this way, and the durable operation controller minted operation ids (round9 + * review 2026-11-12, RR9-05-10, RR9-05-13). + */ + @ArchTest + static final ArchRule USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS = + noClasses() + .that(ARE_IN_USER_TRANSPORT_PACKAGES) + .should() + .dependOnClassesThat(ARE_PERSISTENCE_PORTS.or(ARE_ID_FACTORIES)) + .as( + "USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS: a user transport reads and" + + " writes through a use case; it holds no domain repository port, application" + + " store or query port, and no id factory or id generator port (AGENTS.md HARD-STOP 2," + + " application-core AGENTS 「읽기·query 경로」)") + .allowEmptyShould(true); + /** * A controller is an inbound adapter, so it lives in one. One filed in the composition root or a * composition module is reached by no transport rule that names inbound packages (app {@code diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ApplicationBoundaryArchitectureTest.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ApplicationBoundaryArchitectureTest.java index 2bd3a769..04037b92 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ApplicationBoundaryArchitectureTest.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ApplicationBoundaryArchitectureTest.java @@ -454,7 +454,7 @@ class ApplicationBoundaryArchitectureTest { * inbound package — web, GraphQL, gRPC, WebSocket, and any transport added later — is a way a * user request comes in. A new system-driven inbound family joins this list, nowhere else. */ - private static final String[] SYSTEM_DRIVEN_INBOUND_PACKAGES = { + static final String[] SYSTEM_DRIVEN_INBOUND_PACKAGES = { "..adapter.inbound.messaging..", "..adapter.inbound.scheduler.." }; diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchRuleDiscoveryContractTest.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchRuleDiscoveryContractTest.java index 3c5fc54f..e321b0e3 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchRuleDiscoveryContractTest.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchRuleDiscoveryContractTest.java @@ -100,6 +100,7 @@ class ArchRuleDiscoveryContractTest { "CONTROLLER_REQUEST_MAPPINGS_FOLLOW_AIP122", "DOMAIN_CONTEXT_PROPAGATION_PRIMITIVES_STAY_UNSHIPPED", "ENDPOINTS_DO_NOT_HOLD_PERSISTENCE_PORTS", + "USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS", "DOMAIN_ENTITIES_DO_NOT_CARRY_AUDIT_FIELDS", "DOMAIN_EVENTS_ARE_RECORDS", "DOMAIN_EVENTS_ARE_TRANSPORT_FREE", diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchitectureViolationFixtureTest.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchitectureViolationFixtureTest.java index de48d4ea..262030d3 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchitectureViolationFixtureTest.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchitectureViolationFixtureTest.java @@ -7,13 +7,17 @@ import com.tngtech.archunit.core.domain.JavaClasses; import com.tngtech.archunit.core.importer.ClassFileImporter; import com.tngtech.archunit.lang.EvaluationResult; import dev.caskeleton.application.architecture.violations.ApplicationDiagnosticFrameworkViolation; +import dev.caskeleton.application.realtime.GetReplayWindowUseCase; import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.messaging.LibraryMessageHandlerSystemCallerFixture; import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.messaging.RecordSystemCallerFixture; import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.messaging.SystemCallerHandlerFixture; import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.messaging.SystemCallerHandlerRegistrationFixture; import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.messaging.SystemCallerWithNestedValueTypesFixture; import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.web.PingControllerFixture; +import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.web.UseCaseNamedServiceHoldingControllerFixture; +import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.web.UseCaseReadingStreamBridgeFixture; import dev.caskeleton.bootstrap.architecture.allowed.application.CleanProjectionQueryPort; +import dev.caskeleton.bootstrap.architecture.allowed.application.ReportExportService; import dev.caskeleton.bootstrap.architecture.allowed.configuration.FrameCountingInterceptorFixture; import dev.caskeleton.bootstrap.architecture.allowed.configuration.FrameInterceptorConfigurationFixture; import dev.caskeleton.bootstrap.architecture.allowed.configuration.OrderHandlerFixture; @@ -36,14 +40,20 @@ import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.schedule import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.scheduler.ScheduledWorkFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.scheduler.SystemCallerWithNestedEndpointFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.scheduler.SystemJobLauncherFixture; +import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.CoordinatorHoldingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.DomainRepositoryHoldingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.MessageHandlerInjectingControllerFixture; +import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.ServiceInterfaceHoldingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.SystemCallerControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.SystemCallerInvokingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.SystemCallerRelayingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.SystemInvokedCallingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.SystemJobInterfaceInvokingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.UnmarkedRelayCallingControllerFixture; +import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream.IdGeneratingReceiptHandlerFixture; +import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream.IdMintingReceiptHandlerFixture; +import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream.IdentifierHoldingReceiptHandlerFixture; +import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream.QueryPortHoldingStreamBridgeFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.outbound.RawExternalResponseFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.outbound.messaging.GenericTypeLeakingAdapterFixture; import dev.caskeleton.bootstrap.architecture.violations.application.BulkWriteWithoutWriteAccessUseCase; @@ -56,6 +66,12 @@ import dev.caskeleton.bootstrap.architecture.violations.application.OutboundWith import dev.caskeleton.bootstrap.architecture.violations.application.RawLeakQueryPort; import dev.caskeleton.bootstrap.architecture.violations.application.ReadOnlyRepositoryWriteUseCase; import dev.caskeleton.bootstrap.architecture.violations.application.SystemInvokedFixtureUseCase; +import dev.caskeleton.bootstrap.architecture.violations.application.history.EventHistoryQueryPort; +import dev.caskeleton.bootstrap.architecture.violations.application.history.ReceiptIdFactory; +import dev.caskeleton.bootstrap.architecture.violations.application.history.ReceiptIdGeneratorPort; +import dev.caskeleton.bootstrap.architecture.violations.application.history.ReceiptIdentifierPort; +import dev.caskeleton.bootstrap.architecture.violations.application.maintenance.MaintenanceService; +import dev.caskeleton.bootstrap.architecture.violations.application.maintenance.WriterSlotCoordinator; import dev.caskeleton.bootstrap.architecture.violations.configuration.RoutesBesideASystemCallerConfigurationFixture; import dev.caskeleton.bootstrap.architecture.violations.configuration.RunNowHandlerFunctionConfigurationFixture; import dev.caskeleton.bootstrap.architecture.violations.configuration.RunNowRouteConfigurationFixture; @@ -73,9 +89,12 @@ import dev.caskeleton.bootstrap.architecture.violations.persistence.flyway.Refer import dev.caskeleton.bootstrap.architecture.violations.security.AuthorizationManagerReachingSystemCallerFixture; import dev.caskeleton.bootstrap.architecture.violations.streaming.JakartaWebSocketEndpointFixture; import dev.caskeleton.bootstrap.architecture.violations.streaming.SpringWebSocketHandlerFixture; +import java.util.stream.Stream; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; /** @@ -1811,6 +1830,187 @@ class ArchitectureViolationFixtureTest { .contains(OrderRepository.class.getName())); } + @Test + @DisplayName("controller가 아닌 web stream bridge도 query 포트를 직접 들면 사용자 전송 포트 규칙에 걸린다") + void aWebStreamBridgeHoldingAQueryPortIsCaught() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses(QueryPortHoldingStreamBridgeFixture.class, EventHistoryQueryPort.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()) + .anySatisfy( + detail -> + assertThat(detail) + .contains(QueryPortHoldingStreamBridgeFixture.class.getName()) + .contains(EventHistoryQueryPort.class.getName())); + } + + @Test + @DisplayName("domain repository 포트를 든 controller는 사용자 전송 포트 규칙에도 걸린다") + void aControllerHoldingADomainRepositoryPortIsCaughtByTheTransportRuleToo() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses(DomainRepositoryHoldingControllerFixture.class, OrderRepository.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()) + .anySatisfy( + detail -> + assertThat(detail) + .contains(DomainRepositoryHoldingControllerFixture.class.getName())); + } + + @Test + @DisplayName("web handler가 id factory를 들고 id를 직접 만들면 사용자 전송 포트 규칙에 걸린다") + void aWebHandlerMintingIdsIsCaught() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses(IdMintingReceiptHandlerFixture.class, ReceiptIdFactory.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()) + .anySatisfy( + detail -> + assertThat(detail) + .contains(IdMintingReceiptHandlerFixture.class.getName()) + .contains(ReceiptIdFactory.class.getName())); + } + + static Stream idPortsNamedOtherThanFactory() { + return Stream.of( + Arguments.of(IdGeneratingReceiptHandlerFixture.class, ReceiptIdGeneratorPort.class), + Arguments.of(IdentifierHoldingReceiptHandlerFixture.class, ReceiptIdentifierPort.class)); + } + + @ParameterizedTest(name = "{1}") + @MethodSource("idPortsNamedOtherThanFactory") + void aWebHandlerHoldingAnIdPortOfAnotherNameIsCaught(Class handler, Class port) { + // given + JavaClasses fixtureOnly = new ClassFileImporter().importClasses(handler, port); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()) + .anySatisfy( + detail -> assertThat(detail).contains(handler.getName()).contains(port.getName())); + } + + @Test + void aStreamBridgeReadingThroughAQueryUseCasePassesTheTransportPortRule() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses(UseCaseReadingStreamBridgeFixture.class, GetReplayWindowUseCase.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()).isEmpty(); + } + + @Test + @DisplayName("controller가 시스템 유스케이스가 감싸는 서비스 interface를 들면 inbound 서비스 규칙에 걸린다") + void aControllerHoldingAServiceInterfaceIsCaught() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses( + ServiceInterfaceHoldingControllerFixture.class, MaintenanceService.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()) + .anySatisfy( + detail -> + assertThat(detail) + .contains(ServiceInterfaceHoldingControllerFixture.class.getName()) + .contains(MaintenanceService.class.getName())); + } + + @Test + void aControllerHoldingACoordinatorIsCaught() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses(CoordinatorHoldingControllerFixture.class, WriterSlotCoordinator.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()) + .anySatisfy( + detail -> + assertThat(detail) + .contains(CoordinatorHoldingControllerFixture.class.getName()) + .contains(WriterSlotCoordinator.class.getName())); + } + + @Test + void aControllerHoldingAUseCaseNamedServicePassesTheInboundServiceRule() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses( + UseCaseNamedServiceHoldingControllerFixture.class, ReportExportService.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()).isEmpty(); + } + + @Test + void aStreamBridgeReadingThroughAQueryUseCasePassesTheInboundServiceRule() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses(UseCaseReadingStreamBridgeFixture.class, GetReplayWindowUseCase.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()).isEmpty(); + } + @Test @DisplayName("포트를 들지 않고 inbound 패키지에 있는 controller는 endpoint 포트 규칙과 배치 규칙을 통과한다") void anInboundControllerHoldingNoPersistencePortPasses() { diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/FileserverUseCaseContractDeviationTest.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/FileserverUseCaseContractDeviationTest.java index 4170e12f..1bfd3980 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/FileserverUseCaseContractDeviationTest.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/FileserverUseCaseContractDeviationTest.java @@ -8,7 +8,6 @@ import com.tngtech.archunit.core.importer.ClassFileImporter; import dev.caskeleton.application.usecase.UseCase; import java.util.List; import java.util.Set; -import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; /** @@ -21,12 +20,12 @@ import org.junit.jupiter.api.Test; * *

Every operation the transport reaches is now a use case (owner decision 32): one port per * operation under {@code application.fileserver.port.in}, each implementation declaring its - * capability and, for the writes, a {@code file:*} permission. What remains here is what no - * transport calls. The cleanup and reconciliation services are driven by background workers that - * have no caller identity; a mutating use case must declare a permission - * (MUTATING_USE_CASES_DECLARE_REQUIRED_PERMISSION), and a permission a scheduler cannot hold would - * refuse every run. The writer-lease coordinator is a collaborator the append use case holds, not - * an entrypoint. + * capability and, for the writes, a {@code file:*} permission. What remains here are collaborators, + * not entrypoints: the cleanup and reconciliation services, interface and implementation, are held + * by the operator's use cases and by the cleanup worker's two {@code @SystemInvoked} use cases + * ({@code RunScheduledCleanupUseCase}, {@code ReconcileRecoverableFilesUseCase}, round9 review + * 2026-11-12 RR9-05-06), and the writer-lease coordinator by the append use case. No transport and + * no worker reaches any of them without a use case in between. * *

A new Fileserver application service still fails this test, so the exemption cannot quietly * grow back. @@ -40,68 +39,46 @@ class FileserverUseCaseContractDeviationTest { * a pattern would silently absorb the next service, which is the behaviour being prevented. */ private static final Set KNOWN_DEVIATIONS = - Set.of("DefaultCleanupService", "DefaultFileReconciliationService", "WriterLeaseCoordinator"); + Set.of( + "CleanupService", + "DefaultCleanupService", + "DefaultFileReconciliationService", + "FileReconciliationService", + "WriterLeaseCoordinator"); @Test - @DisplayName("no new Fileserver application service may bypass the use-case contract") - void theExemptionIsFrozenAndMayOnlyShrink() { + void theServicesOutsideTheUseCaseContractAreExactlyTheFrozenList() { + // given JavaClasses imported = new ClassFileImporter() .withImportOption(new ProductionClassImportOption()) .importPackages("dev.caskeleton.application.fileserver"); + // when List bypassing = imported.stream() .filter(JavaClass::isTopLevelClass) - .filter(type -> !type.isInterface() && !type.isEnum()) + .filter(type -> !type.isEnum()) .filter( type -> type.getModifiers() .contains(com.tngtech.archunit.core.domain.JavaModifier.PUBLIC)) - .filter(FileserverUseCaseContractDeviationTest::looksLikeAnInboundService) - .filter(type -> !type.isAssignableTo(UseCase.class)) + // The inbound rule's own definition, interfaces and coordinators included, so a type + // that rule refuses a transport can only be one this list names. + .filter(AdapterBoundaryArchitectureTest.ARE_APPLICATION_SERVICES) .map(JavaClass::getSimpleName) .sorted() .toList(); + // then assertThat(bypassing) .as( "a Fileserver application service that is not a use case declares none of the " + "transaction mode, idempotency or repository access the contract requires, and " + "the ArchUnit rules cannot see it. Make it a use case behind a port.in " - + "interface (docs/fileserver/design-deviations.md §5).") - .allSatisfy(name -> assertThat(KNOWN_DEVIATIONS).contains(name)); - } - - @Test - @DisplayName("the exemption list names only services that still exist") - void theExemptionListDoesNotOutliveItsEntries() { - JavaClasses imported = - new ClassFileImporter() - .withImportOption(new ProductionClassImportOption()) - .importPackages("dev.caskeleton.application.fileserver"); - - Set present = - imported.stream() - .map(JavaClass::getSimpleName) - .collect(java.util.stream.Collectors.toSet()); - - assertThat(KNOWN_DEVIATIONS) - .as( - "an entry for a service that no longer exists hides the fact that the deviation " - + "shrank, which is the only progress signal this list carries") - .allSatisfy(name -> assertThat(present).contains(name)); - } - - /** - * An orchestrating service, as opposed to a value, a policy or a port. - * - *

Name-based because that is what the repository's own naming convention makes reliable here: - * these types are reached from a controller and coordinate ports, which is precisely the shape - * the use-case contract governs. - */ - private static boolean looksLikeAnInboundService(JavaClass type) { - String name = type.getSimpleName(); - return name.endsWith("Service") || name.endsWith("Coordinator"); + + "interface (docs/fileserver/design-deviations.md §5). An entry whose service " + + "became a use case or went away is struck from the list, the only progress " + + "signal it carries.") + .containsExactlyInAnyOrderElementsOf(KNOWN_DEVIATIONS); } } diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/NotificationUseCaseContractDeviationTest.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/NotificationUseCaseContractDeviationTest.java new file mode 100644 index 00000000..f84281a0 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/NotificationUseCaseContractDeviationTest.java @@ -0,0 +1,81 @@ +package dev.caskeleton.bootstrap.architecture; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.tngtech.archunit.core.domain.JavaClass; +import com.tngtech.archunit.core.domain.JavaClasses; +import com.tngtech.archunit.core.domain.JavaModifier; +import com.tngtech.archunit.core.importer.ClassFileImporter; +import dev.caskeleton.application.usecase.UseCase; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; + +/** + * Freezes the notification services that are not use cases, as {@link + * FileserverUseCaseContractDeviationTest} does for the Fileserver. + * + *

The ArchUnit rules that check capability, transaction boundary and permission only match types + * that implement {@link UseCase}; a service that never does is invisible to them. Every entrypoint + * of the platform is now a use case: the transport's through {@code platform.port.in}, the + * operator's four, and the system's — claim and dispatch, lease recovery, reconciliation, + * provider-event replay — as {@code @SystemInvoked} use cases the composition's workers hold + * (round9 review 2026-11-12, RR9-05-06). What remains below are the collaborators those use cases + * share: each opens its own boundaries around each write, which is why it is not one use case with + * one declared boundary. + * + *

A new notification service that is not a use case fails this test, so the list cannot quietly + * grow back. + */ +class NotificationUseCaseContractDeviationTest { + + /** + * The collaborators that predate the rule. + * + *

May shrink and must not grow. Spelled out rather than derived: a pattern would absorb the + * next service, which is the behaviour being prevented. + */ + private static final Set KNOWN_COLLABORATORS = + Set.of( + "DeduplicationService", + "LeaseRecoveryService", + "NotificationDispatchService", + "NotificationSubmissionService", + "ProviderEventProjectionService", + "ReconciliationService"); + + @Test + void theServicesOutsideTheUseCaseContractAreExactlyTheFrozenList() { + // given + JavaClasses imported = notificationClasses(); + + // when + List bypassing = + imported.stream() + .filter(JavaClass::isTopLevelClass) + .filter(type -> !type.isEnum()) + .filter(type -> type.getModifiers().contains(JavaModifier.PUBLIC)) + // The inbound rule's own definition, interfaces and orchestrators included, so a type + // that rule refuses a transport can only be one this list names. + .filter(AdapterBoundaryArchitectureTest.ARE_APPLICATION_SERVICES) + .map(JavaClass::getSimpleName) + .sorted() + .toList(); + + // then + assertThat(bypassing) + .as( + "a new notification service that is not a use case declares no capability, boundary" + + " or permission, and the ArchUnit rules cannot see it: make the entrypoint a use" + + " case and keep the service a collaborator it holds. An entry whose service" + + " became a use case or went away is struck from the list, the only progress" + + " signal it carries") + .containsExactlyInAnyOrderElementsOf(KNOWN_COLLABORATORS); + } + + private static JavaClasses notificationClasses() { + return new ClassFileImporter() + .withImportOption(new ProductionClassImportOption()) + .importPackages("dev.caskeleton.application.notification"); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseNamedServiceHoldingControllerFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseNamedServiceHoldingControllerFixture.java new file mode 100644 index 00000000..6c950da1 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseNamedServiceHoldingControllerFixture.java @@ -0,0 +1,25 @@ +package dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.web; + +import dev.caskeleton.bootstrap.architecture.allowed.application.ReportExportService; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * Positive fixture for {@code INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES}: a + * controller handed a use case that happens to be named {@code *Service}. The rule refuses services + * that are not use cases, so this one passes. + */ +@RestController +public class UseCaseNamedServiceHoldingControllerFixture { + + private final ReportExportService exports; + + public UseCaseNamedServiceHoldingControllerFixture(ReportExportService exports) { + this.exports = exports; + } + + @PostMapping("/exports") + public int export() { + return exports.handle(new ReportExportService.Export()); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseReadingStreamBridgeFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseReadingStreamBridgeFixture.java new file mode 100644 index 00000000..186972dc --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseReadingStreamBridgeFixture.java @@ -0,0 +1,22 @@ +package dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.web; + +import dev.caskeleton.application.realtime.GetReplayWindowQuery; +import dev.caskeleton.application.realtime.GetReplayWindowUseCase; + +/** + * Positive fixture for {@code USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS} and {@code + * INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES}: a web stream bridge that reads + * through a query use case, which both rules let through. + */ +public final class UseCaseReadingStreamBridgeFixture { + + private final GetReplayWindowUseCase window; + + public UseCaseReadingStreamBridgeFixture(GetReplayWindowUseCase window) { + this.window = window; + } + + public boolean resumable(String stream, long position) { + return window.handle(new GetReplayWindowQuery(stream)).canResumeAfter(position); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/package-info.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/package-info.java index 2ed3ad1c..3cf8408f 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/package-info.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/package-info.java @@ -2,5 +2,8 @@ * "Allowed-as-data" web endpoint for the system-caller rules: a controller in a user-transport * package that reaches no system caller. A configuration may register it beside a system caller * (review 2026-11-04 RR4-01-02). + * + *

Also the inbound adapters the transport and inbound service rules let through: a controller + * holding a use case named {@code *Service}, and a stream bridge reading through a query use case. */ package dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.web; diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/ReportExportService.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/ReportExportService.java new file mode 100644 index 00000000..2d21e091 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/ReportExportService.java @@ -0,0 +1,28 @@ +package dev.caskeleton.bootstrap.architecture.allowed.application; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.command.Command; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.usecase.CommandUseCase; + +/** + * A use case whose name ends in {@code Service}, as older application code still has: the target of + * {@code UseCaseNamedServiceHoldingControllerFixture}, which the inbound service rule lets through + * because the type is a use case. + */ +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.NONE) +public class ReportExportService implements CommandUseCase { + + @Override + public Integer handle(Export command) { + return 0; + } + + /** The export request. */ + public record Export() implements Command {} +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/package-info.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/package-info.java index b3d4464f..f30a8790 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/package-info.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/package-info.java @@ -4,5 +4,8 @@ * QUERY_PORTS_DO_NOT_LEAK_DOMAIN_JPA_OR_WEB_TYPES} rule must NOT flag — loaded in isolation by * {@link dev.caskeleton.bootstrap.architecture.ArchitectureViolationFixtureTest} to prove the rule * does not over-block clean projection reads. + * + *

Also the targets of the inbound service rule's positive fixtures: a use case whose name ends + * in {@code Service} ({@code ReportExportService}) is a port, not a wrapped service. */ package dev.caskeleton.bootstrap.architecture.allowed.application; diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/CoordinatorHoldingControllerFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/CoordinatorHoldingControllerFixture.java new file mode 100644 index 00000000..63d0f205 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/CoordinatorHoldingControllerFixture.java @@ -0,0 +1,24 @@ +package dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller; + +import dev.caskeleton.bootstrap.architecture.violations.application.maintenance.WriterSlotCoordinator; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * Negative fixture for {@code INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES}: a + * controller handed an application coordinator instead of the use case that holds it. + */ +@RestController +public class CoordinatorHoldingControllerFixture { + + private final WriterSlotCoordinator writers; + + public CoordinatorHoldingControllerFixture(WriterSlotCoordinator writers) { + this.writers = writers; + } + + @PostMapping("/writers") + public boolean take() { + return writers.take("key"); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/ServiceInterfaceHoldingControllerFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/ServiceInterfaceHoldingControllerFixture.java new file mode 100644 index 00000000..4a033bba --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/ServiceInterfaceHoldingControllerFixture.java @@ -0,0 +1,25 @@ +package dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller; + +import dev.caskeleton.bootstrap.architecture.violations.application.maintenance.MaintenanceService; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * Negative fixture for {@code INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES}: a + * controller handed the service interface a {@code @SystemInvoked} use case wraps, which runs the + * write with no permission check. The rule once skipped interfaces (round9 review 2026-11-12). + */ +@RestController +public class ServiceInterfaceHoldingControllerFixture { + + private final MaintenanceService maintenance; + + public ServiceInterfaceHoldingControllerFixture(MaintenanceService maintenance) { + this.maintenance = maintenance; + } + + @PostMapping("/maintenance") + public int run() { + return maintenance.runOnce(); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdGeneratingReceiptHandlerFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdGeneratingReceiptHandlerFixture.java new file mode 100644 index 00000000..58e2fd26 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdGeneratingReceiptHandlerFixture.java @@ -0,0 +1,21 @@ +package dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream; + +import dev.caskeleton.bootstrap.architecture.violations.application.history.ReceiptIdGeneratorPort; + +/** + * Negative fixture for {@code USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS}: a web handler + * minting ids through a port named {@code *IdGeneratorPort}, as the notification platform's is, + * rather than {@code *IdFactory}. + */ +public final class IdGeneratingReceiptHandlerFixture { + + private final ReceiptIdGeneratorPort ids; + + public IdGeneratingReceiptHandlerFixture(ReceiptIdGeneratorPort ids) { + this.ids = ids; + } + + public String accept() { + return ids.nextId(); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdMintingReceiptHandlerFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdMintingReceiptHandlerFixture.java new file mode 100644 index 00000000..d947c15d --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdMintingReceiptHandlerFixture.java @@ -0,0 +1,21 @@ +package dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream; + +import dev.caskeleton.bootstrap.architecture.violations.application.history.ReceiptIdFactory; + +/** + * Negative fixture for {@code USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS}: a web handler + * that mints the id the use case will store instead of leaving it to the use case, as the durable + * operation controller did (round9 review 2026-11-12). + */ +public final class IdMintingReceiptHandlerFixture { + + private final ReceiptIdFactory ids; + + public IdMintingReceiptHandlerFixture(ReceiptIdFactory ids) { + this.ids = ids; + } + + public String accept() { + return ids.next(); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdentifierHoldingReceiptHandlerFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdentifierHoldingReceiptHandlerFixture.java new file mode 100644 index 00000000..792acdd2 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdentifierHoldingReceiptHandlerFixture.java @@ -0,0 +1,20 @@ +package dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream; + +import dev.caskeleton.bootstrap.architecture.violations.application.history.ReceiptIdentifierPort; + +/** + * Negative fixture for {@code USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS}: a web handler + * minting ids through a port named {@code *IdentifierPort}, as the fileserver's upload id port is. + */ +public final class IdentifierHoldingReceiptHandlerFixture { + + private final ReceiptIdentifierPort ids; + + public IdentifierHoldingReceiptHandlerFixture(ReceiptIdentifierPort ids) { + this.ids = ids; + } + + public String accept() { + return ids.nextId(); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/QueryPortHoldingStreamBridgeFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/QueryPortHoldingStreamBridgeFixture.java new file mode 100644 index 00000000..6bef9ce2 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/QueryPortHoldingStreamBridgeFixture.java @@ -0,0 +1,21 @@ +package dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream; + +import dev.caskeleton.bootstrap.architecture.violations.application.history.EventHistoryQueryPort; + +/** + * Negative fixture for {@code USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS}: a web stream + * bridge — no controller — that reads a query port itself instead of through a query use case + * (round9 review 2026-11-12). + */ +public final class QueryPortHoldingStreamBridgeFixture { + + private final EventHistoryQueryPort history; + + public QueryPortHoldingStreamBridgeFixture(EventHistoryQueryPort history) { + this.history = history; + } + + public int replayed(String stream) { + return history.count(stream); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/EventHistoryQueryPort.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/EventHistoryQueryPort.java new file mode 100644 index 00000000..40e74c4b --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/EventHistoryQueryPort.java @@ -0,0 +1,8 @@ +package dev.caskeleton.bootstrap.architecture.violations.application.history; + +/** A read-model query port, the target of {@code QueryPortHoldingStreamBridgeFixture}. */ +public interface EventHistoryQueryPort { + + /** How many events the stream holds. */ + int count(String stream); +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdFactory.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdFactory.java new file mode 100644 index 00000000..105e97bb --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdFactory.java @@ -0,0 +1,8 @@ +package dev.caskeleton.bootstrap.architecture.violations.application.history; + +/** A server-assigned id factory, the target of {@code IdMintingReceiptHandlerFixture}. */ +public interface ReceiptIdFactory { + + /** The next receipt id. */ + String next(); +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdGeneratorPort.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdGeneratorPort.java new file mode 100644 index 00000000..024978b7 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdGeneratorPort.java @@ -0,0 +1,8 @@ +package dev.caskeleton.bootstrap.architecture.violations.application.history; + +/** An id port named the older way, the target of {@code IdGeneratingReceiptHandlerFixture}. */ +public interface ReceiptIdGeneratorPort { + + /** The next receipt id. */ + String nextId(); +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdentifierPort.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdentifierPort.java new file mode 100644 index 00000000..09106f54 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdentifierPort.java @@ -0,0 +1,11 @@ +package dev.caskeleton.bootstrap.architecture.violations.application.history; + +/** + * An id port named {@code *IdentifierPort}, as the fileserver's upload id port is: the target of + * {@code IdentifierHoldingReceiptHandlerFixture}. + */ +public interface ReceiptIdentifierPort { + + /** The next receipt id. */ + String nextId(); +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/MaintenanceService.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/MaintenanceService.java new file mode 100644 index 00000000..3433f2a5 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/MaintenanceService.java @@ -0,0 +1,11 @@ +package dev.caskeleton.bootstrap.architecture.violations.application.maintenance; + +/** + * An application service interface that a system use case wraps, the target of {@code + * ServiceInterfaceHoldingControllerFixture}. + */ +public interface MaintenanceService { + + /** Runs one maintenance pass with no permission check of its own. */ + int runOnce(); +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/WriterSlotCoordinator.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/WriterSlotCoordinator.java new file mode 100644 index 00000000..f2e64e56 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/WriterSlotCoordinator.java @@ -0,0 +1,10 @@ +package dev.caskeleton.bootstrap.architecture.violations.application.maintenance; + +/** An application coordinator, the target of {@code CoordinatorHoldingControllerFixture}. */ +public final class WriterSlotCoordinator { + + /** Takes the writer slot for a key. */ + public boolean take(String key) { + return !key.isEmpty(); + } +} diff --git a/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverCleanupWorker.java b/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverCleanupWorker.java index fd0a7aae..fbd7b694 100644 --- a/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverCleanupWorker.java +++ b/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverCleanupWorker.java @@ -4,7 +4,11 @@ import static net.logstash.logback.argument.StructuredArguments.kv; import dev.caskeleton.application.fileserver.cleanup.CleanupBatchResult; import dev.caskeleton.application.fileserver.cleanup.CleanupService; +import dev.caskeleton.application.fileserver.cleanup.RunScheduledCleanupCommand; +import dev.caskeleton.application.fileserver.cleanup.RunScheduledCleanupUseCase; import dev.caskeleton.application.fileserver.recovery.FileReconciliationService; +import dev.caskeleton.application.fileserver.recovery.ReconcileRecoverableFilesCommand; +import dev.caskeleton.application.fileserver.recovery.ReconcileRecoverableFilesUseCase; import dev.caskeleton.application.fileserver.recovery.ReconciliationResult; import java.util.List; import org.slf4j.Logger; @@ -28,28 +32,30 @@ import org.slf4j.LoggerFactory; * interval rather than declared here with a property placeholder. Two declarations of the same * interval — one typed and validated, one a raw string with its own default — is one declaration * too many: they drift, and the one that wins is whichever the scheduler happened to read. + * + *

Each step is a {@code @SystemInvoked} use case the worker constructs from the services it is + * given and keeps to itself, the way the outbox relay's scheduler holds its use case: a write the + * system starts is a use case of its own, apart from the one an operator starts with a permission + * (manual 07 「시스템이 부르는 쓰기」). Neither is a bean. */ public final class FileserverCleanupWorker { private static final Logger LOG = LoggerFactory.getLogger(FileserverCleanupWorker.class); - private final CleanupService cleanupService; - private final FileReconciliationService reconciliationService; - private final int maxItems; - private final long maxBytes; + private final RunScheduledCleanupUseCase cleanup; + private final ReconcileRecoverableFilesUseCase reconciliation; + private final RunScheduledCleanupCommand batch; + private final ReconcileRecoverableFilesCommand pass; public FileserverCleanupWorker( CleanupService cleanupService, FileReconciliationService reconciliationService, int maxItems, long maxBytes) { - if (maxItems < 1 || maxBytes < 1) { - throw new IllegalArgumentException("cleanup batch bounds must be positive"); - } - this.cleanupService = cleanupService; - this.reconciliationService = reconciliationService; - this.maxItems = maxItems; - this.maxBytes = maxBytes; + this.cleanup = new RunScheduledCleanupUseCase(cleanupService); + this.reconciliation = new ReconcileRecoverableFilesUseCase(reconciliationService); + this.batch = new RunScheduledCleanupCommand(maxItems, maxBytes); + this.pass = new ReconcileRecoverableFilesCommand(maxItems); } /** @@ -66,7 +72,7 @@ public final class FileserverCleanupWorker { private void runReconciliation() { try { - List results = reconciliationService.reconcileRecoverable(maxItems); + List results = reconciliation.handle(pass); if (!results.isEmpty()) { LOG.info( "fileserver.reconcile examined={}", @@ -88,7 +94,7 @@ public final class FileserverCleanupWorker { private void runCleanup() { try { - CleanupBatchResult result = cleanupService.runBatch(maxItems, maxBytes); + CleanupBatchResult result = cleanup.handle(batch); if (result.deleted() > 0 || result.failed() > 0) { LOG.info( "fileserver.cleanup deleted={} skippedActiveLease={} skippedStateChanged={} " diff --git a/src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/autoconfigure/FileserverMaintenanceServiceConfiguration.java b/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverMaintenanceServiceConfiguration.java similarity index 79% rename from src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/autoconfigure/FileserverMaintenanceServiceConfiguration.java rename to src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverMaintenanceServiceConfiguration.java index 6cdc398f..5b3bf455 100644 --- a/src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/autoconfigure/FileserverMaintenanceServiceConfiguration.java +++ b/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverMaintenanceServiceConfiguration.java @@ -1,5 +1,7 @@ -package dev.caskeleton.adapter.outbound.fileserver.platform.autoconfigure; +package dev.caskeleton.composition.fileserver; +import dev.caskeleton.adapter.outbound.fileserver.platform.autoconfigure.FileserverMetricsSupport; +import dev.caskeleton.adapter.outbound.fileserver.platform.autoconfigure.FileserverPlatformSettings; import dev.caskeleton.application.fileserver.api.metadata.FileMetadataPort; import dev.caskeleton.application.fileserver.api.metadata.UploadSessionPort; import dev.caskeleton.application.fileserver.cleanup.CleanupContentPort; @@ -21,6 +23,15 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +/** + * The cleanup and reconciliation services: the operator's cleanup use case and the cleanup worker + * both hold them. + * + *

Here rather than in {@code adapter:outbound:fileserver}, where they were: each one decides + * what may be deleted or restored and opens its own transactions around each write, and an outbound + * adapter assembling application services is the adapter deciding where a unit of work begins + * (round9 review 2026-11-12, RR9-05-08). + */ @Configuration(proxyBeanMethods = false) class FileserverMaintenanceServiceConfiguration { diff --git a/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverPlatformAutoConfiguration.java b/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverPlatformAutoConfiguration.java index 6b093f55..51104259 100644 --- a/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverPlatformAutoConfiguration.java +++ b/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverPlatformAutoConfiguration.java @@ -45,6 +45,7 @@ import org.springframework.context.annotation.Import; FileserverFileUseCaseConfiguration.class, FileserverAdminReadUseCaseConfiguration.class, FileserverAdminActionUseCaseConfiguration.class, + FileserverMaintenanceServiceConfiguration.class, FileserverCleanupConfiguration.class, FileserverCleanupDisabledConfiguration.class, // The HTTP surface itself. Its controllers used to be found by the composition root's component diff --git a/src/bootstrap/notification-platform/README.md b/src/bootstrap/notification-platform/README.md index 962b47b1..3396d220 100644 --- a/src/bootstrap/notification-platform/README.md +++ b/src/bootstrap/notification-platform/README.md @@ -13,8 +13,10 @@ composition:notification-platform ├─ NotificationApplicationCompositionAutoConfiguration (use case 조립, `enabled` 조건) │ ├─ NotificationSubmissionConfiguration 접수·예약·취소·조회·템플릿 발행 use case - │ ├─ NotificationDeliveryConfiguration 발송·재조정·lease 회수·ledger 투영·운영자 plane - │ └─ NotificationCallbackConfiguration provider 콜백 수집 + ProviderCallbackIngressBridge + │ ├─ NotificationDeliveryConfiguration 발송·재조정·lease 회수·ledger 투영 서비스 + │ ├─ NotificationAdminConfiguration 운영자 use case 넷(redrive·reconcile·suppress·provider 상태) + │ ├─ NotificationCallbackConfiguration provider 콜백 수집 + ProviderCallbackIngressBridge + │ └─ NotificationWorkerConfiguration (SERVING만) 발송 scheduler와 회복·replay·맞추기 worker ├─ config/module-defaults/notification.yml (설정 기본값, support 로더가 읽음) ├─ adapter:outbound:notification provider(SMTP·SES), 저장 외 런타임, NotificationPlatformAutoConfiguration ├─ adapter:outbound:persistence-notification (runtimeOnly) 알림 entity·저장소·Flyway stream @@ -25,6 +27,11 @@ composition:notification-platform - 조립 순서: 이 모듈의 root는 `NotificationPlatformAutoConfiguration`(outbound leaf) 뒤에 온다 (`afterName`). +- worker: 두 worker(`NotificationSchedulerWorker`, `NotificationBackgroundWorkers`)는 스레드와 일정만 가진다. 하는 일은 + `@SystemInvoked` use case(claim·발송·lease 회복·provider event replay·맞추기)와 backlog 측정 query use case이고, worker + 설정이 `new`로 만들어 넘긴다. + bean이 아니어서 어떤 전송도 닿지 못한다(manual 07 「시스템이 부르는 쓰기」). `mode=INGEST_ONLY`면 worker 설정 자체가 + 실리지 않는다. - 설정: `config/module-defaults/notification.yml`을 support의 `ModuleDefaultsEnvironmentPostProcessor`가 읽는다. 우선순위는 가장 낮다. 그래서 `application*.yml`, 환경 변수, 시스템 속성이 항상 이긴다. app-bootstrap은 이 파일을 모른다. - 저장소: 요청·발송·ledger 테이블과 migration은 `adapter:outbound:persistence-notification`에 있고, diff --git a/src/bootstrap/notification-platform/build.gradle b/src/bootstrap/notification-platform/build.gradle index 06dedbf6..bdb28475 100644 --- a/src/bootstrap/notification-platform/build.gradle +++ b/src/bootstrap/notification-platform/build.gradle @@ -27,6 +27,8 @@ dependencies { implementation 'org.springframework.boot:spring-boot-autoconfigure' implementation 'org.springframework:spring-context' + // The two worker threads log a failed pass instead of letting it cancel the schedule. + implementation 'org.slf4j:slf4j-api' testImplementation 'org.springframework.boot:spring-boot-starter-test' // NotificationModuleDefaultsTest reads the shipped file through the loader that serves it at diff --git a/src/bootstrap/notification-platform/gradle.lockfile b/src/bootstrap/notification-platform/gradle.lockfile index a981ae24..03e15727 100644 --- a/src/bootstrap/notification-platform/gradle.lockfile +++ b/src/bootstrap/notification-platform/gradle.lockfile @@ -151,7 +151,7 @@ org.reactivestreams:reactive-streams:1.0.4=runtimeClasspath,testRuntimeClasspath org.reflections:reflections:0.10.2=checkstyle org.skyscreamer:jsonassert:1.5.3=testCompileClasspath,testRuntimeClasspath org.slf4j:jul-to-slf4j:2.0.20=runtimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:2.0.20=runtimeClasspath,spotbugs,spotbugsSlf4j,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:2.0.20=compileClasspath,runtimeClasspath,spotbugs,spotbugsSlf4j,testCompileClasspath,testRuntimeClasspath org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j org.slf4j:slf4j-simple:2.0.20=checkstyle org.springdoc:springdoc-openapi-starter-common:3.0.0=runtimeClasspath,testRuntimeClasspath diff --git a/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationAdminConfiguration.java b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationAdminConfiguration.java new file mode 100644 index 00000000..92ff423f --- /dev/null +++ b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationAdminConfiguration.java @@ -0,0 +1,92 @@ +package dev.caskeleton.composition.notification; + +import dev.caskeleton.application.notification.platform.admin.AdminAuthorizationGuard; +import dev.caskeleton.application.notification.platform.admin.AdminOperationStorePort; +import dev.caskeleton.application.notification.platform.admin.ChangeProviderStateUseCase; +import dev.caskeleton.application.notification.platform.admin.ChangeSuppressionUseCase; +import dev.caskeleton.application.notification.platform.admin.DuplicateRiskGuard; +import dev.caskeleton.application.notification.platform.admin.ProviderRuntimeControlPort; +import dev.caskeleton.application.notification.platform.admin.ReconcileDeliveriesUseCase; +import dev.caskeleton.application.notification.platform.admin.RedriveDeliveryUseCase; +import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort; +import dev.caskeleton.application.notification.platform.dispatch.NotificationIdGeneratorPort; +import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort; +import dev.caskeleton.application.notification.platform.dispatch.ReconciliationService; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; +import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort; +import dev.caskeleton.application.transaction.TransactionPort; +import java.time.Clock; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * The operator plane: redrive, reconcile, suppress and provider control, one use case each. + * + *

Beans, so the method-security proxy applies each one's {@code @RequiresPermission}. The + * template ships no transport that calls them; a deployment that exposes the operator plane adds + * one. Which roles hold the {@code notification-admin:*} permissions by default is this module's + * {@code config/module-defaults/notification.yml}. + */ +@Configuration(proxyBeanMethods = false) +class NotificationAdminConfiguration { + + @Bean + RedriveDeliveryUseCase notificationRedriveDeliveryUseCase( + DeliveryAttemptStorePort attempts, + RecipientDeliveryStorePort recipients, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + Clock clock) { + return new RedriveDeliveryUseCase( + new AdminAuthorizationGuard(), + new DuplicateRiskGuard(), + attempts, + recipients, + operations, + audit, + transactions, + clock); + } + + @Bean + ReconcileDeliveriesUseCase notificationReconcileDeliveriesUseCase( + DeliveryAttemptStorePort attempts, + ReconciliationService reconciliation, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + Clock clock) { + return new ReconcileDeliveriesUseCase( + new AdminAuthorizationGuard(), + attempts, + reconciliation, + operations, + audit, + transactions, + clock); + } + + @Bean + ChangeSuppressionUseCase notificationChangeSuppressionUseCase( + SuppressionStorePort suppressions, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + NotificationIdGeneratorPort ids, + Clock clock) { + return new ChangeSuppressionUseCase( + new AdminAuthorizationGuard(), suppressions, operations, audit, transactions, ids, clock); + } + + @Bean + ChangeProviderStateUseCase notificationChangeProviderStateUseCase( + ProviderRuntimeControlPort runtimes, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + Clock clock) { + return new ChangeProviderStateUseCase( + new AdminAuthorizationGuard(), runtimes, operations, audit, transactions, clock); + } +} diff --git a/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationApplicationCompositionAutoConfiguration.java b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationApplicationCompositionAutoConfiguration.java index bfb43440..afbcef41 100644 --- a/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationApplicationCompositionAutoConfiguration.java +++ b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationApplicationCompositionAutoConfiguration.java @@ -7,9 +7,10 @@ import org.springframework.context.annotation.Import; /** * Application-owned Notification orchestration and transaction-bound use-case assembly. * - *

The root holds the capability switch and imports one configuration per concern. It orders - * itself after the outbound leaf's root, whose stores, providers and policies every use case here - * consumes. + *

The root holds the capability switch and imports one configuration per concern, the workers + * included: nothing else loads {@link NotificationWorkerConfiguration}, which carries the mode + * condition. It orders itself after the outbound leaf's root, whose stores, providers and policies + * every use case here consumes. */ @AutoConfiguration( afterName = @@ -22,6 +23,8 @@ import org.springframework.context.annotation.Import; @Import({ NotificationSubmissionConfiguration.class, NotificationDeliveryConfiguration.class, - NotificationCallbackConfiguration.class + NotificationAdminConfiguration.class, + NotificationCallbackConfiguration.class, + NotificationWorkerConfiguration.class }) public class NotificationApplicationCompositionAutoConfiguration {} diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationBackgroundWorkers.java b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationBackgroundWorkers.java similarity index 55% rename from src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationBackgroundWorkers.java rename to src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationBackgroundWorkers.java index b4585d25..2e8b2ea7 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationBackgroundWorkers.java +++ b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationBackgroundWorkers.java @@ -1,42 +1,58 @@ -package dev.caskeleton.adapter.outbound.notification.platform.dispatch; +package dev.caskeleton.composition.notification; -import dev.caskeleton.application.notification.platform.dispatch.LeaseRecoveryService; +import dev.caskeleton.application.notification.platform.callback.ReplayProviderEventsCommand; +import dev.caskeleton.application.notification.platform.callback.ReplayProviderEventsUseCase; +import dev.caskeleton.application.notification.platform.dispatch.ReconcileOutstandingAttemptsCommand; +import dev.caskeleton.application.notification.platform.dispatch.ReconcileOutstandingAttemptsUseCase; +import dev.caskeleton.application.notification.platform.dispatch.RecoverAbandonedDeliveriesCommand; +import dev.caskeleton.application.notification.platform.dispatch.RecoverAbandonedDeliveriesUseCase; import java.time.Duration; +import java.util.List; import java.util.Objects; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.IntSupplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Runs the recovery and replay passes, and stops them on shutdown. + * Runs the lease recovery, provider-event replay and reconciliation passes, and stops them on + * shutdown. * - *

Both existed as classes with no caller. {@code LeaseRecoveryService} was written to recover - * deliveries a dead worker left in flight and was never scheduled; the ledger's replay queries were - * implemented in persistence and never read. The dispatch scheduler's own error path assumed - * recovery existed — it leaves a lease to expire rather than releasing it optimistically, on the - * grounds that recovery will decide — so the absence turned a deliberate design into a leak. + *

The first two existed as classes with no caller. {@code LeaseRecoveryService} was written to + * recover deliveries a dead worker left in flight and was never scheduled; the ledger's replay + * queries were implemented in persistence and never read. The dispatch scheduler's own error path + * assumed recovery existed — it leaves a lease to expire rather than releasing it optimistically, + * on the grounds that recovery will decide — so the absence turned a deliberate design into a leak. * - *

One executor for both, because they are cheap, periodic, and must stop together. A failure in - * one pass is logged and the schedule continues: an exception escaping a scheduled task cancels it - * silently, which is how a background worker stops running without anybody being told. + *

One executor for the three, because they are cheap, periodic, and must stop together. A + * failure in one pass is logged and the schedule continues: an exception escaping a scheduled task + * cancels it silently, which is how a background worker stops running without anybody being told. + * + *

Each pass is a {@code @SystemInvoked} use case this composition constructs and hands in, the + * way the outbox relay's scheduler holds its use case. The class owns the thread and the schedule + * and nothing else; it lived in the notification adapter, which owned the reconciliation and replay + * decisions too (round9 review 2026-11-12, RR9-05-06, RR9-05-08). */ +// Public, unlike the scheduler worker: its bean is @ConditionalOnMissingBean, so a deployment may +// replace it with one of its own, built from the same use cases. public final class NotificationBackgroundWorkers implements AutoCloseable { private static final Logger log = LoggerFactory.getLogger(NotificationBackgroundWorkers.class); - private final LeaseRecoveryService recovery; - private final ProviderEventReplayWorker replay; - private final ReconciliationJobWorker reconciliation; + private final RecoverAbandonedDeliveriesUseCase recovery; + private final ReplayProviderEventsUseCase replay; + private final ReconcileOutstandingAttemptsUseCase reconciliation; private final Duration interval; private final Duration shutdownGrace; private final ScheduledExecutorService scheduler; private final AtomicBoolean started = new AtomicBoolean(); - private final java.util.List> passes = - new java.util.concurrent.CopyOnWriteArrayList<>(); + private final List> passes = new CopyOnWriteArrayList<>(); /** * Creates the workers. @@ -48,9 +64,9 @@ public final class NotificationBackgroundWorkers implements AutoCloseable { * @param shutdownGrace how long shutdown waits for a pass in flight */ public NotificationBackgroundWorkers( - LeaseRecoveryService recovery, - ProviderEventReplayWorker replay, - ReconciliationJobWorker reconciliation, + RecoverAbandonedDeliveriesUseCase recovery, + ReplayProviderEventsUseCase replay, + ReconcileOutstandingAttemptsUseCase reconciliation, Duration interval, Duration shutdownGrace) { this.recovery = Objects.requireNonNull(recovery, "recovery"); @@ -69,21 +85,22 @@ public final class NotificationBackgroundWorkers implements AutoCloseable { }); } - /** Starts both passes. Idempotent. */ + /** Starts the three passes. Idempotent. */ public void start() { if (!started.compareAndSet(false, true)) { return; } - schedule("recovery", recovery::recoverOnce); - schedule("provider-event-replay", replay::replayOnce); - schedule("reconciliation", reconciliation::reconcileOnce); + schedule("recovery", () -> recovery.handle(new RecoverAbandonedDeliveriesCommand())); + schedule("provider-event-replay", () -> replay.handle(new ReplayProviderEventsCommand())); + schedule( + "reconciliation", () -> reconciliation.handle(new ReconcileOutstandingAttemptsCommand())); } - private void schedule(String name, java.util.function.IntSupplier pass) { + private void schedule(String name, IntSupplier pass) { // The handle is kept so cancellation is possible and so the ignored-future check has an // answer: the task swallows its own exceptions, so the future never completes exceptionally // and there is nothing for a caller to observe on it. - java.util.concurrent.ScheduledFuture scheduled = + ScheduledFuture scheduled = scheduler.scheduleWithFixedDelay( () -> { try { @@ -135,7 +152,7 @@ public final class NotificationBackgroundWorkers implements AutoCloseable { private static Duration requirePositive(Duration value, String name) { Objects.requireNonNull(value, name); if (value.isNegative() || value.isZero()) { - throw new IllegalArgumentException(name + " must be positive and finite"); + throw new IllegalStateException(name + " must be positive and finite"); } return value; } diff --git a/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationDeliveryConfiguration.java b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationDeliveryConfiguration.java index 44156285..8fa0728f 100644 --- a/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationDeliveryConfiguration.java +++ b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationDeliveryConfiguration.java @@ -1,12 +1,6 @@ package dev.caskeleton.composition.notification; import dev.caskeleton.adapter.outbound.notification.platform.dispatch.NotificationDispatchProperties; -import dev.caskeleton.application.notification.platform.admin.AdminAuthorizationGuard; -import dev.caskeleton.application.notification.platform.admin.AdminOperationStorePort; -import dev.caskeleton.application.notification.platform.admin.DuplicateRiskGuard; -import dev.caskeleton.application.notification.platform.admin.NotificationAdminApplicationService; -import dev.caskeleton.application.notification.platform.admin.NotificationAdminService; -import dev.caskeleton.application.notification.platform.admin.ProviderRuntimeControlPort; import dev.caskeleton.application.notification.platform.callback.DeliveryAttemptResolverPort; import dev.caskeleton.application.notification.platform.callback.DeliveryProjectionStorePort; import dev.caskeleton.application.notification.platform.callback.NotificationSideEffectPort; @@ -42,8 +36,9 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; /** - * Doing the work and correcting it: dispatch, reconciliation, lease recovery, ledger projection and - * the operator plane that redrives and suppresses deliveries. + * Doing the work and correcting it: dispatch, reconciliation, lease recovery and ledger projection. + * The operator plane that redrives and suppresses deliveries is {@link + * NotificationAdminConfiguration}. * *

Here rather than in the notification adapter because each of these opens its own transactions * around each write, and the adapter must not decide where a unit of work begins (NTF-020). @@ -163,34 +158,4 @@ class NotificationDeliveryConfiguration { return new ProviderEventProjectionService( ledger, projector, resolver, projections, sideEffects, transactions, metrics); } - - /** The N4 operator plane. */ - @Bean - NotificationAdminService notificationAdminService( - AdminAuthorizationGuard authorization, - DuplicateRiskGuard duplicateRiskGuard, - DeliveryAttemptStorePort attempts, - RecipientDeliveryStorePort recipients, - ReconciliationService reconciliation, - SuppressionStorePort suppressions, - ProviderRuntimeControlPort runtimes, - AdminOperationStorePort operations, - NotificationAuditPort audit, - TransactionPort transactions, - NotificationIdGeneratorPort ids, - Clock clock) { - return new NotificationAdminApplicationService( - authorization, - duplicateRiskGuard, - attempts, - recipients, - reconciliation, - suppressions, - runtimes, - operations, - audit, - transactions, - ids, - clock); - } } diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationSchedulerWorker.java similarity index 80% rename from src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java rename to src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationSchedulerWorker.java index f54fa001..da822066 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java +++ b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationSchedulerWorker.java @@ -1,11 +1,15 @@ -package dev.caskeleton.adapter.outbound.notification.platform.dispatch; +package dev.caskeleton.composition.notification; -import dev.caskeleton.application.notification.platform.dispatch.NotificationDispatchService; +import dev.caskeleton.adapter.outbound.notification.platform.dispatch.NotificationDispatchProperties; +import dev.caskeleton.application.notification.platform.dispatch.ClaimDueDeliveriesCommand; +import dev.caskeleton.application.notification.platform.dispatch.ClaimDueDeliveriesUseCase; +import dev.caskeleton.application.notification.platform.dispatch.DispatchClaimedDeliveryCommand; +import dev.caskeleton.application.notification.platform.dispatch.DispatchClaimedDeliveryUseCase; import dev.caskeleton.application.notification.platform.dispatch.RecipientLease; -import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort; +import dev.caskeleton.application.notification.platform.observation.MeasureDispatchBacklogQuery; +import dev.caskeleton.application.notification.platform.observation.MeasureDispatchBacklogUseCase; import dev.caskeleton.application.notification.platform.observation.NotificationMetricName; import dev.caskeleton.application.notification.platform.observation.NotificationMetricsPort; -import dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort; import java.util.List; import java.util.Map; import java.util.Objects; @@ -27,16 +31,21 @@ import org.slf4j.LoggerFactory; *

Claiming is bounded twice over: by the claim batch size and by a global concurrency permit. * The second bound matters because a slow provider would otherwise let the queue depth become the * thread count. + * + *

What is claimed and how one delivery is sent are two {@code @SystemInvoked} use cases this + * composition constructs and hands in, and the backlog it publishes is a query use case; the worker + * owns the polling thread, the executor and the permits. It lived in the notification adapter + * (round9 review 2026-11-12, RR9-05-06, RR9-05-08). */ -public final class NotificationSchedulerWorker implements AutoCloseable { +final class NotificationSchedulerWorker implements AutoCloseable { private static final Logger log = LoggerFactory.getLogger(NotificationSchedulerWorker.class); - private final RecipientLeaseStorePort leases; - private final NotificationDispatchService dispatcher; + private final ClaimDueDeliveriesUseCase claim; + private final DispatchClaimedDeliveryUseCase dispatch; private final NotificationMetricsPort metrics; private final NotificationDispatchProperties properties; - private final NotificationServingStatePort servingState; + private final MeasureDispatchBacklogUseCase backlog; private final String workerId; private final ExecutorService dispatchExecutor; private final Semaphore globalConcurrency; @@ -46,18 +55,18 @@ public final class NotificationSchedulerWorker implements AutoCloseable { /** The polling thread, kept so shutdown can actually stop it. */ private volatile Thread pollingThread; - public NotificationSchedulerWorker( - RecipientLeaseStorePort leases, - NotificationDispatchService dispatcher, + NotificationSchedulerWorker( + ClaimDueDeliveriesUseCase claim, + DispatchClaimedDeliveryUseCase dispatch, NotificationMetricsPort metrics, NotificationDispatchProperties properties, - NotificationServingStatePort servingState, + MeasureDispatchBacklogUseCase backlog, String workerId) { - this.leases = Objects.requireNonNull(leases, "leases"); - this.dispatcher = Objects.requireNonNull(dispatcher, "dispatcher"); + this.claim = Objects.requireNonNull(claim, "claim"); + this.dispatch = Objects.requireNonNull(dispatch, "dispatch"); this.metrics = Objects.requireNonNull(metrics, "metrics"); this.properties = Objects.requireNonNull(properties, "properties"); - this.servingState = Objects.requireNonNull(servingState, "servingState"); + this.backlog = Objects.requireNonNull(backlog, "backlog"); this.workerId = Objects.requireNonNull(workerId, "workerId"); this.dispatchExecutor = Executors.newVirtualThreadPerTaskExecutor(); this.globalConcurrency = new Semaphore(properties.maxGlobalConcurrency()); @@ -76,22 +85,24 @@ public final class NotificationSchedulerWorker implements AutoCloseable { if (executable < 1) { return 0; } - List claimed = leases.claim(workerId, executable, properties.leaseDuration()); + List claimed = + claim.handle( + new ClaimDueDeliveriesCommand(workerId, executable, properties.leaseDuration())); // The backlog, not the batch. This gauge used to report claimed.size(), which is bounded above // by the claim batch size — so a queue of ten and a queue of ten million published the same // number, and the one metric named "queue depth" was the one that could not show a queue // growing. - // The depth alone: the full serving snapshot is several statements and is the health - // endpoint's, not something to run on every poll. metrics.gauge( - NotificationMetricName.QUEUE_DEPTH, Map.of(), (double) servingState.backlogDepth()); + NotificationMetricName.QUEUE_DEPTH, + Map.of(), + (double) backlog.handle(new MeasureDispatchBacklogQuery())); for (RecipientLease lease : claimed) { globalConcurrency.acquireUninterruptibly(); dispatchExecutor.execute( () -> { try { - dispatcher.dispatch(lease); + dispatch.handle(new DispatchClaimedDeliveryCommand(lease)); } catch (RuntimeException failure) { // The lease is left to expire rather than being released optimistically: a worker // that diff --git a/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationWorkerConfiguration.java b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationWorkerConfiguration.java new file mode 100644 index 00000000..5aff20af --- /dev/null +++ b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationWorkerConfiguration.java @@ -0,0 +1,129 @@ +package dev.caskeleton.composition.notification; + +import dev.caskeleton.adapter.outbound.notification.platform.dispatch.NotificationDispatchProperties; +import dev.caskeleton.application.notification.platform.callback.ProviderEventProjectionService; +import dev.caskeleton.application.notification.platform.callback.ReplayProviderEventsUseCase; +import dev.caskeleton.application.notification.platform.dispatch.ClaimDueDeliveriesUseCase; +import dev.caskeleton.application.notification.platform.dispatch.DispatchClaimedDeliveryUseCase; +import dev.caskeleton.application.notification.platform.dispatch.LeaseRecoveryService; +import dev.caskeleton.application.notification.platform.dispatch.NotificationDispatchService; +import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort; +import dev.caskeleton.application.notification.platform.dispatch.ReconcileOutstandingAttemptsUseCase; +import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJobStorePort; +import dev.caskeleton.application.notification.platform.dispatch.ReconciliationService; +import dev.caskeleton.application.notification.platform.dispatch.RecoverAbandonedDeliveriesUseCase; +import dev.caskeleton.application.notification.platform.observation.MeasureDispatchBacklogUseCase; +import dev.caskeleton.application.notification.platform.observation.NotificationMetricsPort; +import dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort; +import dev.caskeleton.application.transaction.TransactionPort; +import java.time.Clock; +import java.util.UUID; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Everything that runs on a schedule rather than on a request, and therefore everything {@code + * INGEST_ONLY} must not have. + * + *

The system's writes are {@code @SystemInvoked} use cases constructed here with {@code new} and + * handed to the two thread-owning workers, the way the outbox relay's composition hands its use + * case to its scheduler (manual 07 「시스템이 부르는 쓰기」). None of them is a bean: a bean is what a + * transport could reach, and these write with no permission check. The workers and this + * configuration lived in the notification adapter, which decided when to reconcile and replay there + * (round9 review 2026-11-12, RR9-05-06, RR9-05-08). + * + *

Gated on the mode as one configuration rather than a condition per bean. Both beans call + * {@code start()}, so a deployment in {@code INGEST_ONLY} — the mode whose whole purpose is to + * accept and store without delivering — would otherwise start the dispatch scheduler, lease + * recovery, provider-event replay and reconciliation. A per-bean condition is one a future bean can + * forget; a configuration that is not loaded cannot be. {@code matchIfMissing = true} because + * {@code SERVING} is the shipped default: an absent mode must still deliver. + */ +@Configuration(proxyBeanMethods = false) +@ConditionalOnProperty( + prefix = "app.notification.platform", + name = "mode", + havingValue = "SERVING", + matchIfMissing = true) +class NotificationWorkerConfiguration { + + /** + * Schedules lease recovery, provider-event replay and reconciliation, and stops them on shutdown. + * + * @param recovery the lease recovery service + * @param projection the projection the events feed + * @param jobs the outstanding reconciliation questions + * @param reconciliation asks one attempt's provider + * @param transactions opens the use cases' own boundaries + * @param clock the clock + * @param properties the dispatch bounds + * @return the started background workers + */ + @Bean(destroyMethod = "close") + @ConditionalOnMissingBean + NotificationBackgroundWorkers notificationBackgroundWorkers( + LeaseRecoveryService recovery, + ProviderEventProjectionService projection, + ReconciliationJobStorePort jobs, + ReconciliationService reconciliation, + TransactionPort transactions, + Clock clock, + NotificationDispatchProperties properties) { + NotificationBackgroundWorkers workers = + new NotificationBackgroundWorkers( + new RecoverAbandonedDeliveriesUseCase(recovery), + new ReplayProviderEventsUseCase(projection, properties.claimBatchSize()), + new ReconcileOutstandingAttemptsUseCase( + jobs, + reconciliation::reconcile, + transactions, + clock, + properties.leaseDuration(), + properties.claimBatchSize(), + properties.maxAdditionalAttempts()), + properties.leaseDuration(), + properties.shutdownGrace()); + workers.start(); + return workers; + } + + /** + * The dispatch scheduler. + * + *

The owner carries a per-process nonce, not just a configured name. Every instance used to + * call itself {@code notification-worker-1}, so a restarted process and the one it replaced were + * the same owner as far as the database was concerned — and a stale process's release could clear + * the lease its successor was working under. The nonce makes each incarnation distinguishable; + * the lease fence handles the rest. + * + * @param leases the recipient lease store + * @param dispatcher the dispatch service + * @param transactions opens the claim's write transaction and the backlog's read + * @param metrics the metrics port + * @param properties the dispatch bounds + * @param servingState the serving-state port + * @return the started scheduler worker + */ + @Bean(destroyMethod = "close") + NotificationSchedulerWorker notificationSchedulerWorker( + RecipientLeaseStorePort leases, + NotificationDispatchService dispatcher, + TransactionPort transactions, + NotificationMetricsPort metrics, + NotificationDispatchProperties properties, + NotificationServingStatePort servingState) { + String owner = "notification-worker-1:" + UUID.randomUUID(); + NotificationSchedulerWorker worker = + new NotificationSchedulerWorker( + new ClaimDueDeliveriesUseCase(leases, transactions), + new DispatchClaimedDeliveryUseCase(dispatcher), + metrics, + properties, + new MeasureDispatchBacklogUseCase(servingState, transactions), + owner); + worker.start(); + return worker; + } +} diff --git a/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml b/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml index 1aaac56e..1223ee20 100644 --- a/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml +++ b/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml @@ -21,16 +21,20 @@ info: enabled: ${app.notification.platform.enabled:true} app: - # The notification platform's own two permissions, granted to two roles rather than one. - # Publishing a template changes what every future submission renders, so a service that may send - # notifications is not thereby a service that may rewrite them; a deployment that wants both - # grants both roles. Here rather than in app-bootstrap's security.yml (07-08): the mapping goes - # when this module's line is removed. Map entries from every source merge, so a deployment's own - # app.authz.role-permissions adds roles beside these and can override either key. + # The notification platform's permissions, granted to three roles. Sending and publishing a + # template are two roles rather than one: publishing changes what every future submission + # renders, so a service that may send notifications is not thereby a service that may rewrite + # them; a deployment that wants both grants both roles. The operator plane's four permissions go + # to one operator role; the operator's authorities and tenant scope are checked again inside each + # use case, on the AdminActor the transport resolves. Here rather than in app-bootstrap's + # security.yml (07-08): the mapping goes when this module's line is removed. Map entries from + # every source merge, so a deployment's own app.authz.role-permissions adds roles beside these + # and can override any of the three keys. authz: role-permissions: notification-submit: notification:submit notification-template-publish: notification-template:publish + notification-admin: notification-admin:redrive,notification-admin:reconcile,notification-admin:suppress,notification-admin:provider-control notification: platform: # The capability switch. False binds nothing at all. @@ -57,6 +61,15 @@ app: # An ambiguous attempt means the request reached the provider and the outcome could not be # read. Falling back would risk sending the same notification twice, so it is off. allow-ambiguous-fallback: ${APP_NOTIFICATION_PLATFORM_ALLOW_AMBIGUOUS_FALLBACK:false} + # How long a failed delivery waits before its next attempt (full jitter between base and max; a + # provider's Retry-After counts up to max), and how long after an ambiguous submission + # reconciliation first asks the provider. Tune them against the provider's own rate limits. + # Left unset here: the shipped timing (1s, 5m, 1m) lives in NotificationPlatformSettings.Retry + # alone. Override with + # retry: + # backoff-base: 1s + # backoff-max: 5m + # reconcile-delay: 1m # Readiness is deployment-specific: throughput and operational SLOs determine what backlog # is acceptable. These defaults preserve the original behavior but are no longer hard-coded. serving-thresholds: diff --git a/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationModuleDefaultsTest.java b/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationModuleDefaultsTest.java index a4aa0d25..87d7f94d 100644 --- a/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationModuleDefaultsTest.java +++ b/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationModuleDefaultsTest.java @@ -6,6 +6,7 @@ import dev.caskeleton.adapter.inbound.web.notification.platform.callback.Notific import dev.caskeleton.adapter.outbound.notification.platform.provider.ses.SesCallbackAdapter; import dev.caskeleton.adapter.outbound.notification.platform.security.AesGcmCallbackPayloadProtection; import dev.caskeleton.adapter.outbound.support.startup.ModuleDefaultsEnvironmentPostProcessor; +import dev.caskeleton.application.notification.platform.admin.NotificationAdminPermissions; import java.time.Duration; import java.util.List; import java.util.Map; @@ -113,11 +114,20 @@ class NotificationModuleDefaultsTest { List permissions = List.of( environment.getProperty("app.authz.role-permissions.notification-submit", ""), - environment.getProperty( - "app.authz.role-permissions.notification-template-publish", "")); + environment.getProperty("app.authz.role-permissions.notification-template-publish", ""), + environment.getProperty("app.authz.role-permissions.notification-admin", "")); // then - assertThat(permissions).containsExactly("notification:submit", "notification-template:publish"); + assertThat(permissions) + .containsExactly( + "notification:submit", + "notification-template:publish", + String.join( + ",", + NotificationAdminPermissions.REDRIVE, + NotificationAdminPermissions.RECONCILE, + NotificationAdminPermissions.SUPPRESS, + NotificationAdminPermissions.PROVIDER_CONTROL)); } private static StandardEnvironment processed(Map deployment) { diff --git a/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationWorkerLifecycleTest.java b/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationWorkerLifecycleTest.java new file mode 100644 index 00000000..90de9d87 --- /dev/null +++ b/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationWorkerLifecycleTest.java @@ -0,0 +1,111 @@ +package dev.caskeleton.composition.notification; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.application.security.SystemInvoked; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; + +/** + * {@code INGEST_ONLY} starts no dispatch, recovery or reconciliation thread (NTF-INT-003). + * + *

The worker configuration once carried the master switch and nothing else, and both of its + * worker beans called {@code start()} in the factory method. So a deployment in {@code INGEST_ONLY} + * — the mode whose entire purpose is to accept and store without delivering — started the dispatch + * scheduler, the lease recovery pass, the provider-event replay worker and the reconciliation job. + * + *

The gate is structural rather than a condition repeated per bean: the two started beans live + * in one configuration that carries the mode condition. Asserted on the configuration's own + * annotations rather than by booting the platform: a full notification context needs a database, + * provider credentials and a secrets resolver. The live thread count is asserted by the {@code + * local-notification-ingest} Compose lane. + */ +class NotificationWorkerLifecycleTest { + + /** The bean methods that start something, and therefore may not exist outside SERVING. */ + private static final List THREAD_STARTING_BEANS = + List.of("notificationBackgroundWorkers", "notificationSchedulerWorker"); + + @Test + @DisplayName("스레드를 시작하는 worker bean은 모두 SERVING 모드 조건이 걸린 configuration에 있다") + void everyThreadStartingBeanIsServingOnly() { + // given + ConditionalOnProperty condition = + NotificationWorkerConfiguration.class.getAnnotation(ConditionalOnProperty.class); + + // when + List declared = + Arrays.stream(NotificationWorkerConfiguration.class.getDeclaredMethods()) + .filter(method -> method.isAnnotationPresent(Bean.class)) + .map(Method::getName) + .sorted() + .toList(); + + // then + assertThat( + List.of( + declared, + condition.prefix(), + List.of(condition.name()), + condition.havingValue(), + condition.matchIfMissing())) + .as( + "SERVING is the shipped default, so an absent mode must still deliver; a worker bean " + + "outside this configuration would run in INGEST_ONLY too") + .containsExactly( + THREAD_STARTING_BEANS.stream().sorted().toList(), + "app.notification.platform", + List.of("mode"), + "SERVING", + true); + } + + @Test + @DisplayName("worker configuration 밖의 조립은 SERVING 조건이 없어 INGEST_ONLY에서도 받는다") + void theIngestSideStaysAvailableInBothModes() { + // given + Import imports = + NotificationApplicationCompositionAutoConfiguration.class.getAnnotation(Import.class); + + // when + List gatedOnTheMode = + Arrays.stream(imports.value()) + .filter(type -> type.isAnnotationPresent(ConditionalOnProperty.class)) + .filter( + type -> + Arrays.asList(type.getAnnotation(ConditionalOnProperty.class).name()) + .contains("mode")) + .map(Class::getSimpleName) + .toList(); + + // then + assertThat(gatedOnTheMode) + .as( + "only the workers stop in INGEST_ONLY; submission, delivery services, callbacks and " + + "the operator plane are what an ingest-only deployment still serves") + .containsExactly("NotificationWorkerConfiguration"); + } + + @Test + @DisplayName("worker가 쥐는 시스템 쓰기 use case는 bean으로 내놓지 않는다") + void noSystemInvokedUseCaseIsABeanOfTheWorkerConfiguration() { + // when + List systemInvokedBeans = + Arrays.stream(NotificationWorkerConfiguration.class.getDeclaredMethods()) + .filter(method -> method.isAnnotationPresent(Bean.class)) + .filter(method -> method.getReturnType().isAnnotationPresent(SystemInvoked.class)) + .map(Method::getName) + .toList(); + + // then + assertThat(systemInvokedBeans) + .as("a bean is what a transport could reach, and these write with no permission check") + .isEmpty(); + } +} From 9fc881be2756c9859392372624385be38e4b299b Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 13 Nov 2026 18:41:55 +0900 Subject: [PATCH 7/8] =?UTF-8?q?test(persistence-jpa):=20=EB=8F=99=EC=8B=9C?= =?UTF-8?q?=20writer=20=EC=8B=9C=ED=97=98=EC=9D=B4=20replay=20=EC=88=9C?= =?UTF-8?q?=EC=84=9C=EB=A5=BC=20=EB=B0=9B=EC=9D=80=20=EC=9C=84=EC=B9=98=20?= =?UTF-8?q?=EC=88=9C=EC=84=9C=EB=A1=9C=20=EB=B3=B8=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 문제: LiveEventLogContractTest의 「두 writer 동시 쓰기」 시험은 두 writer를 latch 하나로 동시에 푼다. 위치는 순서 없이 1·2인지 보면서, replay는 from-first, from-second 순서로 단언한다. 「먼저 제출한 writer가 1번을 받는다」고 가정하는 셈이다. 어느 writer가 allocator에 먼저 닿는지는 스케줄러가 정한다. 그래서 부하 아래에서는 second가 1번을 받고 시험이 실패한다(main 검사 run 260, d32b4b53, 434건 중 1건). 2026-09-29부터 있던 흔들림이다. 고른 방법: 각 Future가 돌려준 위치를 키로 값을 TreeMap에 모은다. 키가 정확히 1·2인지 보고, replay 순서가 위치 순서로 정렬한 값과 같은지 본다. 이 시험이 지키는 성질은 「위치가 겹치지 않고 replay가 위치 순서를 따른다」 이다. 스레드 시작 순서는 지키는 성질이 아니다. 고친 메서드의 영어 @DisplayName은 테스트 작성 규칙대로 한국어로 바꿨다. 버린 대안: - latch를 둘로 나눠 writer 순서를 강제한다. 동시에 쓰는 경우를 더는 보지 않게 되어 이 시험의 목적이 사라진다. - replay를 순서 없이(containsExactlyInAnyOrder) 본다. replay가 위치 순서를 따르는지를 놓친다. 같은 모양 찾기: 백엔드 시험 source set 전체에서 동시에 일을 푸는 파일 (CountDownLatch·CyclicBarrier·Phaser·Executors·invokeAll·supplyAsync· runAsync)과 순서 단언(containsExactly·get(0)·getFirst)이 함께 있는 파일을 grep으로 골라 읽었다. 이 시험을 빼면 27개다. 모두 앞 단계를 latch로 기다려 순서를 강제하거나, 개수·집합으로 보거나, 한 스레드가 정한 순서를 본다. 스레드 시작 순서를 결과 순서로 단언하는 곳은 이 시험 하나였다. 위험·한계: 시험 코드만 바뀐다. production 동작은 그대로다. 찾기는 위 이름 목록에 기댄다. 다른 방식으로 스레드를 만드는 시험은 보지 않았다. 검증(desktop, --no-build-cache): - RED 탐침(main + second가 먼저 위치를 잡게 강제): jpaPlatformContractTest --tests '*LiveEventLogContractTest' rc=1. 실패는 replay 단언 하나이고 실제 값은 [from-second, from-first]다. main 검사 run 260과 같은 실패다. - 고친 시험 + 같은 강제 순서: rc=0(11건 실패 0). - 고친 시험 클래스 --rerun 5회: 모두 rc=0. - :adapter:outbound:persistence-jpa:jpaPlatformContractTest 필터 없이: rc=0(434건 실패 0). - scripts/verify.sh pr --no-cache: rc=0(트리 461ffa24, 6905건 실패 0, skip 35, FROM-CACHE 0). Refs: main 검사 #148(run 260, d32b4b53) --- .../liveevent/LiveEventLogContractTest.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventLogContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventLogContractTest.java index d847e9f4..bbe5a45b 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventLogContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventLogContractTest.java @@ -17,6 +17,8 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -87,23 +89,28 @@ class LiveEventLogContractTest { } @Test - @DisplayName("two concurrent writers receive different positions and both commit") + @DisplayName("두 writer가 동시에 쓰면 서로 다른 위치 1·2를 받고, replay는 받은 위치 순서대로 둘 다 돌려준다") void concurrentWritersAllocateDistinctPositions() throws Exception { + // given CountDownLatch start = new CountDownLatch(1); + // Which writer reaches the allocator first is up to the scheduler, so the expected replay + // order is the order of the positions the writers got back, not the order they were started. + Map payloadByPosition = new TreeMap<>(); + // when try (ExecutorService executor = Executors.newFixedThreadPool(2)) { Future first = executor.submit(() -> allocateAndAppendAfter(start, "orders", "from-first")); Future second = executor.submit(() -> allocateAndAppendAfter(start, "orders", "from-second")); - start.countDown(); - - assertThat(List.of(first.get(10, TimeUnit.SECONDS), second.get(10, TimeUnit.SECONDS))) - .containsExactlyInAnyOrder(1L, 2L); + payloadByPosition.put(first.get(10, TimeUnit.SECONDS), "from-first"); + payloadByPosition.put(second.get(10, TimeUnit.SECONDS), "from-second"); } - assertThat(replayAfter("orders", 0, NOW)).containsExactly("from-first", "from-second"); + // then + assertThat(payloadByPosition).containsOnlyKeys(1L, 2L); + assertThat(replayAfter("orders", 0, NOW)).containsExactlyElementsOf(payloadByPosition.values()); } @Test From a03cca738d2060212d593aab4c01ce39b8b41820 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 13 Nov 2026 21:13:57 +0900 Subject: [PATCH 8/8] =?UTF-8?q?test:=20#34=EA=B0=80=20=EB=82=A8=EA=B8=B4?= =?UTF-8?q?=20SpotBugs=20=EC=A7=80=EC=A0=81=EA=B3=BC=20=EB=B2=BD=EC=8B=9C?= =?UTF-8?q?=EA=B3=84=20=EC=83=81=ED=95=9C=20=EC=85=8B=EC=9D=84=20=EA=B3=A0?= =?UTF-8?q?=EC=B9=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit main 단계의 :adapter:inbound:graphql:spotbugsSystemTest가 #34의 `var unused = release.await(...)`를 DLS_DEAD_LOCAL_STORE로 잡아 main이 빨갛다. 이 태스크는 main 단계에만 있어 PR run에서 보이지 않았다. latch의 반환값을 쓰고, 풀려나지 않고 시한까지 간 호출 수를 센다. #34가 벽시계 시험을 고칠 때 httpclient 셋이 빠졌다. 셋 다 시한에 0.5초 여유를 둔 상한이라 부하 아래에서 원인과 상관없이 실패한다(Jetty 시험이 부하 11~15에서 1.68초 > 1.5초). #34와 같은 방식으로 고친다. - 재는 구간을 호출 하나로 좁힌다(준비 시간을 뺀다). - 상한은 시한 + 3초이고, 그 상한이 서버의 흘림·지연보다 작다는 것을 함께 단언한다(작지 않으면 상한이 아무것도 증명하지 못한다). - 늦은 헤더의 지연은 mock 서버의 종료 대기(5초)보다 짧게 둔다. Refs: RR9-03-36 --- .../runtime/RuntimeLimitsTestSupport.java | 9 +++++- .../http3/JettyAttemptDeadlineTest.java | 28 +++++++++++++++---- .../AttemptDeadlineEnforcementTest.java | 22 +++++++++++++-- .../StreamingSetupDeadlineTest.java | 18 ++++++++++-- 4 files changed, 65 insertions(+), 12 deletions(-) diff --git a/src/adapter/inbound/graphql/src/systemTest/java/dev/caskeleton/adapter/inbound/graphql/runtime/RuntimeLimitsTestSupport.java b/src/adapter/inbound/graphql/src/systemTest/java/dev/caskeleton/adapter/inbound/graphql/runtime/RuntimeLimitsTestSupport.java index 00b48c15..0956d4af 100644 --- a/src/adapter/inbound/graphql/src/systemTest/java/dev/caskeleton/adapter/inbound/graphql/runtime/RuntimeLimitsTestSupport.java +++ b/src/adapter/inbound/graphql/src/systemTest/java/dev/caskeleton/adapter/inbound/graphql/runtime/RuntimeLimitsTestSupport.java @@ -91,6 +91,10 @@ final class RuntimeLimitsTestSupport { static final AtomicInteger SLOW_STARTED = new AtomicInteger(); static final AtomicInteger DETAIL_INVOCATIONS = new AtomicInteger(); + + /** How many {@code slowNode} calls ran to their {@code millis} cap without being released. */ + static final AtomicInteger SLOW_CAPPED = new AtomicInteger(); + static volatile CountDownLatch slowFinished = new CountDownLatch(1); /** @@ -114,6 +118,7 @@ final class RuntimeLimitsTestSupport { static void reset() { SLOW_STARTED.set(0); DETAIL_INVOCATIONS.set(0); + SLOW_CAPPED.set(0); slowFinished = new CountDownLatch(1); slowExecutionEnded = new CountDownLatch(1); release = new CountDownLatch(1); @@ -153,7 +158,9 @@ final class RuntimeLimitsTestSupport { CountDownLatch release = SlowWork.release; try { // Released by the test, or the cap ran out; either way the resolver returns normally. - var unused = release.await(millis, TimeUnit.MILLISECONDS); + if (!release.await(millis, TimeUnit.MILLISECONDS)) { + SlowWork.SLOW_CAPPED.incrementAndGet(); + } } finally { SlowWork.slowFinished.countDown(); } diff --git a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/http3/JettyAttemptDeadlineTest.java b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/http3/JettyAttemptDeadlineTest.java index 1b81c012..1725fdaa 100644 --- a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/http3/JettyAttemptDeadlineTest.java +++ b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/http3/JettyAttemptDeadlineTest.java @@ -28,7 +28,18 @@ import org.springframework.http.client.JettyClientHttpRequestFactory; class JettyAttemptDeadlineTest { private static final Duration DEADLINE = Duration.ofSeconds(1); - private static final Duration TOLERANCE = Duration.ofMillis(500); + private static final int DRIP_BYTES = 200; + private static final Duration DRIP_GAP = Duration.ofMillis(50); + + /** How long the drip alone takes to deliver the body: 200 bytes, 50 ms apart (10 s). */ + private static final Duration DRIP_TOTAL = DRIP_GAP.multipliedBy(DRIP_BYTES); + + /** + * Time a loaded machine may add between the deadline firing and the caller seeing the failure. + * The bound only has to tell "cut at the deadline" from "waited for the drip", which the total + * leaves room for (deadline + slack is under half of it). + */ + private static final Duration SCHEDULING_SLACK = Duration.ofSeconds(3); @Test @DisplayName("Jetty 클라이언트는 바이트를 조금씩 흘리는 응답이라도 attempt deadline에 교환을 끊는다") @@ -39,13 +50,14 @@ class JettyAttemptDeadlineTest { client.start(); try (MockHttpServer server = MockHttpServer.start()) { // given - server.enqueueSlowDripBody(200, 100, Duration.ofMillis(50)); + server.enqueueSlowDripBody(200, DRIP_BYTES, DRIP_GAP); JettyClientHttpRequestFactory factory = new JettyClientHttpRequestFactory(client); - long started = System.nanoTime(); // when Throwable failure; + Duration elapsed; try (AttemptDeadline.Scope scope = AttemptDeadline.open(DEADLINE)) { + long started = System.nanoTime(); failure = catchThrowable( () -> { @@ -54,12 +66,18 @@ class JettyAttemptDeadlineTest { response.getBody().readAllBytes(); } }); + // The exchange alone: client and server setup are not part of the deadline. + elapsed = Duration.ofNanos(System.nanoTime() - started); } // then - Duration elapsed = Duration.ofNanos(System.nanoTime() - started); assertThat(failure).isInstanceOf(IOException.class); - assertThat(elapsed).isLessThan(DEADLINE.plus(TOLERANCE)); + assertThat(DEADLINE.plus(SCHEDULING_SLACK)) + .as("the bound must stay well under the drip, or it proves nothing") + .isLessThan(DRIP_TOTAL.dividedBy(2)); + assertThat(elapsed) + .as("the deadline, not the drip, ended the exchange") + .isLessThan(DEADLINE.plus(SCHEDULING_SLACK)); } finally { client.stop(); } diff --git a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/AttemptDeadlineEnforcementTest.java b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/AttemptDeadlineEnforcementTest.java index 02e0d264..75d83d41 100644 --- a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/AttemptDeadlineEnforcementTest.java +++ b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/AttemptDeadlineEnforcementTest.java @@ -28,7 +28,18 @@ import org.junit.jupiter.params.provider.EnumSource; class AttemptDeadlineEnforcementTest { private static final Duration TOTAL_CALL = Duration.ofSeconds(1); - private static final Duration TOLERANCE = Duration.ofMillis(500); + private static final int DRIP_BYTES = 200; + private static final Duration DRIP_GAP = Duration.ofMillis(50); + + /** How long the drip alone takes to deliver the body: 200 bytes, 50 ms apart (10 s). */ + private static final Duration DRIP_TOTAL = DRIP_GAP.multipliedBy(DRIP_BYTES); + + /** + * Time a loaded machine may add between the deadline firing and the caller seeing the failure. + * The bound only has to tell "cut at the deadline" from "waited for the server", which the + * server's delay leaves room for (deadline + slack is under half of it). + */ + private static final Duration SCHEDULING_SLACK = Duration.ofSeconds(3); @ParameterizedTest(name = "{0}") @EnumSource( @@ -45,7 +56,7 @@ class AttemptDeadlineEnforcementTest { .timeout(withTotalCall(TOTAL_CALL)) .build())) { // given - server.enqueueSlowDripBody(200, 100, Duration.ofMillis(50)); + server.enqueueSlowDripBody(200, DRIP_BYTES, DRIP_GAP); long started = System.nanoTime(); // when @@ -62,7 +73,12 @@ class AttemptDeadlineEnforcementTest { // then Duration elapsed = Duration.ofNanos(System.nanoTime() - started); assertThat(failure).isInstanceOf(HttpDeadlineExceededException.class); - assertThat(elapsed).isLessThan(TOTAL_CALL.plus(TOLERANCE)); + assertThat(TOTAL_CALL.plus(SCHEDULING_SLACK)) + .as("the bound must stay well under the drip, or it proves nothing") + .isLessThan(DRIP_TOTAL.dividedBy(2)); + assertThat(elapsed) + .as("the deadline, not the drip, ended the call") + .isLessThan(TOTAL_CALL.plus(SCHEDULING_SLACK)); } } diff --git a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/StreamingSetupDeadlineTest.java b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/StreamingSetupDeadlineTest.java index 9c85615e..5179dbf9 100644 --- a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/StreamingSetupDeadlineTest.java +++ b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/StreamingSetupDeadlineTest.java @@ -37,8 +37,15 @@ import org.junit.jupiter.params.provider.EnumSource; class StreamingSetupDeadlineTest { private static final Duration TOTAL_CALL = Duration.ofSeconds(1); - private static final Duration HEADERS_DELAY = Duration.ofMillis(1800); - private static final Duration TOLERANCE = Duration.ofMillis(500); + // Under the mock server's 5 s shutdown wait: it sleeps the delay out after the client gives up. + private static final Duration HEADERS_DELAY = Duration.ofSeconds(5); + + /** + * Time a loaded machine may add between the deadline firing and the caller seeing the failure. + * The bound only has to tell "cut at the deadline" from "waited for the late headers", which the + * delay leaves room for (deadline + slack is under it). + */ + private static final Duration SCHEDULING_SLACK = Duration.ofSeconds(3); @ParameterizedTest(name = "{0}") @EnumSource( @@ -70,7 +77,12 @@ class StreamingSetupDeadlineTest { // then Duration elapsed = Duration.ofNanos(System.nanoTime() - started); assertThat(failure).isInstanceOf(HttpDeadlineExceededException.class); - assertThat(elapsed).isLessThan(TOTAL_CALL.plus(TOLERANCE)); + assertThat(TOTAL_CALL.plus(SCHEDULING_SLACK)) + .as("the bound must stay under the headers delay, or it proves nothing") + .isLessThan(HEADERS_DELAY); + assertThat(elapsed) + .as("the deadline, not the late headers, ended the setup") + .isLessThan(TOTAL_CALL.plus(SCHEDULING_SLACK)); } }