diff --git a/.github/workflows/ci-quality-gates.yml b/.github/workflows/ci-quality-gates.yml index 6c25d1a3..83986f9e 100644 --- a/.github/workflows/ci-quality-gates.yml +++ b/.github/workflows/ci-quality-gates.yml @@ -65,9 +65,12 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # actions/checkout@v4.2.2 - uses: ./.github/actions/setup-gradle-java + # `qualityCheck` too: the root one covers the production leaves only, so the sample — the code + # adopters copy first — was analysed by no stage, and its SpotBugs exclusions matched nothing + # (review 2026-11-03 RR-06-05). - name: Verify the standalone sample application working-directory: src - run: ./gradlew :sample-portfolio:check --stacktrace + run: ./gradlew :sample-portfolio:check :sample-portfolio:qualityCheck --stacktrace optional-platforms: runs-on: ubuntu-latest @@ -77,6 +80,11 @@ jobs: - name: Verify the optional gRPC platform build working-directory: src run: ./gradlew -p optional-platforms ci --stacktrace + # The optional build's own lane-reachability check: the main build's cannot see these lanes + # (review 2026-11-03 RR-06-03). + - name: Require every optional-platform strict lane to be in an executed stage + working-directory: src + run: ./gradlew -p optional-platforms :verifyStrictLaneStages --dry-run --stacktrace # gRPC schema policy (owner decision 18). protoc compiles src/grpc/grpc-contract inside the # optional-platforms build above; lint and breaking-change policy belong to Buf, configured by diff --git a/.github/workflows/integration-main.yml b/.github/workflows/integration-main.yml index 79090826..7bd70230 100644 --- a/.github/workflows/integration-main.yml +++ b/.github/workflows/integration-main.yml @@ -203,6 +203,12 @@ jobs: - name: Run every performance-category strict lane working-directory: src run: ./gradlew :performanceCheck --continue --stacktrace + # The optional gRPC build's performance lanes (grpcPerformanceTest). A separate build, so a + # separate command; its own verifyStrictLaneStages counts this line (review 2026-11-03 RR-06-03). + - name: Run every optional-platform performance-category strict lane + if: always() + working-directory: src + run: ./gradlew -p optional-platforms performanceCheck --continue --stacktrace - name: Publish the performance lane reports if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4.6.2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 43d2ff14..8cdecab9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -120,9 +120,10 @@ jobs: # container and no fixed cadence — grpcNettyContractTest opens an ephemeral socket, not a broker — # so the guard is worth asserting once per tag and is not worth a runner every night. # - # grpcPerformanceTest is deliberately absent. The leaf excludes it from `test` and says why: "a - # measurement in the release gate is a flaky test on a shared CI runner; it runs when somebody asks - # for it, by name." It is recorded as a manual entrypoint in the repository README instead. + # The optional build's `releaseCheck` is those hermetic lanes, so a lane added there is here + # without a workflow edit. grpcPerformanceTest is deliberately absent: a measurement is not release + # evidence on a shared runner. It runs in integration-main.yml's performance-check job (the + # optional build's `performanceCheck`). grpc-stable-release-gate: runs-on: ubuntu-latest timeout-minutes: 45 @@ -131,12 +132,7 @@ jobs: - uses: ./.github/actions/setup-gradle-java - name: Run the gRPC Stable certification lanes working-directory: src - run: >- - ./gradlew -p optional-platforms - :grpc:grpc-testkit:grpcInProcessContractTest - :grpc:grpc-testkit:grpcNettyContractTest - :grpc:grpc-testkit:grpcFaultTest - --stacktrace + run: ./gradlew -p optional-platforms releaseCheck --stacktrace # The operation ledger's real-PostgreSQL lane ran only on main and nightly # (integration-main.yml), so a tag could be cut from a main whose ledger lane was red. It needs # Docker, which a release runner has (review 2026-11-01 03-12). diff --git a/AGENTS.md b/AGENTS.md index 27b27f35..0b68af7d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,10 +77,13 @@ root `CLAUDE.md`는 이 목록의 동기화된 요약이다. 두 문서가 어 `ca.config-contract`다. `src/build-tools/` included build는 비워서 삭제했다. - `src/build.gradle`: 루트 라이프사이클(`ci`, `mainCheck`, `integrationCheck`, `performanceCheck`, `releaseCheck`, `qualityCheck`, `configContractCheck`)과 버전/리비전. `verifyStrictLaneStages - --dry-run`(PR 게이트)은 `explicitOnly`가 아닌 모든 strict lane이 workflow가 실제로 실행하는 단계 - (`ci`·`sample-portfolio:check`·`mainCheck`·`integrationCheck`·`performanceCheck`·`releaseCheck`) - 중 하나에 들어 있는지, 그리고 workflow의 접힌(`>`) run 블록이 빈 줄 때문에 옵션을 별개 명령으로 - 떼어 내지 않는지 검사한다. + --dry-run`(PR 게이트)은 그 빌드에서 `explicitOnly`가 아닌 strict lane이 workflow가 실제로 실행하는 + 단계(main 빌드: `ci`·`sample-portfolio:check`·`mainCheck`·`integrationCheck`·`performanceCheck`· + `releaseCheck`, optional-platforms 빌드: `-p optional-platforms`로 부르는 `ci`·`integrationCheck`· + `performanceCheck`·`releaseCheck`) 중 하나에 들어 있는지, 그리고 workflow의 접힌(`>`) run 블록이 빈 + 줄 때문에 옵션을 별개 명령으로 떼어 내지 않는지 검사한다. 두 빌드가 각자 검사를 돌린다. `-x`로 뺀 + lane, `continue-on-error` 아래 명령, 상태 함수(`always()`·`success()`·`!cancelled()`)가 아닌 `if:` + 아래 명령은 실행으로 세지 않는다. 작업 파일의 소유 leaf는 registry의 `source_path`로 판단하고 가장 가까운 `src/**/CLAUDE.md`를 함께 읽는다. focused test는 registry의 `gradle_path`에서 @@ -202,10 +205,10 @@ cd src ./gradlew architectureCheck # 의존 방향·domain-core/shared-contract/application-* 순수성·ArchUnit ./gradlew qualityCheck # SpotBugs + FindSecBugs (leaf check에는 없다) ./gradlew ci # PR 게이트 = leaf check + architecture + quality -./gradlew mainCheck # main 단계 = env 계약·전체 기동·public path·전송/메시징 레인·leaf 문서 대조 레인 +./gradlew mainCheck # main 단계 = env 계약·전체 기동(샘플 포함)·public path·전송/메시징 레인·leaf 문서 대조 레인·기능 생성 왕복(verifyFeatureScaffold) ./gradlew integrationCheck # 실제 컨테이너 integration 레인 전체(main·nightly, Docker 필요) ./gradlew performanceCheck # 부하·종료·자원 한계 레인(main·nightly) -./gradlew verifyStrictLaneStages --dry-run # 모든 strict lane이 workflow가 실행하는 단계에 속하는지 +./gradlew verifyStrictLaneStages --dry-run # 이 빌드의 strict lane이 workflow가 실행하는 단계에 속하는지(optional 빌드는 -p optional-platforms로 따로) ./gradlew verifyPublicPathSnapshot ./gradlew :app-bootstrap:verifyEnvKeys ``` diff --git a/CLAUDE.md b/CLAUDE.md index d749111f..facf5c12 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -38,8 +38,10 @@ If this summary drifts from `AGENTS.md`, `AGENTS.md` wins and this summary must `ca.config-contract`. The `src/build-tools/` included build was emptied and removed. - `src/build.gradle`: the root lifecycle (`ci`, `mainCheck`, `integrationCheck`, `performanceCheck`, `releaseCheck` and the named repository-wide gates). `verifyStrictLaneStages --dry-run` (pull - request) fails when a non-`explicitOnly` strict lane is in no stage a workflow executes, or a - folded workflow `run:` block splits an option off its command. + request; the main build and, as `-p optional-platforms`, the optional build each run their own) + fails when a non-`explicitOnly` strict lane is in no stage a workflow executes — a lane the + command removes with `-x` and a command under `continue-on-error` or a non-status `if:` do not + count — or a folded workflow `run:` block splits an option off its command. Commit policy is `human-only`: agents do not stage, commit, amend, or push implementation changes. @@ -132,9 +134,11 @@ declared strict integration lanes; a lane that needs an input the build was not and named), `performanceCheck` (the declared performance lanes; main and nightly). `ci` (pull request) is leaf checks (API surfaces included, owner decision 20) + architectureCheck + qualityCheck, and the workflow also runs `verifyStrictLaneStages --dry-run`; `mainCheck` (main branch) is configContractCheck + -qualificationCheck + the sample web contract + the public-path snapshot + the transport and -messaging qualification lanes + every leaf's `mainCheck` (document-contract lanes, MinIO lanes); -`releaseCheck` is the tag-only checks: release provenance and the public-path snapshot. +qualificationCheck + the sample web contract + the sample full boot (`:sample-portfolio:systemTest`) ++ the public-path snapshot + the transport and messaging qualification lanes + the feature +generator round trip (`verifyFeatureScaffold`) + every leaf's `mainCheck` (document-contract lanes +and any lane the leaf adds). `releaseCheck` is the tag-only checks: release provenance and the +public-path snapshot. Test-writing rules (behavior-sentence names that assert what they promise, `@DisplayName` in Korean only when the name cannot say it (owner decision 5), given/when/then comments, one `// when`, one diff --git a/docker-compose.infra.yml b/docker-compose.infra.yml index 93de6c85..4ae89ea3 100644 --- a/docker-compose.infra.yml +++ b/docker-compose.infra.yml @@ -137,10 +137,12 @@ services: start_period: 30s # ---- Redis — local compose contract for the shipped Redis runtime ---------------- - # Authenticated, with the accounts a deployment loads: infra/redis-sdk/acl/all-accounts.acl, the - # same file the Redis topology lanes use, mounted as it is rather than copied. Its first line turns - # the passwordless `default` user off, so the application connects as the named accounts the lane - # contract gives it (compose-profile-contracts.json, $redisComment). + # Authenticated the way a deployment authenticates, with the lanes' test-fixture accounts: + # infra/redis-sdk/acl/lane-fixture-accounts.acl, the same file the Redis topology lanes use, mounted + # as it is rather than copied. It is not a deployment file — its passwords are public and it carries + # a lane-only `+@all` account (infra/redis-sdk/acl/README.md). Its first line turns the passwordless + # `default` user off, so the application connects as the named accounts the lane contract gives it + # (compose-profile-contracts.json, $redisComment). # # This Redis was open and the lanes declared app.redis.authentication.anonymous-access-accepted. # The application then logged that trade, and the missing advanced account, as two startup @@ -162,7 +164,7 @@ services: - --save - "" - --aclfile - - /etc/redis/acl/all-accounts.acl + - /etc/redis/acl/lane-fixture-accounts.acl volumes: - type: bind source: ./infra/redis-sdk/acl diff --git a/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md b/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md index 495d3480..7307c654 100644 --- a/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md +++ b/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md @@ -69,6 +69,36 @@ method can run longer than any fixed TTL; revisit when one exists. A sweeper job rows — it needs a scheduler this family does not have, and deleting a claim makes the next retry a fresh claim with no record that an attempt was abandoned. +**A taken-over owner is fenced by a claim token** (re-review 2026-11-03, RR-02-04). The owner a +takeover presumes dead may only be slow. Without knowing which attempt held the claim, its late +`markFailed` turned the new attempt's claim `FAILED_TERMINAL`, the next retry re-claimed it, and two +attempts ran the mutation at once; its late `markCommitted` committed beside the new attempt. Every +claim now carries a `GrpcClaimToken` (`claim_attempt`, migration V003): 1 for the first claim, one more +for each `reclaimFailed` or `reclaimStale`. `decide` returns it on `PROCEED`, and `recordCommit` / +`recordFailure` hand it back. A commit or a failure is one conditional +`UPDATE … WHERE state = 'IN_PROGRESS' AND claim_attempt = :attempt`: a commit by an attempt that lost +the claim is refused with an exception, so its business transaction rolls back; a failure by one is +ignored (`markFailed` returns false). The same conditional update closes the JPA commit race — +load-check-save let two commits that both read `IN_PROGRESS` both write; now exactly one matches. +A claim whose conflicting row is purged between insert and read retries the insert once instead of +failing the call. + +Chosen over the claim moment as the token (`WHERE claimed_at = :claimedAt`), which needs no column: +PostgreSQL stores microseconds while `Instant` carries nanoseconds, so the token a caller holds and +the one stored can differ by rounding, and two processes' clocks can give two attempts the same +moment. The counter has neither problem. The heartbeat lease above stays rejected for the same reason +as before; the token fences without one. + +**What `operation-ledger-enabled=true` promises (03-11 (b), recorded 2026-11-03, RR-02-07).** The +2026-11-01 review offered (a) delete the flag and judge by whether a ledger bean exists, or (b) keep the +flag and have the starter assemble the decision function from it. (b) is kept: the assembly and the +TTL validation live in one place instead of in every adopter. (a) was rejected because it removes the +false promise only by removing the assembly with it. The limit stays and is stated where an operator +meets it: the starter builds no server, so the flag assembles the decision function only, and keys are +honoured once the adopter's `IDEMPOTENCY` stage interceptor or service adapter calls `decide` and +`recordCommit`. `GrpcPlatformStartupValidator`'s refusal for a keyed method says exactly that; a running +process cannot see whether the stage exists, so the validator does not claim to check it. + **Finished rows are purged after a retention; in-progress rows never are** (03-10 follow-up). `GrpcOperationLedger.purgeCompletedBefore(cutoff, limit)` deletes at most `limit` `COMMITTED` or `FAILED_TERMINAL` rows that completed before the cutoff, oldest first — on PostgreSQL one bounded diff --git a/docs/capability-status.md b/docs/capability-status.md index ab9f8972..7b701ff0 100644 --- a/docs/capability-status.md +++ b/docs/capability-status.md @@ -30,14 +30,14 @@ lanes and CI; it is not a deployment artifact. | Relational persistence (JPA/PostgreSQL) | `ca-skeleton.persistence-jpa.enabled` / `APP_PERSISTENCE_JPA_ENABLED`, `true` | yes | yes (one line: `runtimeOnly project(':adapter:outbound:persistence-jpa')`) | yes | Not in the default composition. The one line brings everything — its own `AutoConfiguration.imports` (`PersistenceJpaAutoConfiguration`, `PersistenceJpaRuntimeConfig`, `JpaStartupAutoConfiguration`), its environment handling (`JpaActivationEnvironmentPostProcessor`, `spring.factories`: readiness `db` while on; with `false` it also excludes Boot's JDBC, Hibernate, Spring Data JPA and Flyway auto-configuration, so no DataSource is bound), the startup guards (Flyway strategy, datasource/OSIV/DDL/pool/TLS/vendor checks, exit codes unchanged) and its settings (`config/module-defaults/persistence-jpa.yml`). `persistence-jpa:PersistenceJpaCapabilityAssemblyTest` proves the module assembles on its own; the `postgresql*IntegrationTest` readiness lanes, run by `integration-main.yml` → `integration-check` (`:integrationCheck`), qualify it against PostgreSQL. `sample-portfolio` selects it by its build dependency and the module's auto-configuration (no broad scan of the persistence package). `bootRunH2` adds the module to its own classpath only. | | Document persistence (MongoDB) | `ca-skeleton.persistence-mongo.enabled` / `APP_PERSISTENCE_MONGO_ENABLED`, `true` | yes | yes (one line: `runtimeOnly project(':adapter:outbound:persistence-mongo')`) | yes | On when present; `false` also excludes Boot's Mongo auto-configuration (`MongoCapabilityActivationTest`). The connection-source rule (`SPRING_MONGODB_URI` or the selected profile's `uri-secret`) is checked by the module's own `MongoDefaultsEnvironmentPostProcessor`. Six container lanes (`mongoReplicaSetTest`, `mongoFailoverTest`, …), run by `integration-main.yml` → `integration-check` (`:integrationCheck`) | | Messaging (Kafka) | `app.messaging.enabled` / `APP_MESSAGING_ENABLED`, `true` | yes | yes (one line: `runtimeOnly project(':adapter:outbound:messaging-kafka')`, which brings the application bridge, the legacy `KafkaSender` seam and the vendored platform with its Kafka transport) | Kafka only | On when present (`KafkaCapabilityActivationTest`). `messaging-kafka:messagingCertificationTest` behind `verifyMessagingCertificationEvidence`, run by `messaging-certification.yml`; the starter's `integrationTest` is the live Kafka round trip. Kafka is the only registered broker; the RabbitMQ broker keys were removed (no transport). | -| Redis capabilities (cache / rate-limit / lease / idempotency) | `app.redis.enabled` / `APP_REDIS_ENABLED`, `true` | yes | yes (one line: `runtimeOnly project(':adapter:outbound:cache-redis')`, which brings `redis-runtime`) | yes | `cache-redis:redisContainerContractTest` (Testcontainers Redis, `integrationCheck` and `mainCheck`); `cache-redis:redisTopologyTest`, run by `redis-sdk-topology.yml`. Not in the default composition. On when present; `false` contributes no Redis bean — no settings, client, connection, thread or health contributor. The module owns its readiness contribution (`RedisReadinessGroupPostProcessor`, `spring.factories`) and its settings (`config/module-defaults/cache-redis.yml`: `app.redis.enabled`, `app.rate-limit.*`, the Redis `ca-skeleton.capabilities.*`). No `app-bootstrap` main source names a Redis type. | -| Request idempotency | `APP_IDEMPOTENCY_PROVIDER` (`disabled` \| `redis` \| `postgresql`; a selector, not an on/off key) | yes | yes | Redis only | One generation: the application `IdempotencyExecutor`, built once by `app-bootstrap` `IdempotencyProviderSelectionConfig` over the one store the selected provider contributes (`cache-redis` or `persistence-jpa`); startup is refused unless exactly one store exists. `sample-portfolio` cannot depend on `app-bootstrap`, so its own `SampleIdempotencyConfig` builds its one executor from the same keys. Redis is covered by the row above. PostgreSQL has `persistence-jpa:postgresqlIdempotencyIntegrationTest`, but no workflow runs it. The `postgresql` provider refuses a non-PostgreSQL data source, so `bootRunH2` uses `disabled` or `redis`.. The executor stays in the composition root on purpose — it needs only application-core and its settings — while each store comes with its module: `postgresql` needs the persistence-jpa line, `redis` the cache-redis one. | +| Redis capabilities (cache / rate-limit / lease / idempotency) | `app.redis.enabled` / `APP_REDIS_ENABLED`, `true` | yes | yes (one line: `runtimeOnly project(':adapter:outbound:cache-redis')`, which brings `redis-runtime`) | yes | `cache-redis:redisContainerContractTest` (Testcontainers Redis, `integrationCheck`); `cache-redis:redisTopologyTest`, run by `redis-sdk-topology.yml`. Not in the default composition. On when present; `false` contributes no Redis bean — no settings, client, connection, thread or health contributor. The module owns its readiness contribution (`RedisReadinessGroupPostProcessor`, `spring.factories`) and its settings (`config/module-defaults/cache-redis.yml`: `app.redis.enabled`, `app.rate-limit.*`, the Redis `ca-skeleton.capabilities.*`). No `app-bootstrap` main source names a Redis type. | +| Request idempotency | `APP_IDEMPOTENCY_PROVIDER` (`disabled` \| `redis` \| `postgresql`; a selector, not an on/off key) | yes | yes | Redis and PostgreSQL | One generation: the application `IdempotencyExecutor`, built once by `composition:idempotency` (`IdempotencyAutoConfiguration`) over the one store the selected provider contributes (`cache-redis` or `persistence-jpa`); startup is refused unless exactly one store exists. `app-bootstrap` and `sample-portfolio` both take that module with one `runtimeOnly` line, so there is no second assembly. Redis is covered by the row above. PostgreSQL has `persistence-jpa:postgresqlIdempotencyIntegrationTest`, run by `integration-main.yml` → `integration-check` (`:integrationCheck`). The `postgresql` provider refuses a non-PostgreSQL data source, so `bootRunH2` uses `disabled` or `redis`. Each store comes with its module: `postgresql` needs the persistence-jpa line, `redis` the cache-redis one. | | Transactional outbox relay | `ca-skeleton.outbox.enabled` / `APP_OUTBOX_ENABLED`, `false` (+ `APP_OUTBOX_RELAY_ENABLED`, `false`; persistence-jpa settings) | yes | yes (with the persistence-jpa line; the scheduler also needs the messaging-kafka line) | yes | The store and its settings are persistence-jpa's; the relay (`OutboxConfig`) is application composition in `app-bootstrap` because a driven adapter must not build or call a use case. It turns on only when an `OutboxStorePort` bean exists, and the scheduler only when an `OutboxMessagePublishPort` does (the messaging bridge), each ordered after its contributor by `afterName` — so an environment with no publisher keeps its PENDING rows. The bridge registers no stand-in publisher: a present-and-enabled bridge with a blank broker and no canonical transport is refused at startup. `postgresqlLegacyOutboxConcurrentRelayIntegrationTest` qualifies claim/fencing against PostgreSQL. | | Notification platform | `ca-skeleton.notification.platform.enabled` / `APP_NOTIFICATION_PLATFORM_ENABLED`, `true` | yes | yes (one line: `runtimeOnly project(':composition:notification-platform')`, plus the persistence-jpa line) | claim-scoped | Not in the default composition. The one line brings the use-case assembly, `adapter:outbound:notification`, the HTTP surface `adapter:inbound:web-notification` and its own settings (`config/module-defaults/notification.yml`), each registered through the module's own `AutoConfiguration.imports`; `adapter:inbound:web` no longer depends on `application-notification`. `verifyNotificationEvidence` refuses a grade in `docs/notification/support-matrix.md` that no proven claim backs; run by `notification-platform.yml`. There is no real-SMTP lane, and the matrix says so rather than implying one. | | File server platform | `app.fileserver-platform.enabled` / `APP_FILESERVER_PLATFORM_ENABLED`, `true` | yes | yes (one line: `runtimeOnly project(':composition:fileserver-platform')`, plus the persistence-jpa line) | yes | On when present; module-owned `config/module-defaults/fileserver.yml` also carries the servlet multipart ceiling. `app-bootstrap:integrationTest` → `FileserverCompositionIntegrationTest` (required by the strict lane), run by `integration-main.yml` → `integration-check` (`:integrationCheck`) | | Outbound HTTP client | `app.httpclient.enabled` / `APP_HTTPCLIENT_ENABLED`, `true` | yes | yes (one line: `runtimeOnly project(':adapter:outbound:httpclient')`) | yes | On when present (`HttpClientPlatformActivationTest`). `httpclient:httpClientFailureInjectionTest` against a faulted upstream, run by `integration-main.yml` → `integration-check` (`:integrationCheck`) | | GraphQL transport | `backend.graphql.enabled` / `APP_GRAPHQL_ENABLED`, `true` | yes | yes (one line: `runtimeOnly project(':adapter:inbound:graphql')`) | leaf-level | On when present; `false` also excludes Spring GraphQL's own auto-configuration, so no `/graphql` (`GraphQlCapabilityActivationTest`). `graphqlTransportQualificationTest` runs the HTTP boundary. The deployment-mode/runtime-profile rule is the module's own `GraphQlPlatformStartupValidator`. The default jar does not ship GraphQL, so a compose lane that exercises it runs the qualification jar. | -| Object storage | `app.object-storage.enabled` (no `APP_*` row), `true` | yes | yes (one line: `runtimeOnly project(':adapter:outbound:objectstorage')`) | MinIO only | Own `ObjectStorageAutoConfiguration` (was bound through bootstrap's properties scan); on when present, `false` leaves only the explicit legacy seam (`ObjectStorageAutoConfigurationTest`). `app-bootstrap:integrationTest` assembles it against digest-pinned MinIO. | +| Object storage | `app.object-storage.enabled` / `APP_OBJECTSTORAGE_ENABLED`, `true` | yes | yes (one line: `runtimeOnly project(':adapter:outbound:objectstorage')`) | MinIO only | Own `ObjectStorageAutoConfiguration` (was bound through bootstrap's properties scan); on when present, `false` leaves only the explicit legacy seam (`ObjectStorageAutoConfigurationTest`). `app-bootstrap:integrationTest` assembles it against digest-pinned MinIO. | | gRPC platform | — (build-level) | yes | **no** | yes | Absent from the main build entirely: `optional-platforms` is a separate Gradle build, so the switch is the `include`, not an environment variable. `-p optional-platforms ci` runs on every PR and `integrationCheck` certifies the operation ledger against real PostgreSQL. | | gRPC transport (`adapter:inbound:grpc`) | `ca-skeleton.grpc.server.enabled` (no `APP_*` row), `true` | yes | yes (one line: `runtimeOnly project(':adapter:inbound:grpc')`) | leaf-level | Own `AutoConfiguration.imports` (`GrpcServerConfig`); on when present, and the loopback plaintext listener still needs `ca-skeleton.grpc.server.allow-insecure-local=true` (`GrpcSafeActivationTest`, `grpcTransportQualificationTest`). | | WebSocket transport | `app.websocket.enabled` / `APP_WEBSOCKET_ENABLED`, `true` | yes | yes (one line: `runtimeOnly project(':adapter:inbound:websocket')`) | — | Own `AutoConfiguration.imports` (`WebSocketConfig`), on in a servlet app when present (`WebSocketAutoConfigurationTest`); `ConditionalTransportBuildContractTest` asserts it has no default production edge. | diff --git a/docs/fileserver/configuration.md b/docs/fileserver/configuration.md index 2cd42562..60af6511 100644 --- a/docs/fileserver/configuration.md +++ b/docs/fileserver/configuration.md @@ -128,13 +128,17 @@ Set `inline-safe-profile: true` only when downloads are never served inline from | `direct-download-permits` | `64` | concurrent non-delegated downloads | | `soft-high-water` | `0.70` | storage fraction at which pressure is reported | | `hard-high-water` | `0.85` | storage fraction at which uploads and copies are refused | -| `namespace-limit` | unset (unlimited) | most bytes one namespace may hold, live reservations plus committed usage; a create or copy that would pass it is refused with `QUOTA_EXCEEDED` (413) before any record or byte exists | +| `namespace-limit` | unset (unlimited) | most bytes one namespace may hold, live reservations plus committed usage; a create, copy or move that would pass it is refused with `QUOTA_EXCEEDED` (413) before any record, byte or namespace changes | `namespace-limit` is checked against the usage the quota ledger reports and then reserved, in one transaction but without a per-namespace lock: reservations admitted at the same moment can overshoot the limit by at most their own size, and the next one is refused. A service that needs an exact cap adds that lock in its persistence adapter. +A move to another namespace carries the file's bytes in the ledger: in the transaction that relocates +the record it checks the target's limit, books the bytes as committed usage there and reclaims them +from the source. A refused move leaves the file, and both namespaces' usage, as they were. + When the storage fraction cannot be read, admission treats it as unknown and does not apply the high-water rule — a synthetic `0` would silently disable the guard, and a synthetic `1` would take the capability down over a failed syscall. diff --git a/docs/fileserver/http-contract.md b/docs/fileserver/http-contract.md index 08cbb87b..84034bfd 100644 --- a/docs/fileserver/http-contract.md +++ b/docs/fileserver/http-contract.md @@ -15,7 +15,7 @@ and fails if one is missing, so this file cannot silently fall behind the code. | HEAD | `/v1/files/{fileId}/content` | `200`, `304` | identical headers, no body | | DELETE | `/v1/files/{fileId}` | `202`, `204` | logical delete first | | POST | `/v1/files/{fileId}:copy` | `202` | create-only target | -| POST | `/v1/files/{fileId}:move` | `200` | logical namespace change only | +| POST | `/v1/files/{fileId}:move` | `200` | namespace change only, bytes stay; the file's quota moves with it, and a target namespace past its `namespace-limit` is `413` | | OPTIONS | `/v1/uploads` | `204` | tus capability discovery | | POST | `/v1/uploads` | `201` | tus creation | | HEAD | `/v1/uploads/{uploadId}` | `204` | tus offset | diff --git a/docs/httpclient/configuration-reference.md b/docs/httpclient/configuration-reference.md index d4ff4823..206d81f7 100644 --- a/docs/httpclient/configuration-reference.md +++ b/docs/httpclient/configuration-reference.md @@ -102,7 +102,13 @@ greater than zero (`TIMEOUT_NOT_POSITIVE`): both engines read zero as "wait fore `total-call` also bounds the setup of a streaming download — connect, send, and the wait for response headers. Once the stream is handed over, each gap of the body is bounded by `read-idle` -instead, so a long download is not cut at an arbitrary total (design §15.3). +instead, so a long download is not cut at an arbitrary total (design §15.3). This holds on both +transports. The JDK transport keeps its exchange-wide timeout off a streamed body and bounds each +read with its own `read-idle` timer, because the JDK client has no socket read timeout; before +review 2026-11-03 RR-04-03 every JDK download longer than `response-header` was cut mid-body. A body +failure after hand-over is thrown as `HttpResponseTimeoutException` (a gap longer than `read-idle`) +or `HttpResponseTruncatedException` (the connection ended early), stage `RESPONSE_BODY`, evidence +`PARTIAL_RESPONSE`. ### Removed settings @@ -126,6 +132,14 @@ it as a result — and every transport failure. Every other 4xx is a healthy ans only say how many failures, out of how many calls, open the circuit. Defaults are Resilience4j's own, which every profile ran with before the keys existed. +Two deadlines can end an attempt in flight, and they are counted differently (review 2026-11-03 +RR-04-01): + +| Ended by | Breaker | Why | +|---|---|---| +| The caller's own deadline (`HttpOperation.deadline()`, when earlier than `total-call`) | Not recorded; the permission is given back | The caller ran out of time. Counting it opened the circuit on a healthy upstream whenever callers passed short budgets | +| The profile's `timeout.total-call` | Recorded as a failure | `total-call` is the operator's statement of how long a healthy upstream may take; a slow-drip upstream that never trips a per-gap timeout is seen only through it | + | Property | Default | Meaning | |---|---|---| | `failure-rate-threshold` | `50` | Percent of failed calls in the window that opens the circuit, 1–100 | diff --git a/docs/jpa/entity-mapping-guide.md b/docs/jpa/entity-mapping-guide.md index 9aec101d..4d3aa55e 100644 --- a/docs/jpa/entity-mapping-guide.md +++ b/docs/jpa/entity-mapping-guide.md @@ -72,17 +72,23 @@ What to use, in order of preference: A unidirectional `@OneToMany` with its own `@JoinColumn` is the exception, not the default. Use it only for child entities the root genuinely owns and changes only through itself (the testkit's `LifecycleParent` and `PagedParent` are this shape). It still costs an `UPDATE` per child when the -join column is nullable, and the collection still loads as a unit. +join column is nullable, and the collection still loads as a unit. The child then has no +association back to the root — not even a read-only `@ManyToOne` on the same join column +(`insertable = false, updatable = false`): that is the bidirectional pair again, without the +`mappedBy` that names it. If the child needs the root, it holds the root's ID. `CascadeType.ALL` with `orphanRemoval` is correct only for a child the aggregate genuinely owns. Between independent aggregates it deletes rows another part of the system still owns — another reason those are ID references. -`JpaArchitectureRules.associationsAreMappedFromOneSideOnly()` (testkit) fails the build on any -`@OneToMany`, `@OneToOne` or `@ManyToMany` that declares `mappedBy`, on `@Entity`, -`@MappedSuperclass` and `@Embeddable` types. `JpaProductionArchitectureTest` (app-bootstrap +`JpaArchitectureRules.associationsAreMappedFromOneSideOnly()` (testkit) fails the build on every +bidirectional pair, on `@Entity`, `@MappedSuperclass` and `@Embeddable` types: an association that +declares `mappedBy` (only that inverse half is reported), and two entities whose associations point +at each other without `mappedBy` (both halves are reported — the `@JoinColumn` form above, review +RR-03-04). A class that links to itself is a pair only when a second member points back the other +way (`@ManyToOne parent` with `@OneToMany children`). `JpaProductionArchitectureTest` (app-bootstrap `architectureTest`) runs it over the production graph and `SampleArchitectureTest` over the sample. -The failure message names the field and these alternatives. +The failure message names the fields and these alternatives. Chosen and rejected: diff --git a/docs/manual/04-persistence.md b/docs/manual/04-persistence.md index 4b40ee6e..4d5faddd 100644 --- a/docs/manual/04-persistence.md +++ b/docs/manual/04-persistence.md @@ -28,6 +28,9 @@ use case ── tx.inWrite ──▶ XRepository (domain 포트) 3. 엔티티·Spring Data repository·mapper·adapter를 쓴다. adapter는 aggregate만 주고받는다 (`OUTBOUND_ADAPTER_METHOD_RETURNS_ONLY_DOMAIN_OR_PRIMITIVES`). 4. `JpaXRepositoryContractTest extends XRepositoryContract`를 실 PostgreSQL lane에 둔다(08장 lane). + adapter는 호출자 트랜잭션 밖의 호출을 거부하므로(아래 「트랜잭션 안에서만 쓴다」), 계약의 + `inTransaction(Supplier)`을 재정의해 거기서 트랜잭션을 열고 커밋한다. 계약은 모든 저장소 호출을 그 + hook으로 부른다. fake는 기본값(그냥 실행)을 쓴다. 예: `src/sample-portfolio/src/integrationTest/java/dev/caskeleton/sample/portfolio/adapter/outbound/persistence/repository/WorkLogRepositoryAdapterContractTest.java`. 5. adapter를 bean으로 등록하는 자동 설정을 쓰고, 기능의 `XConfig`에 `afterName`으로 그 순서를 적는다. @@ -80,6 +83,13 @@ store는 자기 트랜잭션을 열지 않는다. 유스케이스가 연 트랜 예: `src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/adapter/outbound/persistence/repository/PosterRepositoryAdapter.java` (`DataIntegrityViolationException` → 제약 이름 확인 → `PosterTitleAlreadyExistsException` 409), 제약은 `src/sample-portfolio/src/main/resources/db/sample-migration/V8_1__poster_title_unique.sql`. + - **충돌을 바꾸는 곳은 하나다.** 업무 규칙 충돌(같은 제목)은 포트 옆 domain 패키지의 예외 타입이고, + 포트가 `@throws`로 선언하고 adapter가 던진다. 유스케이스의 사전 조회도 같은 타입을 던져 두 경로가 같은 + 409가 된다. application 예외를 domain 포트가 Javadoc으로만 가리키면 fake·mock은 그 약속을 지킬 수 없다. + 기술 충돌(버전 불일치·교착)은 adapter가 프레임워크 예외 그대로 두고 아래 `PersistenceExceptionTranslator`가 + `DB_*` 코드로 바꾼다(WorkLog의 `ObjectOptimisticLockingFailureException` → `DB_OPTIMISTIC_CONFLICT`). + - 그 약속은 포트 소유자 testFixtures의 공유 계약 테스트로 고정한다. fake와 adapter가 같은 계약을 상속한다. + 예: `PosterRepositoryContract`(`InMemoryPosterRepository`, `PosterRepositoryAdapterContractTest`). - 여러 인스턴스가 집는 작업 → `FOR UPDATE SKIP LOCKED`는 **트랜잭션 안에서**, 그리고 lease(만료 시각)와 fence 값을 함께 저장한다. 예: `ReconciliationJobStorePort.claimDue`(application-notification), gRPC operation ledger의 claim. diff --git a/docs/manual/05-idempotency.md b/docs/manual/05-idempotency.md index 8ffd3d4f..1883f5a7 100644 --- a/docs/manual/05-idempotency.md +++ b/docs/manual/05-idempotency.md @@ -50,6 +50,7 @@ DB 커밋 중 연결 끊김 ───────────────── | 409 `IDEMPOTENT_IN_FLIGHT`, `retryable=true`, `Retry-After: 2` | 같은 키가 아직 실행 중. 결과 불명 | **같은 키로** 기다렸다 재전송. 새 키 금지 | | 422 `IDEMPOTENT_REQUEST_MISMATCH` | 같은 키에 다른 본문 | 클라이언트 버그. 재전송 안 함 | | 503 `IDEMPOTENCY_UNAVAILABLE` | 실행 전 거부 | 같은 키로 나중에 | +| 503 `IDEMPOTENCY_COMPLETION_UNKNOWN`, `retryable=true` | 실행 뒤 결과 기록 실패. 적용됐을 수 있음 | "적용 안 됨"이라 하지 않는다. **같은 키로** 나중에 재전송, 새 키 금지 | | 500 `IDEMPOTENCY_RECOVERY_REQUIRED` | 이전 시도 효과를 증명 못함 | 재전송·재실행 모두 위험. 업무 데이터로 확인 | | 500 `DB_COMPLETION_UNKNOWN`, `retryable=false` | DB 커밋 중 연결 끊김. 커밋됐을 수도 있음 | 키 없이 재시도 금지. KEYED면 같은 키로 재전송하면 서버가 판정 | diff --git a/docs/manual/06-errors.md b/docs/manual/06-errors.md index 4afebb58..d80efb4b 100644 --- a/docs/manual/06-errors.md +++ b/docs/manual/06-errors.md @@ -41,8 +41,9 @@ gRPC/GraphQL/WebSocket: 같은 ApiErrorCarrier를 자기 status 모양으로 | 서버 조립·설정 오류, 있을 수 없는 상태 | `IllegalStateException` | 500, 로그 남음 | | DB·외부 의존 실패 | adapter가 shared-contract carrier(`PersistenceFailureException`, `DependencyFailureException`)로 | 분류표대로(04·05장) | -- `IllegalArgumentException`을 "서버 조립이 틀렸다"에 쓰지 않는다. web은 그것을 로그 없이 400으로 - 바꿔서 서버 결함이 클라이언트 오류로 숨는다(리뷰 01-04). +- `IllegalArgumentException`을 "서버 조립이 틀렸다"에 쓰지 않는다. web은 그것을 400으로 바꾸고 WARN + 한 줄만 남긴다(`GlobalExceptionHandler`). 서버 결함이 클라이언트 오류 통계에 섞여 경보가 울리지 않는다 + (리뷰 01-04). 서버 쪽 잘못은 `IllegalStateException`(500, 로그)이다. - 메시지는 로그용이다. 클라이언트에는 코드와 category별 고정 문구(`ClientErrorMessages`)만 간다. 메시지에 SQLState·내부 경로·토큰을 넣어도 클라이언트에 새지 않지만, 로그 정책은 지킨다. diff --git a/docs/manual/07-authorization.md b/docs/manual/07-authorization.md index bbdfa5a9..4aef0641 100644 --- a/docs/manual/07-authorization.md +++ b/docs/manual/07-authorization.md @@ -62,13 +62,30 @@ JWT의 `realm_access.roles`와 **자기 client**(`app.security.audience`)의 `re batch 메서드로 N+1 회피)를 구현한다. 존재를 숨길 필요가 없는 관리 화면은 403도 된다. - tenant: 인증 정보에서만 만든 `TenantId`를 command에 싣는다. 요청 본문의 tenant 값을 믿지 않는다. -## 지금 확인되지 않는 것 +## 시스템이 부르는 쓰기: `@SystemInvoked` -- web이 빠진 조립(gRPC 전용·배치 전용)에서는 advisor가 없어 선언이 검사되지 않는다. 이때 기동을 막는 - 장치는 아직 없다(리뷰 01-06). web 없이 조립하는 서비스는 그 전에 이 문제를 먼저 푼다. +스케줄러·relay처럼 사람이 아니라 시스템이 부르는 쓰기 유스케이스는 `@RequiresPermission` 대신 +`@SystemInvoked(reason = "...")`를 단다(`src/application-core/src/main/java/dev/caskeleton/application/security/SystemInvoked.java`). +스케줄러 스레드에는 인증이 없어 권한 선언은 장식이 되기 때문이다. 이 표시가 붙은 유스케이스는 **권한 +검사가 없는 쓰기**이므로 전송 계층이 닿으면 안 된다. 그래서 두 가지가 막는다. + +| 막는 것 | 어디 | +| --- | --- | +| bean으로 등록하면 기동 실패. 조립 모듈이 `new`로 만들어 스케줄러에만 준다(예: `composition:outbox-relay`) | `src/composition/service-runtime/src/main/java/dev/caskeleton/composition/runtime/authorization/PermissionEnforcementGuard.java` | +| inbound adapter가 그 클래스를 직접 쓰면 빌드 실패 | `INBOUND_ADAPTERS_DO_NOT_REACH_SYSTEM_INVOKED_USE_CASES`(`ApplicationBoundaryArchitectureTest`) | + +D4 실패(`MUTATING_USE_CASES_DECLARE_REQUIRED_PERMISSION`)를 끄려고 이 표시를 쓰지 않는다. 사용자가 부르는 +쓰기라면 권한을 선언하고 역할에 매핑한다. + +## web 없는 조립 + +web이 빠진 조립(gRPC 전용·배치 전용)에는 `@RequiresPermission`을 검사할 advisor가 없다. 그런 조립에 +권한을 선언한 bean이 있으면 `PermissionEnforcementGuard`가 기동을 실패시킨다(리뷰 01-06). web 없이 +조립하는 서비스는 자기 전송에 맞는 검사 advisor를 함께 넣어야 기동한다. ## 지금 이해하면 되는 것 - 쓰기 = `@RequiresPermission` + non-final + bean + 역할 매핑. +- 시스템만 부르는 쓰기 = `@SystemInvoked` + bean 아님 + 전송에서 안 닿음. - 객체 판정은 유스케이스 안, 남의 것은 404. - 역할은 자기 client 것만 읽는다. diff --git a/docs/manual/09-outbox-events.md b/docs/manual/09-outbox-events.md index 1c6f7999..753781b4 100644 --- a/docs/manual/09-outbox-events.md +++ b/docs/manual/09-outbox-events.md @@ -32,6 +32,10 @@ DB 저장과 broker 발행을 따로 하면(dual write) 둘 중 하나만 성공 등록을 빠뜨리면 append 때 실패한다. 예: `DurableOperationAutoConfiguration.durableOperationSubmittedContract()`. 5. 테스트: 계약 contribution 테스트(해시 고정), 유스케이스가 append했는지(`InMemoryOutboxStore`), 소비자 중복 제거 계약(sample `WorkLogReservedConsumerDedupeContractTest`). +6. schema를 출시한다: `src/test/resources/contracts/released-payload-schemas.sha256`에 새 version 파일의 + 해시를 한 줄 더한다. 출시한 version 파일은 고치지 않는다 — 바뀌면 `ReleasedPayloadSchemaContract` + 테스트가 실패한다. 필드를 바꾸는 변경은 새 version으로 낸다. 절차: `src/application-core/README.md` + "payload version 올리기"(리뷰 06-10). ## 판단 지점 @@ -49,22 +53,38 @@ DB 저장과 broker 발행을 따로 하면(dual write) 둘 중 하나만 성공 ### event id·correlation·순서 -- event id: UUIDv7, 소비자 중복 제거 키. 같은 작업의 재시도가 같은 event id를 내야 하면(durable - operation) 작업 id에서 파생한다. -- correlation: 요청의 correlation id가 문법에 맞으면 그것, 아니면 event id. 문법 밖 헤더가 업무 쓰기를 - 실패시키면 안 된다. +draft는 `IntegrationEventDrafts.draft(contract, eventId, aggregate, order, occurredAt, requestCorrelationId, payload)`로 +만든다(`src/application-core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventDrafts.java`). +계약 id·payload 버전·목적지는 계약에서 오고, 기능이 정하는 것은 aggregate·순서·payload뿐이다. +`new IntegrationEventDraft<>(…)`를 직접 쓰면 빌드가 막는다(`INTEGRATION_EVENT_DRAFTS_ARE_BUILT_BY_THE_SHARED_HELPER`). +예: sample `PortfolioEventOutbox`, `SubmitDurableOperationUseCase`. + +- event id: 정규 소문자 UUIDv7, 소비자 중복 제거 키. draft(`IntegrationEventDraft`)가 다른 값(대문자, + v4, `event-1` 같은 문자열)을 거부하므로 업무 쓰기에서 바로 실패한다 — 커밋된 뒤 relay에서 DEAD가 되지 + 않는다(리뷰 06-06). 같은 작업의 재시도가 같은 event id를 내야 하면(durable operation) 작업 id를 쓰고, + 작업 id가 UUIDv7이 아니면 작업 id와 접수 시각에서 UUIDv7을 파생한다. +- correlation: 헬퍼가 정한다. 요청의 correlation id가 `IntegrationIdentity` 문법·길이에 맞으면 그것, + 아니면 event id. 문법 밖 헤더가 업무 쓰기를 실패시키지 않는다. 값을 미리 검사해야 하면 문법을 베끼지 + 말고 `IntegrationIdentity.isCanonical(value, IntegrationIdentity.CORRELATION_MAX_LENGTH)`를 부른다 + (aggregate id는 `AGGREGATE_ID_MAX_LENGTH`). +- tenant 범위: tenant가 없는 기능은 `IntegrationEventDrafts.SINGLE_TENANT_SCOPE`(`global`). - aggregate 순서: `OutboxAggregateSequencePort.nextSequence(aggregate)`로 같은 `tx.inWrite` 안에서 받는다(V21 카운터, 커밋 순서를 따른다). 이벤트가 하나뿐인 aggregate는 `AggregateOrder.first()`. 발생 - 시각(벽시계)은 쓰지 않는다 — 서버 시계가 어긋나면 순서가 뒤집힌다(리뷰 06-07). -- 이 규칙은 아직 기능마다 다시 쓴다(sample `PortfolioEventOutbox`). application-core 헬퍼로 올리는 일은 - 리뷰 01-09 후속이다. 그 전까지는 위 세 규칙을 지킨다. + 시각(벽시계)은 쓰지 않는다 — 서버 시계가 어긋나면 순서가 뒤집힌다(리뷰 06-07). 벽시계 번호를 쓰던 + 서비스가 포트로 옮기면 V21 floor 덕분에 새 번호는 옛 번호 뒤에서 시작한다. 그래서 첫 번호가 1이 + 아니라 약 1.7×10¹⁵일 수 있다. 소비자는 번호를 "마지막으로 본 값보다 큰가"로만 비교하고 연속성을 + 가정하지 않는다. ## relay와 실패 -- relay는 `composition:outbox-relay`가 조립한다. 유스케이스로 만든 relay를 bean으로 등록하지 않는다 - (스케줄러 스레드에는 인증이 없어 권한 프록시가 매번 거부한다). -- broker의 **재시도 가능한** 거부는 DEAD가 아니라 backoff 재시도다. broker가 멈춰도 대량 DEAD가 되지 - 않는다(1라운드 06-01·06-03). 진짜 인증 오류도 재시도로 남으니 경보로 잡는다. +- relay는 `composition:outbox-relay`가 `new`로 조립해 스케줄러에만 준다. relay는 `@SystemInvoked`라 권한 + 선언이 없어서, bean이 되면 전송 계층이 **권한 검사 없이** 닿는다. 그래서 bean으로 등록하면 + `PermissionEnforcementGuard`가 기동을 실패시킨다(07장 「시스템이 부르는 쓰기」). +- broker의 **재시도 가능한** 거부와 응답 없음은 DEAD가 아니라 backoff 재시도다. broker가 멈춰도 대량 + DEAD가 되지 않는다(1라운드 06-01·06-03). 대신 끝이 없다: 진짜 인증 오류, Kafka adapter가 이름을 모르는 + broker 거부도 영원히 재시도된다. 행의 `last_failure_code`와 로그의 `error.reason`이 원인 코드를 남기고, + `outbox.retry.long.size`가 오래 재시도 중인 행을 센다. 처분(보류·재개·skip)은 publish-failed runbook + §4.1이다(재리뷰 RR-05-03). - 발행 응답을 못 받으면 결과 불명(`OutboxPublishAmbiguousException`) — 05장 규칙과 같다. - 운영: `docs/runbooks/outbox-dead-letter.md`, `docs/runbooks/outbox-publish-failed.md`. diff --git a/docs/messaging/configuration-reference.md b/docs/messaging/configuration-reference.md index 3fc19e98..07130a20 100644 --- a/docs/messaging/configuration-reference.md +++ b/docs/messaging/configuration-reference.md @@ -26,7 +26,9 @@ producing-service identity다. ## Outbox -outbox는 canonical 하나다(2026-09-28). use case가 `IntegrationEventEncoderPort`로 만든 +application의 발행 경로는 하나다(2026-09-28): `outbox_event` 표와 `composition:outbox-relay`. vendored +platform의 `messaging_outbox`(`outbox-inbox.md`)는 이 경로에 쓰이지 않는다 — 두 모델의 상태 대응은 +`outbox-inbox.md` 머리의 표에 있다. use case가 `IntegrationEventEncoderPort`로 만든 `ValidatedIntegrationEvent`를 `OutboxAppendPort`로 append하고, relay는 claim한 row의 exact envelope bytes를 `IntegrationEventPublishPort`로 보낸다. 예전의 `app.outbox.canonical-transport-enabled` gate와 legacy `MessageBroker → KafkaSender` 경로는 삭제됐다. 발행할 contract의 logical destination마다 diff --git a/docs/messaging/outbox-inbox.md b/docs/messaging/outbox-inbox.md index 6ec2d046..08f18189 100644 --- a/docs/messaging/outbox-inbox.md +++ b/docs/messaging/outbox-inbox.md @@ -1,5 +1,25 @@ # Outbox · Inbox +> **이 문서의 Outbox는 vendored messaging platform의 outbox다**(`messaging_outbox` 표, +> `messaging-reliability`의 `OutboxRelay`). 이 템플릿 application이 쓰는 outbox는 그것이 아니다. +> application은 `outbox_event` 표(`adapter:outbound:persistence-jpa`)에 append하고, relay는 +> `composition:outbox-relay`가 돌린다. 그 동작과 운영은 `docs/manual/09-outbox-events.md`, +> `src/composition/outbox-relay/README.md`, `docs/runbooks/outbox-dead-letter.md`, +> `docs/runbooks/outbox-publish-failed.md`에 있다. platform outbox는 소유자 결정으로 제품 안에 남아 +> 있지만, platform outbox repository bean이 없으면 그 relay는 만들어지지 않는다(리뷰 2026-11-01 06-11). +> +> 두 모델은 같은 이름의 상태가 반대 뜻이다. 이름으로 옮기면 확정 거부는 영원히 재시도되고 응답 없는 +> 메시지는 버려진다. 대응은 아래 표 하나로만 읽는다(`OutboxPublishOutcome` Javadoc과 같은 내용). +> +> | 일어난 일 | application `outbox_event` | platform `messaging_outbox` | +> |---|---|---| +> | broker가 확인했다 | `PUBLISHED` | `PUBLISHED` | +> | 응답이 없었다(저장됐을 수 있음) | `FAILED`(재시도, 예산 안 씀, 끝 없음) | `AMBIGUOUS`(재시도, 예산을 다 쓰면 `EXHAUSTED`) | +> | 지금만 거부됐다(종료·포화·credential) | `FAILED`(재시도, 예산 안 씀, 끝 없음) | `FAILED`(종단) | +> | 확정 거부됐다(같은 payload는 또 거부) | `DEAD`(종단, 수동 처분) | `FAILED`(종단) | +> | publisher가 예외를 던졌고 시도 예산을 다 썼다 | `DEAD`(종단, 수동 처분) | `EXHAUSTED`(종단) | +> | 운영자가 재시도를 멈췄다 | `PARKED`(수동 재개·skip) | 없음 | +> > **이 템플릿 application의 소비 경로**: application의 `MessageHandler`와 inbox 부품(애플리케이션 > DataSource 위 `JdbcInboxRepository`, `PlatformTransactionManager` 위 `TransactionRunner`)은 > `adapter:inbound:messaging`(`src/adapter/inbound/messaging/README.md`)이 둔다. platform starter는 두 diff --git a/docs/notification/configuration-reference.md b/docs/notification/configuration-reference.md index f7a6fc7b..1e161d49 100644 --- a/docs/notification/configuration-reference.md +++ b/docs/notification/configuration-reference.md @@ -48,7 +48,7 @@ still waiting on gets claimed by a second worker, and the recipient receives the | `callbacks.replay-skew` | `APP_NOTIFICATION_PLATFORM_CALLBACK_REPLAY_SKEW` | `1h` | positive; SNS messages whose `Timestamp` is further than this from receipt, either way, are rejected. The default is SNS's longest HTTP/S retry period (3,600 s), so a bounce or complaint SNS redelivers after an outage is still accepted; a replay inside the window is a duplicate in the provider event ledger, not a second event (review 05-20) | | `callbacks.trusted-proxies` | `APP_NOTIFICATION_PLATFORM_CALLBACK_TRUSTED_PROXIES` | *(empty)* | CSV of peer addresses | -콜백 응답: 기록(중복 포함) `204`, 거절 `400`(서명·topic·시각·형식), 본문 초과 `413`. SNS 서명 인증서를 가져오지 못하면 거절하지 않고 `503`+`Retry-After`로 답한다 — SNS는 4xx를 처리 완료로 보고 재전송을 멈추지만 5xx는 다시 보낸다(review 05-20). +콜백 응답: 기록(중복 포함) `204`, 거절 `400`(서명·topic·시각·형식), 본문 초과 `413`. SNS 서명 인증서를 가져오지 못하면(5xx·408·429·네트워크 실패) 거절하지 않고 `503`+`Retry-After`로 답한다 — SNS는 4xx를 처리 완료로 보고 재전송을 멈추지만 5xx는 다시 보낸다(review 05-20). 인증서 URL이 그 밖의 4xx(403·404 등)로 답하면 SNS가 발급하지 않은 인증서를 가리킨 위조 콜백이므로 서명 검증 실패로 `400` 거절하고 보안 감사(`callbackSignatureRejected`)에 남긴다(review 2026-11-03 RR-07-03). 호스트 검사를 통과한 URL이라 답하는 쪽은 SNS다. 여러 provider가 요청 URL에 서명하므로, 그 URL을 잘못 재구성하면 정상 webhook이 전부 서명 실패가 된다. `trusted-proxies`가 비어 있으면 forwarded 헤더를 **믿지 않고** 컨테이너가 관측한 값을 쓴다. 무조건 믿으면 diff --git a/docs/notification/evidence-manifest.json b/docs/notification/evidence-manifest.json index 752a4402..151f8860 100644 --- a/docs/notification/evidence-manifest.json +++ b/docs/notification/evidence-manifest.json @@ -54,7 +54,8 @@ "status": "satisfied", "lane": ":adapter:outbound:persistence-jpa:jpaPlatformContractTest", "evidence": [ - "adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/WorkerCrashRecoveryContractTest.java" + "adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/WorkerCrashRecoveryContractTest.java", + "adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlNotificationBackgroundPassIntegrationTest.java" ] }, "provider-wire-qualified": { diff --git a/docs/notification/support-matrix.md b/docs/notification/support-matrix.md index 2c89927f..9850179e 100644 --- a/docs/notification/support-matrix.md +++ b/docs/notification/support-matrix.md @@ -49,10 +49,13 @@ assembler is absent because the relay was not configured. | Extension | nothing; the receiving contract owns its own guarantees | | Experimental | nothing; the grade is the warning | -`recoverable` was met on 2026-08-15 by `WorkerCrashRecoveryContractTest`, which walks the four -phases a dispatch passes through — claimed, attempt written, request started, body committed — and -asserts for each that the delivery becomes claimable again or becomes a question for the provider, -never both and never neither. +`recoverable` is backed by two tests. `WorkerCrashRecoveryContractTest` walks the four phases a +dispatch passes through — claimed, attempt written, request started, body committed — and pins the +rows a dead worker leaves at each: a passed lease, and from the attempt insert on an unfinished +attempt carrying that phase's evidence. It writes those rows itself, so it checks recovery's input, +not recovery. `PostgreSqlNotificationBackgroundPassIntegrationTest` runs lease recovery and +reconciliation over rows the dispatcher's own factory wrote and asserts the outcome: requeued before +any attempt, reconciled after one (review 05-01, RR-03-07). The remaining unmet claim, and what would meet it: diff --git a/docs/registries/env-keys.yaml b/docs/registries/env-keys.yaml index c59640f4..61e9ea02 100644 --- a/docs/registries/env-keys.yaml +++ b/docs/registries/env-keys.yaml @@ -1670,7 +1670,7 @@ env_keys: required: false reload_policy: restart-only owner_branch: redis-production-capability-completion - validation: positive_duration_le_1h + validation: duration_gt_in_flight_retry_after_2s_le_1h compatibility_impact: additive required_test: redis-idempotency-contract:processing-lease-bounded @@ -1806,6 +1806,24 @@ env_keys: compatibility_impact: additive required_test: redis-capability:cache-region-composition + - name: APP_CACHE_REDIS_MAXIMUM_SCRIPT_CONNECTIONS + # The cache's own script-lane connections. Separate from APP_REDIS_CAPACITY_MAXIMUM_SCRIPT_CONNECTIONS, + # which idempotency, rate limiting and leases share, so cache fills cannot starve them (review + # 2026-11-03 README section 4, RR-04-02). + property: app.capabilities.cache.maximum-script-connections + owner_module: adapter-outbound-cache-redis + type: integer + default: 4 + allowed_values: null + classification: public-config + required: false + required_when: app.capabilities.cache.bindings.default=redis + reload_policy: restart-only + owner_branch: refactor/maturity-review-2026-11 + validation: positive_integer + compatibility_impact: additive + required_test: redis-capability:cache-script-lane-separate-from-correctness + - name: APP_IDEMPOTENCY_REDIS_COMMAND_TIMEOUT # The ceiling on one owner-safe transition. type: duration @@ -2000,7 +2018,7 @@ env_keys: # secret://@/ form as the application reference. Unset, scripts run # as the application account and startup logs a warning saying so. The Compose Redis lanes use # secret://ca-skeleton-application-advanced@environment/REDIS_LANE_ADVANCED_PASSWORD, a fixture - # account of infra/redis-sdk/acl/all-accounts.acl. + # account of infra/redis-sdk/acl/lane-fixture-accounts.acl. property: app.redis.authentication.advanced-credential-reference owner_module: adapter-outbound-cache-redis type: string @@ -2043,7 +2061,7 @@ env_keys: # . Without the account part the client authenticates as `default`, which a hardened # Redis turns off. The Compose Redis lanes use # secret://ca-skeleton-application@environment/REDIS_LANE_APPLICATION_PASSWORD, a fixture - # account of infra/redis-sdk/acl/all-accounts.acl. + # account of infra/redis-sdk/acl/lane-fixture-accounts.acl. property: app.redis.authentication.credential-reference owner_module: adapter-outbound-cache-redis type: string @@ -4043,6 +4061,39 @@ env_keys: compatibility_impact: behavior-change required_test: websocket-contract:canonical-property-namespace + - name: APP_OBJECTSTORAGE_ENABLED + # 2026-11-03 RR-06-07: the explicit off the Compose smoke wrapper sets for every lane that is + # not about object storage. It bound by relaxed binding and was in no registry, so the env + # contract could not tell it from an unread name. The module is on when present; absent from + # the build is the ordinary off state. + property: app.object-storage.enabled + owner_module: adapter-outbound-objectstorage + type: boolean + default: true + allowed_values: [true, false] + classification: public-config + required: false + reload_policy: restart-only + owner_branch: refactor/maturity-review-2026-11 + validation: boolean + compatibility_impact: none + required_test: off-state-zero-side-effects-and-env-round-trip + - name: APP_GRPC_SERVER_ENABLED + # 2026-11-03 RR-06-07: the explicit off for the gRPC server the Compose lanes set (it replaced + # CASKELETON_GRPC_ENABLED, which nothing read). The module is on when present; absent from the + # build is the ordinary off state. + property: app.grpc.server.enabled + owner_module: adapter-inbound-grpc + type: boolean + default: true + allowed_values: [true, false] + classification: public-config + required: false + reload_policy: restart-only + owner_branch: refactor/maturity-review-2026-11 + validation: boolean + compatibility_impact: none + required_test: off-state-zero-side-effects-and-env-round-trip - name: APP_GRAPHQL_ENABLED # 2026-09-25 capability packaging: the module is on when present (its own config yml maps # this key with default true); false is the explicit off. Absent from the build is the diff --git a/docs/registries/error-codes.yaml b/docs/registries/error-codes.yaml index 9c70a5a4..d569b16b 100644 --- a/docs/registries/error-codes.yaml +++ b/docs/registries/error-codes.yaml @@ -401,6 +401,9 @@ errors: # source: docs/web/http-contract.md §2 (2026-11-01 소유자 결정, 리뷰 02-01 대안 A) — 결과 불명. 같은 Idempotency-Key로 Retry-After 뒤 재전송. 새 키 금지. # 2초 = 기본 processing lease(30s)보다 짧은 고정값. 이전 "client는 polling"(retryable=false)은 조회 엔드포인트가 없어 폐기. + # 값 하나: IdempotencyExecutor.IN_FLIGHT_RETRY_AFTER. processing-lease가 이 값 이하면 기동 실패(재리뷰 RR-02-05). + # behavior-change: retryable false→true는 web Envelope뿐 아니라 gRPC trailer error-retryable(status ABORTED)과 + # GraphQL extension retryable에도 나간다(재리뷰 RR-02-06, GrpcStatusMapperTest·GraphQlWireErrorMapperTest가 고정). - code: IDEMPOTENT_IN_FLIGHT category: CONFLICT http_status: 409 @@ -411,7 +414,7 @@ errors: client_safe_message: "현재 상태에서는 요청을 처리할 수 없습니다." log_level: INFO runbook_link: null - compatibility_impact: none + compatibility_impact: behavior-change required_test: contract-verification:idempotency # source: feature-rate-limit-idempotency-contract L72 — "fingerprint mismatch (same key + different body) = 422 IDEMPOTENT_REQUEST_MISMATCH" @@ -442,6 +445,22 @@ errors: compatibility_impact: additive required_test: contract-verification:idempotency + # source: 재리뷰 2026-11-03 README 4절 결정, RR-02-03 — action이 돈 뒤 complete 단계에서 저장소가 응답하지 않음. + # 실행 전 거부(IDEMPOTENCY_UNAVAILABLE)와 code로 구분한다. 결과 불명: 같은 키로 재전송하면 서버가 replay·in-flight· + # recovery-required로 판정한다. 새 키 금지. docs/web/http-contract.md §2. + - code: IDEMPOTENCY_COMPLETION_UNKNOWN + category: TRANSIENT_DEPENDENCY + http_status: 503 + retryable: true + retry_after_seconds: null + owner_branch: feature-rate-limit-idempotency-contract + owner_layer: application + client_safe_message: "일시적인 오류입니다. 잠시 후 다시 시도해 주세요." + log_level: ERROR + runbook_link: "runbook://idempotency/unavailable" + compatibility_impact: additive + required_test: contract-verification:idempotency + # source: review-followup P15 (X2-P2-2a) — replay cannot decide whether an external effect happened - code: IDEMPOTENCY_RECOVERY_REQUIRED category: DATA_INTEGRITY diff --git a/docs/registries/metrics.yaml b/docs/registries/metrics.yaml index 0435f3a5..043fccf4 100644 --- a/docs/registries/metrics.yaml +++ b/docs/registries/metrics.yaml @@ -349,8 +349,10 @@ metrics: cardinality_limit: 50 - name: outcome cardinality_limit: 5 - # STALE: the relay's claim was taken over before its result could be written (V17 lease fence). - allowed_values: [PUBLISHED, FAILED, DEAD, IN_FLIGHT, STALE] + # The relay cycle outcomes (OutboxRelayResult.Outcome), not row statuses: IN_FLIGHT is a status + # and is never counted. STALE: the relay's claim was taken over before its result could be + # written (V17 lease fence). RELEASED: a stopping relay gave back a claim it had not used. + allowed_values: [PUBLISHED, FAILED, DEAD, STALE, RELEASED] percentiles: null histogram_buckets: null alert_severity_thresholds: @@ -378,13 +380,14 @@ metrics: required_test: contract-verification:metrics-cardinality # source: feature-domain-event-outbox-contract — row status enum + Outbox Defaults attemptCount + # PARKED: an operator held a row that was retried without end (re-review 2026-11-03 RR-05-03). - name: outbox.pending.size type: gauge unit: total tags: - name: status - cardinality_limit: 5 - allowed_values: [PENDING, IN_FLIGHT, PUBLISHED, FAILED, DEAD] + cardinality_limit: 6 + allowed_values: [PENDING, IN_FLIGHT, PUBLISHED, FAILED, DEAD, PARKED] percentiles: null histogram_buckets: null alert_severity_thresholds: @@ -394,6 +397,35 @@ metrics: compatibility_impact: additive required_test: contract-verification:metrics-cardinality + # source: re-review 2026-11-03 RR-05-03 — a refusal for now and an unanswered publish are retried + # without end; these count the FAILED rows deferred at least 10 times (OutboxMetrics + # LONG_RETRY_DEFERRED_CLAIMS, the publish-failed runbook's §4.1 query) and age the oldest of them. + - name: outbox.retry.long.size + type: gauge + unit: total + tags: [] + percentiles: null + histogram_buckets: null + alert_severity_thresholds: + p2: "> 0 for 10m (runbook://outbox/publish-failed §4.1)" + owner_branch: feature-domain-event-outbox-contract + log_field_mapping: [] + compatibility_impact: additive + required_test: contract-verification:metrics-cardinality + + - name: outbox.retry.long.age + type: gauge + unit: seconds + tags: [] + percentiles: null + histogram_buckets: null + alert_severity_thresholds: + p2: "> 3600 (runbook://outbox/publish-failed §4.1)" + owner_branch: feature-domain-event-outbox-contract + log_field_mapping: [] + compatibility_impact: additive + required_test: contract-verification:metrics-cardinality + # === Cache === # source: feature-cache-consistency-contract — Decisionized Work Items "cache pattern | cache-aside default" - name: cache.gets.total @@ -501,240 +533,122 @@ metrics: compatibility_impact: additive required_test: contract-verification:metrics-cardinality - # source: redis-production-capability Task 16 — closed semantic operation outcomes - - name: redis.capability.operations.total - type: counter - unit: total - tags: - - name: capability - cardinality_limit: 6 - allowed_values: [cache, rate_limit, idempotency, efficiency_lease, session, runtime] - - name: role - cardinality_limit: 3 - allowed_values: [cache, coordination, session] - - name: operation - cardinality_limit: 24 - allowed_values: - - lookup - - record - - invalidate - - refresh_claim - - refresh_release - - rate_evaluate - - idempotency_claim - - idempotency_start - - idempotency_renew - - idempotency_complete - - idempotency_fail - - idempotency_release - - idempotency_inspect - - lease_acquire - - lease_inspect - - lease_renew - - lease_release - - session_create - - session_inspect - - session_save - - session_touch - - session_revoke - - session_rotate - - route_command - - name: redis_outcome - cardinality_limit: 15 - allowed_values: - - success - - hit - - miss - - denied - - contended - - conflict - - incompatible - - unavailable - - overloaded - - closed - - indeterminate - - stale - - skipped - - tombstoned - - absolute_expired - - name: certainty - cardinality_limit: 3 - allowed_values: [definite, not_applied, indeterminate] - percentiles: null - histogram_buckets: null - alert_severity_thresholds: - p1: "required coordination/session unavailable or indeterminate mutation sustained for 2m" - p2: "optional cache unavailable or overloaded above baseline for 5m" - owner_branch: redis-production-capability - log_field_mapping: [capability, role, operation, redis_outcome, certainty] - compatibility_impact: additive - required_test: contract-verification:metrics-cardinality - - # source: redis-production-capability Task 16 — monotonic semantic operation duration - - name: redis.capability.duration.seconds + # === Redis connection lanes === + # source: review 2026-11-03 RR-04-02 — the cache's conditional writes got their own lane + # (CACHE_SCRIPT) so that cache fills cannot take the SCRIPT lane's connections from idempotency, + # rate limiting and leases. These meters show, per lane, whether that holds. Emitted by + # redis-runtime RedisLaneMetrics for every lane RedisRuntimeOwner pools, only when the context has + # a MeterRegistry. Every counter series is registered at zero at startup. + - name: redis.lane.acquire.wait.seconds type: timer unit: seconds tags: - - name: capability - cardinality_limit: 6 - allowed_values: [cache, rate_limit, idempotency, efficiency_lease, session, runtime] - - name: role - cardinality_limit: 3 - allowed_values: [cache, coordination, session] - - name: operation - cardinality_limit: 24 - allowed_values: - - lookup - - record - - invalidate - - refresh_claim - - refresh_release - - rate_evaluate - - idempotency_claim - - idempotency_start - - idempotency_renew - - idempotency_complete - - idempotency_fail - - idempotency_release - - idempotency_inspect - - lease_acquire - - lease_inspect - - lease_renew - - lease_release - - session_create - - session_inspect - - session_save - - session_touch - - session_revoke - - session_rotate - - route_command - - name: redis_outcome - cardinality_limit: 15 - allowed_values: - - success - - hit - - miss - - denied - - contended - - conflict - - incompatible - - unavailable - - overloaded - - closed - - indeterminate - - stale - - skipped - - tombstoned - - absolute_expired + - name: lane + # The lane's name: REGULAR, SCRIPT (redis-runtime), CACHE_SCRIPT (cache-redis), BLOCKING, + # TRANSACTION, ADMIN (redis-sdk), plus any lane a service composes as a RedisLaneContribution. + cardinality_limit: 12 + - name: outcome + cardinality_limit: 2 + allowed_values: [admitted, refused] percentiles: [0.5, 0.9, 0.95, 0.99] histogram_buckets: slo_driven alert_severity_thresholds: - p2: "p99 approaches the configured command or caller deadline for 10m" + p2: "SCRIPT admitted p99 > 100ms for 10m (the correctness lane is queueing)" + p3: "CACHE_SCRIPT admitted p99 > 100ms for 30m" owner_branch: redis-production-capability - log_field_mapping: [capability, role, operation, redis_outcome] + log_field_mapping: [lane, outcome] compatibility_impact: additive required_test: contract-verification:metrics-cardinality - # source: redis-production-capability Task 16 — admission rejected before command ownership - - name: redis.capability.admission.rejected.total + - name: redis.lane.acquire.refused.total type: counter unit: total tags: - - name: role - cardinality_limit: 3 - allowed_values: [cache, coordination, session] - - name: admission - cardinality_limit: 2 - allowed_values: [rejected_saturated, rejected_closed] + - name: lane + # The lane's name: REGULAR, SCRIPT (redis-runtime), CACHE_SCRIPT (cache-redis), BLOCKING, + # TRANSACTION, ADMIN (redis-sdk), plus any lane a service composes as a RedisLaneContribution. + cardinality_limit: 12 + - name: reason + cardinality_limit: 4 + allowed_values: [saturated, closing, interrupted, connect_failed] percentiles: null histogram_buckets: null alert_severity_thresholds: - p1: "required role rejection sustained above zero for 2m" - p2: "optional cache saturation sustained for 5m" + p1: "SCRIPT or REGULAR saturated or connect_failed above zero for 2m while a correctness role is selected" + p2: "CACHE_SCRIPT saturated above baseline for 10m" owner_branch: redis-production-capability - log_field_mapping: [role, admission] + log_field_mapping: [lane, reason] compatibility_impact: additive required_test: contract-verification:metrics-cardinality - # source: redis-production-capability Task 16 — bounded admitted command count observation - - name: redis.capability.inflight.total + - name: redis.lane.connections.opened.total + type: counter + unit: total + tags: + - name: lane + # The lane's name: REGULAR, SCRIPT (redis-runtime), CACHE_SCRIPT (cache-redis), BLOCKING, + # TRANSACTION, ADMIN (redis-sdk), plus any lane a service composes as a RedisLaneContribution. + cardinality_limit: 12 + percentiles: null + histogram_buckets: null + alert_severity_thresholds: + p3: "opened rate after warm-up above baseline for 15m (reconnect churn against a slow Redis)" + owner_branch: redis-production-capability + log_field_mapping: [lane] + compatibility_impact: additive + required_test: contract-verification:metrics-cardinality + + - name: redis.lane.connections.discarded.total + type: counter + unit: total + tags: + - name: lane + # The lane's name: REGULAR, SCRIPT (redis-runtime), CACHE_SCRIPT (cache-redis), BLOCKING, + # TRANSACTION, ADMIN (redis-sdk), plus any lane a service composes as a RedisLaneContribution. + cardinality_limit: 12 + - name: reason + cardinality_limit: 2 + allowed_values: [invalidated, broken] + percentiles: null + histogram_buckets: null + alert_severity_thresholds: + p3: "invalidated rate above baseline for 15m (timed-out or failed calls discarding connections)" + owner_branch: redis-production-capability + log_field_mapping: [lane, reason] + compatibility_impact: additive + required_test: contract-verification:metrics-cardinality + + - name: redis.lane.connections.active type: gauge unit: total tags: - - name: role - cardinality_limit: 3 - allowed_values: [cache, coordination, session] - - name: state - cardinality_limit: 3 - allowed_values: [idle, active, saturated] + - name: lane + # The lane's name: REGULAR, SCRIPT (redis-runtime), CACHE_SCRIPT (cache-redis), BLOCKING, + # TRANSACTION, ADMIN (redis-sdk), plus any lane a service composes as a RedisLaneContribution. + cardinality_limit: 12 percentiles: null histogram_buckets: null alert_severity_thresholds: - p2: "saturated series remains nonzero for 5m" + p2: "SCRIPT active == configured maximum-script-connections for 5m" owner_branch: redis-production-capability - log_field_mapping: [role, state] + log_field_mapping: [lane] compatibility_impact: additive required_test: contract-verification:metrics-cardinality - # source: redis-production-capability Task 16 — observations of exact sanitized RoleHealth - - name: redis.capability.readiness.total - type: counter + - name: redis.lane.connections.pending + type: gauge unit: total tags: - - name: capability - cardinality_limit: 5 - allowed_values: [cache, rate_limit, idempotency, efficiency_lease, session] - - name: role - cardinality_limit: 3 - allowed_values: [cache, coordination, session] - - name: state - cardinality_limit: 3 - allowed_values: [available, unavailable, overloaded] - - name: reason - cardinality_limit: 11 - allowed_values: - - command_unavailable - - route_closed - - semantic_probe_succeeded - - semantic_read_write_failed - - semantic_program_acl_denied - - semantic_program_failed - - server_version_unsupported - - semantic_probe_in_progress - - semantic_observation_stale - - command_saturated - - recent_command_failure - - name: requirement - cardinality_limit: 2 - allowed_values: [optional, required] + - name: lane + # The lane's name: REGULAR, SCRIPT (redis-runtime), CACHE_SCRIPT (cache-redis), BLOCKING, + # TRANSACTION, ADMIN (redis-sdk), plus any lane a service composes as a RedisLaneContribution. + cardinality_limit: 12 percentiles: null histogram_buckets: null alert_severity_thresholds: - p1: "required coordination/session unavailable for 2m" - p2: "optional cache unavailable or overloaded for 5m" + p2: "SCRIPT pending > 0 for 5m" owner_branch: redis-production-capability - log_field_mapping: [capability, role, state, reason, requirement] - compatibility_impact: additive - required_test: contract-verification:metrics-cardinality - - # source: redis-production-capability Task 16 — bounded router shutdown drain result - - name: redis.capability.lifecycle.drain.total - type: counter - unit: total - tags: - - name: role - cardinality_limit: 3 - allowed_values: [cache, coordination, session] - - name: drain_outcome - cardinality_limit: 3 - allowed_values: [drained, forced_after_timeout, interrupted] - percentiles: null - histogram_buckets: null - alert_severity_thresholds: - p1: "required role forced_after_timeout or interrupted during shutdown" - p2: "optional cache forced close during shutdown" - owner_branch: redis-production-capability - log_field_mapping: [role, drain_outcome] + log_field_mapping: [lane] compatibility_impact: additive required_test: contract-verification:metrics-cardinality diff --git a/docs/registries/secrets-classification.yaml b/docs/registries/secrets-classification.yaml index cf814d6c..c10c2c2a 100644 --- a/docs/registries/secrets-classification.yaml +++ b/docs/registries/secrets-classification.yaml @@ -10,6 +10,9 @@ # (feature-secrets-config-source-contract 2026-05-22: "dev/local sentinel value prefix = __LOCAL_DEV_"). # - prod profile에서 `__LOCAL_DEV_` prefix 발견 시 startup fail # (feature-secrets-config-source-contract 2026-05-22). +# - prod profile에서 `fixture-` prefix 값(이 저장소에 적힌 lane fixture 비밀번호 — infra/redis-sdk/acl, +# compose lane의 REDIS_LANE_*)도 같은 이유로 startup fail. 어떤 변수 이름이든 검사한다 +# (review 2026-11-03 RR-07-04). # - Masking 기본 = `full_except_last_4` (feature-secrets-config-source-contract 2026-05-22: # "full mask except last 4 chars for non-secret tokens"). 진짜 secret(password/private key)은 `full`. # - Naming suffix는 보조 신호 (feature-secrets-config-source-contract: "_TOKEN, _KEY, _PASSWORD"). diff --git a/docs/reviews/2026-11-01-maturity-review/06-messaging.md b/docs/reviews/2026-11-01-maturity-review/06-messaging.md index e9557756..58f27bbf 100644 --- a/docs/reviews/2026-11-01-maturity-review/06-messaging.md +++ b/docs/reviews/2026-11-01-maturity-review/06-messaging.md @@ -39,8 +39,8 @@ | --- | --- | --- | --- | | techlead H10 | 자동조립 publisher가 관측을 기록하지 않음 | 해결 | `messaging-spring-boot-starter/.../MessagingPublishConfiguration.java:224-231` observation 주입 | | techlead H3 | consumer가 배선되지 않음 | 바뀜 | starter-kafka가 `MessageHandler` bean마다 consumer를 연다(`MessagingConsumeConfiguration.java:60-68`). 다만 application leaf 쪽 경로가 없다 → 06-08 | -| techlead M4 | 실행 경로에 스키마 호환성 검사 없음 | 바뀜 | 문서가 "선언만 하고 강제하지 않음"으로 표시(`docs/messaging/configuration-reference.md:123`). 앱 payload 버전에는 여전히 없음 → NN-10 | -| techlead M5 | outbox 두 개 공존 | 남음(소유자 결정으로 유지) | vendored `MessagingReliabilityAutoConfiguration`·`JdbcOutboxRepository`가 runtime classpath에 있음. 문서·종료 순서 비용 → 06-02, NN-11 | +| techlead M4 | 실행 경로에 스키마 호환성 검사 없음 | 바뀜 | 문서가 "선언만 하고 강제하지 않음"으로 표시(`docs/messaging/configuration-reference.md:123`). 앱 payload 버전에는 여전히 없음 → 06-10 | +| techlead M5 | outbox 두 개 공존 | 남음(소유자 결정으로 유지) | vendored `MessagingReliabilityAutoConfiguration`·`JdbcOutboxRepository`가 runtime classpath에 있음. 문서·종료 순서 비용 → 06-02, 06-11 | | techlead M7 / H-1b | producer-id 키 부재, 빈 값 통과 | 해결 | `messaging-kafka/src/main/resources/config/module-defaults/messaging.yml` `producer-id`, `MessagingBridgeRootAutoConfiguration.java:77-87` | | H-1a | bridge 조건이 platform보다 먼저 평가 | 해결 | `MessagingBridgeRootAutoConfiguration.java:32-33` `afterName` | | H-8 | bridge `prepare`가 모든 예외를 REJECTED로 | 해결 | `PlatformIntegrationEventPublishAdapter.java:59-67`, 테스트 `aFailureOtherThanAnIncompatibleValueIsNotTurnedIntoARefusal` | @@ -248,7 +248,7 @@ - 완료 기준: 앱 relay가 실제 Kafka로 발행한 레코드를 테스트가 읽어 envelope 바이트를 비교한다. 이 lane이 이름 있는 게이트에 들어 있다. - 검증 방법: 그 lane의 실행 로그. -### NN-10 — 이벤트 payload 버전을 올리는 방법과 호환성 검사가 없다 +### 06-10 — 이벤트 payload 버전을 올리는 방법과 호환성 검사가 없다 - 성격: 구조적 위험 - 관련 관점: 17, 11, 20 @@ -267,7 +267,7 @@ - 완료 기준: 필드 삭제 같은 깨지는 변경이 테스트에서 실패한다. - 검증 방법: 샘플 contract에 깨지는 v2를 넣는 변이 실험. -### NN-11 — outbox가 둘인데 문서와 주석은 하나처럼 말한다 +### 06-11 — outbox가 둘인데 문서와 주석은 하나처럼 말한다 - 성격: 개선 제안(이해 비용) - 관련 관점: 4, 5, 23, 24 @@ -285,7 +285,7 @@ - 완료 기준: `docs/messaging`에서 앱 outbox 문서로 가는 링크가 있고, 상태 대응표가 하나 있다. - 검증 방법: 문서 리뷰. -### NN-12 — 배치 크기와 lease 시간의 관계를 검사하지 않는다 +### 06-12 — 배치 크기와 lease 시간의 관계를 검사하지 않는다 - 성격: 구조적 위험 - 관련 관점: 12, 19, 21 @@ -300,7 +300,7 @@ - 완료 기준: lease를 넘을 수 있는 설정 조합이 기동에서 거부된다. - 검증 방법: `OutboxSettings` 바인딩 테스트에 경계 조합 추가. -### NN-13 — 이 영역 문서·주석의 낡은 설명 +### 06-13 — 이 영역 문서·주석의 낡은 설명 - 성격: 개선 제안 - 관련 관점: 24, 9, 26 @@ -317,7 +317,7 @@ - 완료 기준: 위 위치의 설명이 코드와 맞는다. - 검증 방법: 문서 리뷰. -### NN-14 — DLQ 격리 레코드가 모든 원인을 `POISON_MESSAGE`로 적는다 +### 06-14 — DLQ 격리 레코드가 모든 원인을 `POISON_MESSAGE`로 적는다 - 성격: 개선 제안 - 관련 관점: 8, 28 @@ -355,7 +355,7 @@ - claim은 `FOR UPDATE SKIP LOCKED`, 인스턴스마다 UUID owner, row마다 token 증가(`JpaOutboxStore.java:105-127`). - 완료는 owner·token이 맞을 때만 적용된다. 아니면 STALE(`:136-162`). `PostgreSqlOutboxConcurrentRelayIntegrationTest`가 있다. - 같은 aggregate의 두 row는 두 인스턴스가 동시에 가져갈 수 없다. 앞 row가 PUBLISHED가 아니면 뒤 row는 gate에 걸린다. -- 남은 위험: 배치 시간이 lease를 넘는 설정(NN-12). +- 남은 위험: 배치 시간이 lease를 넘는 설정(06-12). ### 4.4 외부 응답 지연·유실 (Kafka 느림·정지) @@ -373,7 +373,7 @@ - outbox row는 append 때의 바이트를 그대로 내보낸다. 배포가 바뀌어도 옛 row의 봉투는 변하지 않는다. - destination 선언이 새 배포에서 빠지면 옛 row는 `PUBLISH_PREPARATION_FAILED` REJECTED → DEAD가 된다. 이것은 설정 오류라 확정 거부로 보는 것이 맞다. -- payload 버전 호환성 검사와 절차가 없다(NN-10). +- payload 버전 호환성 검사와 절차가 없다(06-10). ### 4.7 선택 어댑터 제거 @@ -416,7 +416,7 @@ 1. 필요한 것: application은 검증된 봉투를 같은 트랜잭션에 기록하는 outbox가 필요하다. platform은 제품으로서 자기 outbox를 가진다. 2. 지킬 성질: 개발자가 어느 outbox가 동작 중인지 헷갈리지 않는다. platform의 종료·지표 설계가 앱 경로에도 맞는다. -3. 현재 보장: bean 조건으로 vendored relay가 앱에서 생기지 않는다. 문서·종료 순서는 vendored relay 기준이다(06-02, NN-11). +3. 현재 보장: bean 조건으로 vendored relay가 앱에서 생기지 않는다. 문서·종료 순서는 vendored relay 기준이다(06-02, 06-11). 4. 대안: (a) 결정 유지 + 경계 표시 보강, (b) vendored outbox를 starter `implementation`에서 빼서 opt-in 의존으로, (c) 하나로 합치기. 5. 비용: (a) 문서·주석 수정. (b) starter 의존 정리와 vendored 테스트 classpath 조정. (c) 결정 번복. 6. 선택: (a)를 먼저, (b)는 다음 단계 후보. 결정은 유지한다. @@ -445,7 +445,7 @@ - 이벤트 소비(handler 위치, inbox 조립) 예제와 leaf(06-08). - aggregate 순서 번호를 만드는 올바른 예(06-07). -- payload 버전 올리기 절차와 호환성 테스트(NN-10). +- payload 버전 올리기 절차와 호환성 테스트(06-10). - 앱 relay를 실제 Kafka로 돌리는 테스트 틀(06-09). - `MessagePublisher` fake와 본체의 공통 계약(TEST-FAKE-MessagePublisher). @@ -457,7 +457,7 @@ 4. payload를 바꾸는가? 바꾸면 새 `payloadVersion`을 추가하고 소비자가 먼저 두 버전을 읽게 한다. 5. 이 destination의 소비자는 부작용이 있는가? 있으면 INBOX_TRANSACTIONAL을 쓴다. 6. 한 aggregate가 초당 몇 건을 만드는가? 0.2건/초를 넘으면 현재 relay 기본값으로는 지연이 쌓인다(06-04). -7. relay를 몇 개 인스턴스에서 켤 것인가? batch-size를 올리면 lease 시간과 함께 계산한다(NN-12). +7. relay를 몇 개 인스턴스에서 켤 것인가? batch-size를 올리면 lease 시간과 함께 계산한다(06-12). ## 8. 28개 관점 판정표 @@ -467,27 +467,27 @@ | --- | --- | --- | --- | | 1 | 코드 성숙도 | P | 실패 경로를 의식한 코드가 많다. platform과 application 사이 실패 의미 번역에 결함(06-01, 06-06) | | 2 | 구조적 성숙도 | P | 발행 경로는 port → bridge → platform으로 깨끗하다. 소비 경로에 application leaf가 없다(06-08) | -| 3 | 변경 용이성 | P | 새 이벤트 추가 경로는 짧다. payload 버전 변경 경로가 없다(NN-10) | -| 4 | 인지 부하 | P | 두 outbox 모델과 반대 뜻의 상태 이름(NN-11) | +| 3 | 변경 용이성 | P | 새 이벤트 추가 경로는 짧다. payload 버전 변경 경로가 없다(06-10) | +| 4 | 인지 부하 | P | 두 outbox 모델과 반대 뜻의 상태 이름(06-11) | | 5 | 위험 가시성 | P | 중복 가능 구간은 README에 있다. "REJECTED = 영구"라는 가정이 코드에서 보이지 않는다(06-01) | | 6 | 실패 성숙도 | X | 일시 거부·종료·짧은 장애가 영구 DEAD로 이어진다(06-01, 06-02, 06-03) | | 7 | 운영 성숙도 | P | runbook·경보·구조화 ERROR가 있다. relay 꺼짐·실패 중 지표 정지(06-05), runbook 조회가 옛 키 | -| 8 | 디버깅 가능성 | P | failure report에 event·aggregate·correlation id가 있다. DLQ 사유가 한 값(NN-14) | -| 9 | 유지보수 성숙도 | P | 주석·README에 낡은 위치 설명(NN-13) | +| 8 | 디버깅 가능성 | P | failure report에 event·aggregate·correlation id가 있다. DLQ 사유가 한 값(06-14) | +| 9 | 유지보수 성숙도 | P | 주석·README에 낡은 위치 설명(06-13) | | 10 | 테스트 성숙도 | P | 단위 테스트 촘촘하고 빠름(합계 수 초). 앱 경로 실연 테스트 없음(06-09), 지표 변이 생존(06-05) | | 11 | 데이터 성숙도 | P | 봉투 바이트·hash 보존이 좋다. 순서 키 유일 제약 없음(06-07) | -| 12 | 동시성 성숙도 | P | SKIP LOCKED + fencing은 좋다. 배치·lease 관계 미검사(NN-12), 종료 순서(06-02) | +| 12 | 동시성 성숙도 | P | SKIP LOCKED + fencing은 좋다. 배치·lease 관계 미검사(06-12), 종료 순서(06-02) | | 13 | 확장성 | P | 인스턴스 수평 확장은 된다. aggregate당 처리량 상한(06-04) | | 14 | 성능 성숙도 | ? | 계산만 했다. 실측 없음. 회차당 약 43 DB 문장, 인스턴스당 최대 약 4건/초 | | 15 | 보안 성숙도 | O | Kafka TLS/SASL·ACL 적용 확인(M-2 해결). 실패 report에 payload를 싣지 못하게 타입으로 막음 | | 16 | 의존성 성숙도 | P | broker SDK는 선택 줄을 지우면 빠진다. starter가 쓰지 않는 vendored outbox 구현을 `implementation`으로 싣는다 | -| 17 | API / Contract 성숙도 | P | 봉투 v1 schema와 hash 검사가 있다. EventId/MessageId 불일치(06-06), payload 호환성 없음(NN-10) | +| 17 | API / Contract 성숙도 | P | 봉투 v1 schema와 hash 검사가 있다. EventId/MessageId 불일치(06-06), payload 호환성 없음(06-10) | | 18 | 배포 성숙도 | X | 배포 종료 중 row가 DEAD가 될 수 있다(06-02). broker 실연 lane이 기본 게이트 밖(06-09) | -| 19 | 설정 성숙도 | P | strict 바인딩, producer-id 필수. relay 기본 꺼짐의 결과가 안 보임(06-05), 값 사이 관계 미검사(NN-12) | -| 20 | 개발자 경험 | P | 발행 한 줄 켜기가 쉽다. 소비·버전·순서 번호 안내가 없다(06-07, 06-08, NN-10) | +| 19 | 설정 성숙도 | P | strict 바인딩, producer-id 필수. relay 기본 꺼짐의 결과가 안 보임(06-05), 값 사이 관계 미검사(06-12) | +| 20 | 개발자 경험 | P | 발행 한 줄 켜기가 쉽다. 소비·버전·순서 번호 안내가 없다(06-07, 06-08, 06-10) | | 21 | Guardrail 성숙도 | P | caller 트랜잭션 강제, 대역 publisher 금지, producer-id 필수가 좋다. 이벤트 id·순서 번호·설정 조합 가드 없음 | | 22 | 추상화 성숙도 | P | bridge는 필요한 만큼만 안다. outcome 어휘가 platform 의미 하나를 담지 못한다(06-01) | -| 23 | 일관성 | P | platform 종료 설계와 앱 relay 방식이 다르다(06-02, NN-11) | +| 23 | 일관성 | P | platform 종료 설계와 앱 relay 방식이 다르다(06-02, 06-11) | | 24 | 네이밍 / 의미 성숙도 | P | `OutboxLeaderElectionToken`이 하지 않는 일을 이름으로 말한다. 두 outbox의 FAILED 의미 반대 | | 25 | 조직 확장성 | P | 발행은 팀별로 독립 추가 가능. 소비는 registry 간선을 각자 만들게 된다(06-08) | | 26 | Ownership 성숙도 | P | leaf README가 근거를 잘 남긴다. composition·messaging-kafka에 `CLAUDE.md` 없음 | diff --git a/docs/reviews/2026-11-01-maturity-review/README.md b/docs/reviews/2026-11-01-maturity-review/README.md index a35d064a..958b7158 100644 --- a/docs/reviews/2026-11-01-maturity-review/README.md +++ b/docs/reviews/2026-11-01-maturity-review/README.md @@ -196,7 +196,9 @@ ## 12. 발견 색인 -영역 문서에 36.3 양식으로 적힌 발견 119건: High 21 · Medium 64 · Low 34, P1 25 · P2 59 · P3 35. Critical 0. 두 영역이 같은 결함을 적은 묶음(07-03+09-03, 02-03+07-05, 08-01+04-04+03-12, 05-09+08-04, 04-09+09-01)은 색인에 각각 남기고 2절에서 합쳤다. 06·08 영역 문서는 원래 `NN-` 접두사로 적혀 있어 `06-`·`08-`로 바꿨다. +영역 문서에 36.3 양식으로 적힌 발견 **124건**: High 21 · Medium 67 · Low 36, P1 25 · P2 60 · P3 39. + +> **정정 (2026-11-03 재리뷰)**: 처음 이 색인은 119건이었다. 06 영역 문서가 `NN-` 접두사로 쓰여 있었는데 오케스트레이터가 `NN-0x`만 `06-0x`로 바꿔 `NN-10`~`NN-14` 5건이 색인과 모든 수정 묶음에서 빠졌다. 13~16절의 "모든 발견 처리"는 이 5건에 대해 틀렸다. 5건은 `06-10`~`06-14`로 다시 매겨 아래 표에 넣었고, 재리뷰 수정(`../2026-11-03-rereview/README.md` 6절)에서 고쳤다. Critical 0. 두 영역이 같은 결함을 적은 묶음(07-03+09-03, 02-03+07-05, 08-01+04-04+03-12, 05-09+08-04, 04-09+09-01)은 색인에 각각 남기고 2절에서 합쳤다. 06·08 영역 문서는 원래 `NN-` 접두사로 적혀 있어 `06-`·`08-`로 바꿨다. | ID | 심각도 | 우선순위 | 내용 | |---|---|---|---| @@ -319,6 +321,11 @@ | [09-09](09-performance-measurement.md) | Low | P3 | persistence-jpa가 H2 드라이버를 운영 아티팩트에 실어 보냄 | | [09-10](09-performance-measurement.md) | Low | P3 | `local` 프로필이 `server.port`를 8080으로 고정 | | [09-11](09-performance-measurement.md) | Low | P3 | 기본 조합 jar에 Jackson 2·3과 kotlin-stdlib가 함께 실림 | +| [06-10](06-messaging.md) | Medium | P2 | 이벤트 payload 버전을 올리는 방법과 호환성 검사가 없다 (색인 누락 정정) | +| [06-11](06-messaging.md) | Medium | P3 | outbox가 둘인데 문서와 주석은 하나처럼 말한다 (색인 누락 정정) | +| [06-12](06-messaging.md) | Medium | P3 | 배치 크기와 lease 시간의 관계를 검사하지 않는다 (색인 누락 정정) | +| [06-13](06-messaging.md) | Low | P3 | 이 영역 문서·주석의 낡은 설명 (색인 누락 정정) | +| [06-14](06-messaging.md) | Low | P3 | DLQ 격리 레코드가 모든 원인을 `POISON_MESSAGE`로 적는다 (색인 누락 정정) | ## 13. 조치 현황 (개선 순서 1~3) diff --git a/docs/reviews/2026-11-03-rereview/README.md b/docs/reviews/2026-11-03-rereview/README.md index b6dee03d..484d3913 100644 --- a/docs/reviews/2026-11-03-rereview/README.md +++ b/docs/reviews/2026-11-03-rereview/README.md @@ -67,3 +67,46 @@ Minor 41건은 각 문서에 있다. ## 5. 다음 위 Important 전부, Minor 41건, 06-10~14를 영역별 수정 묶음으로 고친다. 각 묶음은 `5786126c`에서 만든 worktree에서 RED→GREEN, 오케스트레이터가 합쳐 전체 게이트 후 커밋·push한다. 결과는 이 문서 6절. + +## 6. 수정 결과 (2026-11-03) + +영역별 수정 묶음 7개가 `5786126c`(r4-G는 A·B·C·D·F를 합친 임시 기준)에서 `test-driven-development` 스킬 절차로 RED→GREEN 했고, 오케스트레이터가 합쳤다. 하위 에이전트 없음. + +| 묶음 | 상태 | 검증 (에이전트) | 처리 내용 | 결정·합칠 때 조치·남은 것 | +|---|---|---|---|---| +| r4-a | 완료 | leaf check 9, cache-redis test, architectureCheck, jpaPlatformContractTest fileserver 47, outbox sequence PG, verifyFeatureScaffold, 소유자 규칙 변이 잡힘 | RR-01-02=RR-07-01 @SystemInvoked bean 기동 실패+ArchUnit, RR-01-01·07·10 생성 명령, RR-01-03·09 매뉴얼·IntegrationEventDrafts ArchUnit, RR-01-04+RR-07-06 fileserver 계약 6개·fake 정합·소유자 정본 FileAccessSubject.owns(RED 17), RR-01-05=RR-04-04 move 쿼터(RED 2), RR-01-06 순서 계약 보강, RR-01-08 | 합칠 때 매뉴얼 09장·OutboxAggregateSequenceContract를 r4-c와 수동 병합(r4-a 새 케이스는 첫 번호 기준 상대값이라 floor와 충돌 없음). 결정: 만료·미해제 예약 확정은 받음, move는 복사 후 삭제 안 씀 | +| r4-b | 완료(sample 데모 체인 한 줄은 r4-f로) | leaf check 6, graphql·web systemTest, webJettyCompat, -p optional-platforms ci integrationCheck(ledger PG 41), architectureCheck, verifyEnvKeys, documentContractTest, verifyPublicPathSnapshot, 영향 모듈 test | RR-02-01 union 복잡도, RR-02-02 EnvelopeErrorController+ERROR 디스패치만 허용(JWT 체인 401→400), RR-02-03 IDEMPOTENCY_COMPLETION_UNKNOWN(503 재시도 가능), RR-02-04 GrpcClaimToken·claim_attempt·V003(옛 owner commit 거부·실패 무시, 조건부 commit), RR-02-05 Retry-After 상수·lease 이하면 기동 실패, RR-02-06~08 | 결정: 새 코드 503(code로 구분), 시도 번호는 카운터, ERROR 디스패치 허용(보안 정책 변경) | +| r4-c | 완료 | leaf check 5, notification test, sample·app-bootstrap architectureTest, architectureCheck, 실 PG jpaPlatformContract 334·Migration 45·Failure 6·V21·runbook·트랜잭션 lane 등 skip 0 | RR-03-01 counter ON CONFLICT(RED 실 PG), RR-03-02=RR-05-02 V21 floor 테이블+counter 미리 생성+행 최대값 하한(RED 3), RR-03-05 lock_timeout 5s·무효 인덱스 거부·CONCURRENTLY 절차 runbook, RR-03-03 마감 초과 분류(H2+실 PG), RR-03-04 양쪽 매핑 양방향 금지, RR-03-06~10 | 결정: V21 직접 수정(아직 어디에도 미적용 — 적용한 개발 DB는 재생성), H2 호환 위해 다문장 counter, 계약 '1부터'→'첫 번호부터 1씩'(JPA 첫 번호 ≈1.7e15) | +| r4-d | 완료(RR-04-02 지표·RR-07-04 운영 거부 남음) | leaf check 6, Redis container 52·topology standalone, Toxiproxy 4, spring70 compat, perf, notification 문서·환경 계약, PG 멱등성 계약 14, verifyEnvKeys, architectureCheck, verifyStrictLaneStages, compose 정적 18 | RR-04-01 호출자 deadline은 breaker 미집계(RED), RR-04-03 JDK 스트리밍 read-idle(RED), RR-04-02 CacheScriptLane(기본 4), RR-04-05 lane 연결, RR-04-06~08·10·11, RR-07-03 SNS 4xx 거절+감사(RED), RR-07-04 ACL 파일 이름 lane-fixture-accounts.acl, RR-07-07 SNS MessageId 기준 | 결정: total-call이 끊은 시도는 실패로 셈, 캐시 lane 4. 남음: lane별 지표(meter registry 없음), 운영 SecretSourceValidator가 fixture- 값 거부 | +| r4-e | 완료 | 건드린 leaf 13 check, Kafka relay·inbox, outbox PG 6·migration·jpaPlatformContract, architectureCheck, verifyDependencyLocks, verifyStrictLaneStages, documentContractTest | RR-05-01 재발행 SQL deferred 초기화+runbook SQL lane(RED 실 PG), RR-05-03 last_failure_code(V22)·오래 재시도 게이지·PARKED·runbook SQL(RED 4), RR-05-04 draft가 비UUIDv7 거부, RR-05-05~09, 06-10 schema 버전 해시 고정, 06-11~14 | 합칠 때 4곳 수동 병합: OrderKeyRunbook IT를 공용 OutboxRunbookSql로 + V22 추가로 latest가 22가 되어 'V21 적용' 단언을 applied 목록 기준으로, sha256에 V22, application-core README, 매뉴얼 09. 결정: PARKED는 DDL 없이, 발행 포트·저장소 포트 시그니처 변경(파생 서비스 깨짐), 오래 재시도 기준 10회, DB CHECK 제약 안 넣음. 남음: 2026-11-01 README 색인 정정(오케스트레이터), V21 주석의 없는 클래스명 | +| r4-f | 완료 | build-logic 107, ci, app-bootstrap documentContract·systemTest, sample check·webSystemTest·integrationTest·posterImageMigration, -p optional-platforms ci, 두 빌드 verifyStrictLaneStages, verifyEnvKeys, compose 정적 | RR-06-01=RR-04-09 규칙 생성 중 다른 singleton 생기면 기동 실패(RED), RR-06-02 lane 정책을 Test 입력으로(RED), RR-06-03·04 optional 빌드 단계 검사·-x/continue-on-error/if 해석(RED), RR-07-02 데모 계정 허용 목록 local·test·demo(RED 6), RR-06-05+07-05 CSRF 제외 축소·sample qualityCheck, RR-06-06~12, RR-02-02 샘플 체인 ERROR 디스패치 허용 | 남음: cache-redis 컨테이너 lane main 단계 중복 실행. 결정: release job --rerun-tasks 안 넣음 | +| r4-g | 완료 | redis-runtime 64·redis-sdk 310·cache-redis 192·app-bootstrap test 222·systemTest 11·documentContract 60, redisContainerContract 53, topology standalone, verifyEnvKeys, architectureCheck, verifyDependencyLocks, verifyStrictLaneStages | RR-04-02 redis.lane.* 지표 6개(RED 10), RR-07-04 prod에서 fixture- 값 거부(RED), cache-redis 컨테이너 lane main 중복 제거 | 합친 뒤 오케스트레이터: r4-g가 찾은 '레지스트리의 redis.capability.* 6개를 아무 코드도 내보내지 않음' → 레지스트리에서 빼고 incident runbook을 redis.lane.* 기준으로, cache-redis README에 미구현 지표로 기록. 남음(의도): httpclient failure injection lane main 2회(05-06/05-16 결정) | + +### 6.1 합칠 때 오케스트레이터가 고친 것 + +- 수동 병합: 매뉴얼 09장(r4-A·r4-C·r4-E), `OutboxAggregateSequenceContract`(r4-A·r4-C), `PostgreSqlOutboxOrderKeyRunbookIntegrationTest`(r4-C·r4-E — 공용 `OutboxRunbookSql`로, V22가 생겨 "V21 적용" 단언을 적용 목록 기준으로), `applied-migrations.sha256`(V22), application-core README. +- r4-G가 찾은 거짓 선언: `docs/registries/metrics.yaml`의 `redis.capability.*` 6개를 어떤 코드도 내보내지 않았고 incident runbook이 그것을 보라고 했다. 레지스트리에서 빼고 runbook을 실제로 나오는 `redis.lane.*` 기준으로 고쳤다. 원래 설계는 cache-redis README 「아직 내보내지 않는 지표」. +- 최종 게이트에서 드러난 2건: (1) sample `CreatePosterConcurrencyIntegrationTest`가 v4 event id를 써서 RR-05-04 이후 16건 모두 거부 → 운영과 같은 `UuidOutboxEventIdFactory`로. (2) `EnvelopeErrorController`의 `@RequestMapping`이 FindSecBugs `SPRING_CSRF_UNRESTRICTED_REQUEST_MAPPING` → 오류 디스패치 전용(보안 체인이 ERROR dispatch만 허용)이라 그 메서드에만 이유를 단 제외. +- 원래 리뷰 README 12절 색인에 06-10~06-14를 넣고 발견 수를 124건으로 정정했다. + +## 7. 게이트 (2026-11-03, 합친 메인 작업 트리, 공유 잠금) + +첫 실행(22:27)은 `ci`(FindSecBugs 1건)와 `integrationCheck`(sample 동시 생성 테스트)가 실패했고 6.1절대로 고쳤다. 재실행 결과: + +| 게이트 | 결과 | 시간 | 실행 직전 부하 | +|---|---|---|---| +| `ci` | rc=0 | 215초 | load 8.27 | +| `:integrationCheck` | rc=0 | 1,647초(부하 높음) | load 29.15 | +| `mainCheck` | rc=0 | 260초 | load 35.16 | +| `-p optional-platforms ci integrationCheck performanceCheck` | rc=0 | 62초 | load 16.51 (첫 실행, 두 수정과 무관) | +| `:performanceCheck` | rc=0 | 25초 | load 20.55 (첫 실행) | +| `:releaseCheck -PgitRevision=` | rc=0 | 4초 | 첫 실행 | +| `scripts/verify-compose-profile-contracts.sh` | rc=0, 18 lane | — | 첫 실행 | + +## 8. 남은 것 + +- **의도적으로 둔 것**: httpclient failure-injection lane이 main 단계에서 두 번 돈다(원래 리뷰 05-06·05-16 결정). capability 단위 Redis 지표(`redis.capability.*`)는 설계만 있고 구현하지 않았다(새 기능 범위, cache-redis README). +- **소유자 확인이 필요한 결정**: 4절 표, 그리고 묶음이 기본값으로 정한 것(6절 "결정" 칸) — 특히 서블릿 오류 디스패치 허용(보안 정책), 데모 계정 허용 목록, V21을 V22 없이 직접 수정(이 브랜치의 V21을 적용한 개발 DB는 재생성), 발행·저장소 포트 시그니처 변경, `IDEMPOTENCY_COMPLETION_UNKNOWN` 신설, prod에서 `fixture-` 값 거부. +- **짝 프론트**: `IDEMPOTENCY_COMPLETION_UNKNOWN`(503, 결과 불명)을 인계 문서에 더했다. +- **이 환경에서 확인할 수 없는 것**: 실제 CI runner 실행. + diff --git a/docs/runbooks/grpc-platform-operations.md b/docs/runbooks/grpc-platform-operations.md index 0b632126..aa896ec7 100644 --- a/docs/runbooks/grpc-platform-operations.md +++ b/docs/runbooks/grpc-platform-operations.md @@ -44,7 +44,9 @@ the time, and which half is not knowable from the status. and a later reconciler pass sees that attempt's `COMMITTED` or `FAILED_TERMINAL`. Until a retry arrives, the case stays pending. The re-run is safe only when the ledger commit is recorded in the mutation's own transaction; if this service commits them separately, check the business resource - before letting the retry through. + before letting the retry through. An owner that was only slow cannot spoil the takeover: its claim + attempt (`claim_attempt`) is behind, so its late commit is refused and its late failure ignored. + A log line saying an operation "is no longer held by attempt N" is that refusal, not an outage. **Escalate when** the reconciler's pending list grows across passes. That means the ledger is unreachable rather than slow. A case pending as `IN_PROGRESS` for longer than the claim TTL with no diff --git a/docs/runbooks/idempotency-unavailable.md b/docs/runbooks/idempotency-unavailable.md index f330c93f..423a05cc 100644 --- a/docs/runbooks/idempotency-unavailable.md +++ b/docs/runbooks/idempotency-unavailable.md @@ -1,10 +1,10 @@ --- title: Runbook — IDEMPOTENCY_UNAVAILABLE (멱등성 조정 저장소 사용 불가) category: TRANSIENT_DEPENDENCY -error_codes: [IDEMPOTENCY_UNAVAILABLE, IDEMPOTENCY_RECOVERY_REQUIRED] +error_codes: [IDEMPOTENCY_UNAVAILABLE, IDEMPOTENCY_COMPLETION_UNKNOWN, IDEMPOTENCY_RECOVERY_REQUIRED] severity: P2 owner: oncall -last_updated: 2026-09-28 +last_updated: 2026-11-03 status: active --- @@ -17,6 +17,10 @@ status: active - `IdempotencyExecutor` refused to run the action because the replay store (JPA or Redis, whichever `app.idempotency` selects) did not answer. Nothing was executed: the refusal happens before the action, so a client retry is safe. +- HTTP 503 `IDEMPOTENCY_COMPLETION_UNKNOWN` (`retryable=true`) is the same outage hitting after the + action ran: the store did not answer while the result was being recorded. Effects in the action's + own transaction rolled back; anything else it did (an outbound call, a message) may have happened. + This is not "nothing was executed". Clients resend with the same key. - HTTP 500 `IDEMPOTENCY_RECOVERY_REQUIRED` (`retryable=false`) means the opposite case: the store cannot tell whether an earlier attempt's external effect already happened. @@ -32,6 +36,10 @@ status: active ## Action - `IDEMPOTENCY_UNAVAILABLE`: restore the store; clients retry with the same key and backoff. +- `IDEMPOTENCY_COMPLETION_UNKNOWN`: restore the store, then check the effects outside the + transaction for the failed requests (the ERROR log line names the code). A same-key resend replays + a result that was in fact recorded, gets `IDEMPOTENT_IN_FLIGHT` while the attempt's lease lasts, + and `IDEMPOTENCY_RECOVERY_REQUIRED` after it — handle that one as below. - `IDEMPOTENCY_RECOVERY_REQUIRED`: do not ask the client to retry blindly. Confirm whether the downstream effect of the earlier attempt exists, then complete or release the record so the next request with that key replays or re-executes deterministically. diff --git a/docs/runbooks/job-timeout.md b/docs/runbooks/job-timeout.md index a474ab92..4c239492 100644 --- a/docs/runbooks/job-timeout.md +++ b/docs/runbooks/job-timeout.md @@ -4,7 +4,7 @@ category: TRANSIENT_DEPENDENCY error_codes: [JOB_TIMEOUT] severity: P2 owner: oncall -last_updated: 2026-06-13 +last_updated: 2026-11-03 status: stub --- @@ -48,7 +48,10 @@ status: stub ## 5. Escalation - 의존성 지연이 근본 원인이면 해당 의존성 오너에게 escalate -- shutdown 예산(20s) vs k8s `terminationGracePeriodSeconds`(30s) 정합 이슈면 플랫폼/런타임 오너에게 escalate +- 종료 예산은 차례로 더한 값이다: 웹 drain(`APP_SERVER_SHUTDOWN_TIMEOUT`, 기본 30s)이 끝난 뒤 async 대기 19s, + 합계 49s에 preStop(5s)과 여유를 더한다. k8s `terminationGracePeriodSeconds`가 이보다 작으면(예: 기본 30s) + async 대기 중 SIGKILL이다. compose는 60s(`docker-compose.yml` `stop_grace_period`, 합계는 + `DeveloperExperienceContractTest`가 검사). 정합 이슈면 플랫폼/런타임 오너에게 escalate ## 6. Recovery / Verification diff --git a/docs/runbooks/outbox-dead-letter.md b/docs/runbooks/outbox-dead-letter.md index e926abea..f0d212bd 100644 --- a/docs/runbooks/outbox-dead-letter.md +++ b/docs/runbooks/outbox-dead-letter.md @@ -4,7 +4,7 @@ category: INTERNAL error_codes: [OUTBOX_DEAD_LETTER] severity: P1 owner: oncall -last_updated: 2026-06-11 +last_updated: 2026-11-03 status: stub --- @@ -21,7 +21,12 @@ status: stub ## 2. First Response (5분 이내) ### Step 1 — 확인 -1. ERROR log에서 `OUTBOX_DEAD_LETTER` 라인 확인: `event_id`, `event_type`, `correlation_id`, 마지막 실패 원인 예외 추출 +1. ERROR log에서 `OUTBOX_DEAD_LETTER` 라인 확인: `event_id`, `event_type`, `correlation_id`, `error.reason`(원인 + 코드 — publisher가 붙인 코드, 없으면 거부 결과 이름, 분류되지 않은 예외면 그 타입 이름) 추출. 같은 값이 행의 + `last_failure_code`에 남는다. ERROR가 없고 WARN `outbox claim moved a row ... to DEAD`만 있으면 relay가 아니라 + claim이 행을 DEAD로 옮긴 것이다 — `error.reason`이 `ATTEMPT_COUNTERS_INCONSISTENT`면 `deferred_attempt_count`를 + 되돌리지 않은 수동 재발행이 원인이니 §4의 SQL로 다시 재발행한다. `UNREADABLE_ROW`면 행 자체가 event로 읽히지 + 않는다(canonical 봉투 없음, 저장된 해시와 바이트 불일치) 2. DB에서 DEAD row 확인: `SELECT * FROM outbox_event WHERE status = 'DEAD' ORDER BY occurred_at;` 3. **차단 영향 파악 (중요)**: strict per-aggregate FIFO 정책상 DEAD row는 같은 aggregate(`tenant_scope`, `aggregate_type`, `aggregate_id`)의 후행 이벤트를 계속 차단함. 순서는 `(aggregate_sequence, event_index)`로 비교한다(claim SQL `PostgreSqlOutboxClaimRepository`와 같은 키) — `SELECT count(*) FROM outbox_event b WHERE b.status <> 'PUBLISHED' AND EXISTS (SELECT 1 FROM outbox_event d WHERE d.status='DEAD' AND d.tenant_scope=b.tenant_scope AND d.aggregate_type=b.aggregate_type AND d.aggregate_id=b.aggregate_id AND (d.aggregate_sequence < b.aggregate_sequence OR (d.aggregate_sequence = b.aggregate_sequence AND d.event_index < b.event_index)));` @@ -42,14 +47,35 @@ status: stub ## 4. Mitigation (수동 처분 — 둘 중 하나) -- **재발행 (기본)**: 원인 해소 후 해당 row를 다시 claim 가능 상태로 되돌림 — - `UPDATE outbox_event SET status = 'PENDING', attempt_count = 0, next_attempt_at = now() WHERE event_id = '' AND status = 'DEAD';` - 장애 구간 전체를 한 번에 되돌리려면(원인이 해소된 것을 확인한 뒤): - `UPDATE outbox_event SET status = 'PENDING', attempt_count = 0, next_attempt_at = now() WHERE status = 'DEAD' AND occurred_at BETWEEN '<장애 시작>' AND '<장애 끝>';` (outbox_event에는 DEAD가 된 시각 컬럼이 없어 이벤트 발생 시각으로 고른다 — 먼저 같은 조건의 SELECT로 대상을 확인한다) - (consumer는 at-least-once + idempotencyKey dedupe 의무가 있으므로 중복 발행은 안전) -- **skip (영구 폐기)**: 이벤트가 더 이상 유효하지 않으면 PUBLISHED로 마킹해 FIFO 차단을 해제 — - `UPDATE outbox_event SET status = 'PUBLISHED' WHERE event_id = '' AND status = 'DEAD';` - ⚠️ skip은 다운스트림에 영구 이벤트 갭을 만든다 — 비즈니스 오너 승인 후에만 수행하고 incident 기록에 남김 +**재발행 (기본)**: 원인 해소 후 해당 row를 다시 claim 가능 상태로 되돌린다. 시도 횟수 두 개 +(`attempt_count`, `deferred_attempt_count`)를 함께 0으로 되돌린다. `attempt_count`만 되돌리면 다음 claim에서 +미룬 claim 수가 전체 claim 수보다 많아지고, claim은 그 행을 읽지 않고 다시 DEAD로 옮긴다(재리뷰 2026-11-03 +RR-05-01). + + +```sql +UPDATE outbox_event SET status = 'PENDING', attempt_count = 0, deferred_attempt_count = 0, next_attempt_at = now() WHERE event_id = '' AND status = 'DEAD'; +``` + +장애 구간 전체를 한 번에 되돌리려면(원인이 해소된 것을 확인한 뒤). outbox_event에는 DEAD가 된 시각 +컬럼이 없어 이벤트 발생 시각으로 고른다 — 먼저 같은 조건의 SELECT로 대상을 확인한다. + + +```sql +UPDATE outbox_event SET status = 'PENDING', attempt_count = 0, deferred_attempt_count = 0, next_attempt_at = now() WHERE status = 'DEAD' AND occurred_at BETWEEN '<장애 시작>' AND '<장애 끝>'; +``` + +consumer는 at-least-once + idempotencyKey dedupe 의무가 있으므로 중복 발행은 안전하다. + +**skip (영구 폐기)**: 이벤트가 더 이상 유효하지 않으면 PUBLISHED로 마킹해 FIFO 차단을 해제한다. +⚠️ skip은 다운스트림에 영구 이벤트 갭을 만든다 — 비즈니스 오너 승인 후에만 수행하고 incident 기록에 +남긴다. + + +```sql +UPDATE outbox_event SET status = 'PUBLISHED' WHERE event_id = '' AND status = 'DEAD'; +``` + - 장기: poison event 재발 방지(payload 계약 테스트 보강), DEAD 빈발 event_type의 producer 검증 강화 ## 5. Escalation @@ -67,7 +93,10 @@ status: stub - error-codes.yaml rows: `OUTBOX_DEAD_LETTER` (INTERNAL, retryable=false) - metrics.yaml: `outbox.publisher.published.total{outcome=DEAD}`, `outbox.pending.size{status=DEAD}`, `outbox.publisher.lag` -- 코드: `application-core` `PublishPendingOutboxEventsUseCase`(FAILED→DEAD 전이), `adapter-persistence` `outbox/OutboxEventJpaRepository`(FIFO 게이트 — DEAD가 후행 차단) +- 코드: `application-core` `PublishPendingOutboxEventsUseCase`(FAILED→DEAD 전이), `adapter:outbound:persistence-jpa` `outbox/postgresql/PostgreSqlOutboxClaimRepository`(FIFO 게이트 — DEAD가 후행 차단), `outbox/jpa/JpaOutboxStore#claimBatch`(읽을 수 없는 행을 claim 때 DEAD로) +- §4의 SQL 블록은 문서에서 그대로 읽어 실제 PostgreSQL에서 실행하는 테스트가 있다 + (`PostgreSqlOutboxDispositionRunbookIntegrationTest`, lane `postgresqlOutboxDispositionRunbookIntegrationTest`). + 블록을 고치면 그 lane을 다시 돌린다 - 관련 runbook: [[outbox-publish-failed]] - 관련 branch: [[feature-domain-event-outbox-contract]], [[feature-background-job-async-contract]] (max attempts/DLQ vocabulary SSOT) @@ -80,8 +109,58 @@ V21(`V21__outbox_aggregate_sequence.sql`)은 `outbox_event`의 순서 키 README 15절 결정). 같은 키의 행들은 claim gate가 순서를 정하지 못한 행이다. 그냥 넘기면 순서 위반이 숨는다. V21은 transaction 하나로 적용되므로, 실패하면 V21 전체가 롤백되고 V20 상태로 남는다. +V21이 하는 일은 세 가지 더 있다. + +- **floor 기록.** `outbox_aggregate_sequence_floor`에 V21 적용 시각(epoch 마이크로초)을 한 번 쓴다. 새로 + 생기는 aggregate counter는 이 값과 남은 행의 최대 번호 중 큰 값 다음에서 시작한다. 그래서 reaper가 + 행을 지운 aggregate, tenant scope가 바뀐 aggregate도 소비자가 이미 받은 벽시계 번호(약 1.7×10¹⁵)보다 + 작은 번호를 받지 않는다(리뷰 RR-03-02). 벽시계 번호를 쓴 적 없는 서비스도 첫 번호가 floor + 1이다. +- **counter 미리 만들기.** 남은 행이 있는 aggregate의 counter를 그 최대 번호로 만든다. +- **잠금 대기 5초.** 유일 인덱스는 만드는 동안 `outbox_event`에 SHARE 잠금을 건다. 그동안 outbox를 쓰는 + 업무 쓰기가 모두 멈춘다. V21은 `lock_timeout = 5s`로 잠금을 기다린다. 열린 outbox 쓰기 트랜잭션 때문에 + 5초 안에 못 잡으면 V21이 실패하고 V20에 남는다(`MIGRATION_FAILED`). 덜 바쁜 시각에 다시 배포하거나 + Step 0으로 인덱스를 먼저 만든다. + +rolling 배포 중 아직 옛 버전인 인스턴스가 벽시계 번호로 쓴 행은 floor보다 크다. 새 버전의 counter는 +그 aggregate의 가장 큰 행 번호보다 작아지지 않으므로 그 행 뒤에 번호를 준다. 서로 커밋되지 않은 채 +겹친 옛·새 쓰기 사이의 순서는 보장하지 않는다 — 옛 벽시계 방식도 보장하지 않던 것이다. + +tenant scope를 바꾼 서비스(샘플은 `portfolio` → 단일 scope)는 배포 전에 옛 scope의 미발행 행이 +없는지 확인한다. 옛 scope 행과 새 scope 행은 서로 다른 aggregate로 보여 claim gate가 서로를 막지 않고, +partition key도 달라 다른 partition으로 간다. + +```sql +SELECT tenant_scope, count(*) AS unpublished +FROM outbox_event +WHERE status <> 'PUBLISHED' +GROUP BY tenant_scope; +``` + V20 이하 DB를 V21 이상으로 올리는 배포 전에 아래 순서로 확인한다. +**Step 0 — (outbox가 큰 경우만) 순서 키 인덱스를 먼저 만든다** + +V21이 인덱스를 만드는 동안 outbox 쓰기가 멈춘다. reaper가 `outbox_event`를 보존 기간 안으로 줄이므로 +보통은 짧다. 그 시간도 멈출 수 없을 만큼 크면, Step 1~3으로 중복을 없앤 뒤 같은 이름·같은 열의 인덱스를 +`CONCURRENTLY`로 먼저 만든다. 쓰기를 막지 않는다. transaction 밖(autocommit)에서 실행한다. V21의 +`IF NOT EXISTS`가 이 인덱스를 그대로 쓴다. + + +```sql +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS ux_outbox_event_aggregate_order + ON outbox_event (tenant_scope, aggregate_type, aggregate_id, aggregate_sequence, event_index); +``` + +`CONCURRENTLY` 빌드가 중복이나 중단으로 실패하면 INVALID 인덱스가 남는다. 이 인덱스는 유일성을 지키지 +않는다. V21은 이 이름의 인덱스가 유효한 유일 인덱스가 아니면 건너뛰지 않고 실패한다. 그때는 +`DROP INDEX CONCURRENTLY ux_outbox_event_aggregate_order;` 뒤 Step 1부터 다시 한다. 빌드가 아직 도는 +중에도 INVALID로 보이므로, 지우기 전에 `pg_stat_progress_create_index`에 진행 중인 빌드가 없는지 본다. + +V21 안에서 `CONCURRENTLY`를 쓰지 않은 이유: `CONCURRENTLY`는 transaction 안에서 돌 수 없다. V21을 +transaction 밖으로 빼면 중복을 만났을 때 "V21 전체 롤백, V20 그대로"가 깨지고, 반쯤 적용된 counter·floor와 +INVALID 인덱스가 남는다. 인덱스만 V22로 떼어 내도 실패 시 INVALID 인덱스와 Flyway 실패 이력이 남아 수동 +복구가 필요하다. 그래서 기본은 transaction 안 빌드 + 잠금 대기 5초, 큰 outbox는 이 Step 0이다. + **Step 1 — 중복 확인 (읽기 전용)** @@ -149,9 +228,10 @@ Step 1을 다시 돌려 0행이면 배포한다. 정리와 배포 사이에 옛 순서 번호를 만든 샘플 코드가 그 원인이었다). 그래도 V21은 실패하고 롤백될 뿐이니 Step 1부터 다시 한다. `outbox_event_v21_duplicates`는 확인이 끝나면 지운다. -확인 쿼리와 정리 SQL은 이 문서에서 그대로 읽어 실제 PostgreSQL에서 실행하는 테스트가 있다 -(`PostgreSqlOutboxOrderKeyRunbookIntegrationTest`, lane `postgresqlOutboxOrderKeyRunbookIntegrationTest`). -두 SQL 블록을 고치면 그 lane을 다시 돌린다. +확인 쿼리, 정리 SQL, Step 0의 인덱스 SQL은 이 문서에서 그대로 읽어 실제 PostgreSQL에서 실행하는 +테스트가 있다(`PostgreSqlOutboxOrderKeyRunbookIntegrationTest`, lane +`postgresqlOutboxOrderKeyRunbookIntegrationTest`). 같은 lane이 V21의 counter·floor, 잠금 대기 실패, +INVALID 인덱스 거부도 본다. 이 절의 `runbook-sql` 블록을 고치면 그 lane을 다시 돌린다. --- diff --git a/docs/runbooks/outbox-publish-failed.md b/docs/runbooks/outbox-publish-failed.md index 5d35c179..41e9dd6b 100644 --- a/docs/runbooks/outbox-publish-failed.md +++ b/docs/runbooks/outbox-publish-failed.md @@ -4,7 +4,7 @@ category: TRANSIENT_DEPENDENCY error_codes: [OUTBOX_PUBLISH_FAILED] severity: P2 owner: oncall -last_updated: 2026-06-11 +last_updated: 2026-11-03 status: stub --- @@ -14,17 +14,23 @@ status: stub 이 runbook은 다음 alert에서 발동됩니다. -- alert name: `outbox_publish_failed_rate` 또는 `outbox_publisher_lag` -- alert payload 필수 field: `error.code=OUTBOX_PUBLISH_FAILED`, `event_type`, `correlation_id`, `runbook_link` +- alert name: `outbox_publish_failed_rate`, `outbox_publisher_lag` 또는 `outbox_retry_long` +- alert payload 필수 field: `error.code=OUTBOX_PUBLISH_FAILED`, `error.reason`, `event_type`, `correlation_id`, `runbook_link` - 임계 (metrics.yaml verbatim): - P2: `outbox.publisher.published.total{outcome=FAILED}` rate > 1% for 10m - P2: `outbox.publisher.lag` > 60s for 10m / P1: > 300s for 5m - P2: `outbox.pending.size{status=PENDING}` growing for 10m + - P2: `outbox.retry.long.size` > 0 for 10m — §4.1 "끝나지 않는 재시도"로 간다 ## 2. First Response (5분 이내) ### Step 1 — 확인 -1. ERROR log에서 `OUTBOX_PUBLISH_FAILED` 라인 확인: `event_type`, `event_id`, `correlation_id`, `attempt_count` 추출 +1. ERROR log에서 `OUTBOX_PUBLISH_FAILED` 라인 확인: `event_type`, `event_id`, `correlation_id`, `attempt_count`, + `error.reason` 추출. `error.reason`이 원인 코드다 — publisher가 붙인 코드(예: `KAFKA_AUTHENTICATIONEXCEPTION`, + `KAFKA_TRANSPORT_CLOSED`, `PUBLISH_RUNTIME_UNAVAILABLE`, `PUBLISH_DEADLINE_EXCEEDED`), 코드가 없으면 결과 + 이름(`AMBIGUOUS`, `REJECTED_RETRYABLE`), 분류되지 않고 던져진 실패면 예외 타입 이름이다. 같은 값이 행의 + `last_failure_code`에도 남는다. `error.cause_type`은 어느 결과였는지만 말한다(`OutboxPublishRefusedException`이면 + 거부, `OutboxPublishAmbiguousException`이면 응답 없음) 2. broker 상태 확인: `APP_MESSAGING_BROKER` 값과 broker endpoint 가용성. 이 키는 활성화 스위치가 아니라 **선택자**다 — messaging을 끄는 것은 `APP_MESSAGING_ENABLED=false`이고, 이 값을 비운다고 messaging이 꺼지지는 않는다. @@ -40,8 +46,12 @@ status: stub 3. `outbox.pending.size` status 분포 확인 (FAILED 누적 vs PENDING 누적) ### Step 2 — 임시 격리 -- 일시 실패는 자동 backoff 재시도(30s × 2^(attempt-1) + jitter, 상한 5분)가 동작 — 즉시 수동 개입 불필요 +- 일시 실패는 자동 backoff 재시도(30s × 2^(attempt-1) + jitter, 상한 5분)가 동작 — broker 장애처럼 원인이 저절로 + 풀리는 경우는 수동 개입이 필요 없다 - broker 정지·종료 중 거부·Kafka 인증/인가 실패(`AMBIGUOUS`, `REJECTED_RETRYABLE`)는 시도 예산을 쓰지 않는다 — 장애가 길어도 DEAD가 되지 않고, broker가 돌아오면 늦어도 약 5분 30초 안에 자동 재발행된다 +- **그 대신 끝이 없다.** 원인이 저절로 풀리지 않으면(진짜로 틀린 credential·ACL, Kafka adapter가 이름을 모르는 + broker 거부 — topic timestamp 정책, 없는 topic의 metadata 대기 초과는 기본값으로 `AMBIGUOUS`가 된다) 그 행은 5분 + 간격으로 영원히 FAILED가 되고 그 aggregate의 뒤 이벤트를 막는다. `outbox.retry.long.size` > 0이면 §4.1로 간다 - max attempts 3은 adapter가 분류하지 못하고 던진 발행 예외에만 적용된다 (소진 시 `runbook://outbox/dead-letter`로 이관) - relay 자체를 멈춰야 하면 `app.outbox.relay-enabled=false`로 스케줄러 비활성 (이벤트는 outbox 테이블에 안전하게 보존됨 — 유실 없음) @@ -57,7 +67,10 @@ status: stub - broker outage/네트워크 → broker 측 회복 대기 - Kafka adapter 미구성(enabled인데 brokers 누락은 기동 시 차단됨) / disabled 상태에서 producer 활성화 - poison event (직렬화 불가/payload 계약 위반) → 확정 거부(`REJECTED_*`)면 바로, 분류되지 않은 예외면 attempts 소진 후 DEAD로 흘러감 (의도된 동작) - - 같은 행의 `OUTBOX_PUBLISH_FAILED`가 5분 간격으로 끝없이 반복되고 원인이 Kafka 인증/인가면 credential·ACL이 실제로 틀린 것이다 — 고치면 다음 재시도에서 저절로 빠진다 + - 같은 행의 `OUTBOX_PUBLISH_FAILED`가 5분 간격으로 끝없이 반복되면 §4.1의 조회로 그 행의 `last_failure_code`를 + 본다. `KAFKA_AUTHENTICATIONEXCEPTION`·`KAFKA_*AUTHORIZATIONEXCEPTION`이면 credential·ACL이 실제로 틀린 + 것이다 — 고치면 다음 재시도에서 저절로 빠진다. 그 밖의 코드가 같은 행에서만 반복되면 broker가 그 레코드를 + 받지 않는 것이다 — §4.1로 보류한다 - 동일 aggregate head 실패로 후행 이벤트가 FIFO 게이트에 차단되어 lag 증가 (strict per-aggregate FIFO — 설계 의도) ## 4. Mitigation @@ -66,6 +79,61 @@ status: stub - IN_FLIGHT orphan(claim 후 crash)은 in-flight-timeout(기본 PT5M) 경과 후 자동 재claim — at-least-once이므로 중복 발행 가능, consumer dedupe(idempotencyKey)가 흡수 - 장기: `app.outbox.poll-interval`/`batch-size` 조정, broker 가용성 SLA 점검, 빈발 event_type의 payload 계약 검토 +### 4.1 끝나지 않는 재시도 — 찾기, 보류(park), 재개(resume), 버리기(skip) + +시도 예산을 쓰지 않는 실패(§2 Step 2)는 끝 상태가 없다. 리뷰 2026-11-01 README 15절 결정(재시도 유지)은 그대로 두고, +운영자가 그런 행을 보고 멈출 수 있게 한다(재리뷰 2026-11-03 RR-05-03 결정). + +**Step 1 — 찾기 (읽기 전용).** 미룬 claim(`deferred_attempt_count`)이 10번 이상인 FAILED 행이다. backoff 상한이 +5분이라 10번은 적어도 약 35분 동안 계속 실패했다는 뜻이다. `outbox.retry.long.size`·`outbox.retry.long.age`가 +같은 기준(`OutboxMetrics.LONG_RETRY_DEFERRED_CLAIMS`)으로 센다. + + +```sql +SELECT event_id, event_type, aggregate_type, aggregate_id, attempt_count, deferred_attempt_count, + last_failure_code, occurred_at, next_attempt_at +FROM outbox_event +WHERE status = 'FAILED' + AND deferred_attempt_count >= 10 +ORDER BY occurred_at, event_id; +``` + +`last_failure_code`로 원인을 가른다(§3). 원인이 고쳐지는 것(credential·ACL·topic 설정)이면 고치고 기다린다 — +다음 재시도에서 저절로 빠진다. 고칠 때까지 재시도 소음과 실패 로그를 멈추고 싶거나, 그 레코드만 broker가 +받지 않는 것이면 Step 2로 보류한다. + +**Step 2 — 보류(park).** FAILED 행 하나를 `PARKED`로 둔다. relay는 PARKED 행을 claim하지 않는다. 그 행은 +여전히 미발행이라 같은 aggregate의 뒤 이벤트를 계속 막는다 — 순서는 지킨다. `outbox.publisher.lag`도 계속 +오른다. 보류는 원인을 찾는 동안 재시도만 멈추는 것이지 incident를 닫는 것이 아니다. + + +```sql +UPDATE outbox_event SET status = 'PARKED' WHERE event_id = '' AND status = 'FAILED'; +``` + +`UPDATE 0`이면 그 사이 relay가 행을 claim해 IN_FLIGHT다. `next_attempt_at`이 지난 뒤 다시 실행한다. +보류된 행 수는 `outbox.pending.size{status=PARKED}`로 보인다. + +**Step 3 — 재개(resume) 또는 버리기(skip).** 원인을 고쳤으면 재개한다. 행은 FAILED로 돌아가 바로 claim +대상이 된다. 시도 횟수는 그대로 둔다(미룬 claim은 예산을 쓰지 않으므로 DEAD로 바로 가지 않는다). + + +```sql +UPDATE outbox_event SET status = 'FAILED', next_attempt_at = now() WHERE event_id = '' AND status = 'PARKED'; +``` + +그 이벤트를 영원히 보내지 않기로 했으면 버린다. DEAD skip과 같은 승인 조건이다 — 다운스트림에 영구 이벤트 +갭이 생기므로 비즈니스 오너 승인 후에만 하고 incident 기록에 남긴다. + + +```sql +UPDATE outbox_event SET status = 'PUBLISHED' WHERE event_id = '' AND status = 'PARKED'; +``` + +이 절의 SQL 블록은 문서에서 그대로 읽어 실제 PostgreSQL에서 실행하는 테스트가 있다 +(`PostgreSqlOutboxDispositionRunbookIntegrationTest`, lane `postgresqlOutboxDispositionRunbookIntegrationTest`). +블록을 고치면 그 lane을 다시 돌린다. + ## 5. Escalation - P1 lag(>300s 5m) 지속 + broker 회복 불가면 broker/infra 팀에 page @@ -79,8 +147,8 @@ status: stub ## 7. Related - error-codes.yaml rows: `OUTBOX_PUBLISH_FAILED` (TRANSIENT_DEPENDENCY, retryable=true, retry_after 30s) -- metrics.yaml: `outbox.publisher.published.total`, `outbox.publisher.lag`, `outbox.pending.size` -- 코드: `application-core` `PublishPendingOutboxEventsUseCase`(상태머신), `adapter-persistence` `outbox/OutboxEventJpaRepository`(SKIP LOCKED claim + FIFO 게이트), `adapter-outbound` `messaging/outbox/KafkaOutboxMessagePublishAdapter`(fail-closed) +- metrics.yaml: `outbox.publisher.published.total`, `outbox.publisher.lag`, `outbox.pending.size`, `outbox.retry.long.size`, `outbox.retry.long.age` +- 코드: `application-core` `PublishPendingOutboxEventsUseCase`(상태머신), `adapter:outbound:persistence-jpa` `outbox/postgresql/PostgreSqlOutboxClaimRepository`(SKIP LOCKED claim + FIFO 게이트), `adapter:outbound:messaging` `outbox/OutboxMessagePublishAdapter`·`platformbridge/PlatformIntegrationEventPublishAdapter`(fail-closed, 원인 코드 번역) - 관련 runbook: [[outbox-dead-letter]] - 관련 branch: [[feature-domain-event-outbox-contract]], [[feature-background-job-async-contract]] (retry/DLQ vocabulary SSOT) diff --git a/docs/runbooks/redis-capability-incident.md b/docs/runbooks/redis-capability-incident.md index 4c17b104..a40d93b4 100644 --- a/docs/runbooks/redis-capability-incident.md +++ b/docs/runbooks/redis-capability-incident.md @@ -34,20 +34,19 @@ Redis liveness 실패만으로 pod를 재시작하지 않는다. 재시작 폭 - `evictionValidation=CONFIGURED_EXPECTATION_ONLY`와 `externalEvictionAttestation=INCOMPLETE`는 effective server policy가 증명되지 않았다는 뜻이다. 이를 정상 attestation으로 해석하지 않는다. -- `redis.capability.operations.total`과 `redis.capability.duration.seconds`에서 affected - capability/role/operation의 실제 반환 outcome을 확인한다. mutation의 - `certainty=indeterminate`는 timeout이나 연결 끊김을 미실행 증거로 바꾸지 않는다. -- `redis.capability.admission.rejected.total`에서 `rejected_saturated`와 - `rejected_closed`를 구분하고, `redis.capability.inflight.total`의 같은 role에 대해 현재 0이 - 아닌 state를 확인한다. in-flight gauge는 bounded command count이며 byte 수나 queue depth가 - 아니다. -- `redis.capability.readiness.total`은 현재 상태 gauge가 아니라 exact sanitized - `RoleHealth` 관측 횟수다. 최신 health detail의 state/reason/requirement와 함께 해석한다. - optional cache의 degraded serving과 required coordination/session의 fail-closed 결정을 - 같은 availability 의미로 합치지 않는다. -- 종료 시 `redis.capability.lifecycle.drain.total`에서 `drained`, - `forced_after_timeout`, `interrupted`를 구분한다. repeated close는 새 drain을 시작하거나 - 중복 관측을 만들지 않는다. +- **지금 내보내는 Redis 지표는 connection lane 단위 `redis.lane.*`다**(`docs/registries/metrics.yaml`, + `lane` 태그). capability·operation 단위 지표(`redis.capability.*`)는 설계만 있고 아직 어떤 코드도 + 내보내지 않는다 — 대시보드에서 찾지 않는다(재리뷰 2026-11-03, `src/adapter/outbound/cache-redis/README.md` + 「아직 내보내지 않는 지표」). +- lane 포화: `redis.lane.acquire.refused.total`(거절)과 `redis.lane.acquire.wait.seconds`(대기)를 lane별로 + 본다. `CACHE_SCRIPT`만 차 있으면 캐시 조건부 쓰기의 문제이고, 멱등성·lease가 쓰는 `SCRIPT` lane은 + 따로 본다(재리뷰 RR-04-02). +- 연결 불안정: `redis.lane.connections.opened.total`·`redis.lane.connections.discarded.total`의 급증, + `redis.lane.connections.active`·`pending`. +- operation 결과(hit/miss/denied/conflict, `certainty=indeterminate`)는 지표가 아니라 각 capability의 + 로그와 health detail에서 확인한다. timeout이나 연결 끊김은 미실행 증거가 아니다. +- readiness는 최신 health detail의 state/reason/requirement로 본다. optional cache의 degraded serving과 + required coordination/session의 fail-closed 결정을 같은 availability 의미로 합치지 않는다. - reconnect, cache source-load, session repository error 지표의 변화를 함께 본다. - Redis server 측에서는 memory/eviction, rejected clients, replication link/lag, persistence error, Cluster coverage를 operator dashboard에서 확인한다. @@ -56,9 +55,8 @@ Redis liveness 실패만으로 pod를 재시작하지 않는다. 재시작 폭 ### Observability and lifecycle boundaries -- 여섯 `redis.capability.*` meter의 tag는 닫힌 enum에서만 생성된다. key, subject, session id, - token, endpoint, exception text, script/SHA, value 같은 identity/wire material을 metric이나 - ticket에 복사하지 않는다. +- `redis.lane.*` meter의 tag는 닫힌 lane 이름뿐이다. key, subject, session id, token, endpoint, + exception text, script/SHA, value 같은 identity/wire material을 metric이나 ticket에 복사하지 않는다. - semantic operation 계측은 logical provider가 실제로 반환한 hit/miss/denied/conflict/ unavailable/indeterminate 결과를 기록한다. cache의 `stale`/`skipped`, session의 `tombstoned`/`absolute_expired`도 정상 hit/miss와 분리한다. meter registry, classifier, diff --git a/docs/testing/TESTING_STRATEGY.md b/docs/testing/TESTING_STRATEGY.md index 09723148..1afecf1f 100644 --- a/docs/testing/TESTING_STRATEGY.md +++ b/docs/testing/TESTING_STRATEGY.md @@ -189,8 +189,8 @@ adapter leaf의 실 DB·Redis lane에서 돈다. fake가 실제와 다르게 동 `public interface *Port`)와, `application-notification`이 소유하고 `persistence-notification`이 구현하는 저장소 포트 전부다(review 04-14, 표에 `(notification)`으로 표시). notification 포트의 계약은 `application-notification` testFixtures에 있고, 실제 구현 쪽은 persistence-jpa의 `postgresqlIntegrationTest` -source set에서 태그 `jpa-contract`로 `jpaPlatformContractTest` lane이 돌린다. fileserver 포트는 -`application-fileserver` testFixtures의 `*PortContract`가 맡고 `persistence-fileserver/CLAUDE.md`에 적혀 있다. +source set에서 태그 `jpa-contract`로 `jpaPlatformContractTest` lane이 돌린다. fileserver 저장 포트는 +9.3에 있다. ### 9.1 계약이 있는 포트 @@ -248,8 +248,15 @@ source set에서 태그 `jpa-contract`로 `jpaPlatformContractTest` lane이 돌 고쳤다. in-memory store는 `DirectTransactionPort`를 받으면 같은 규칙(트랜잭션 밖·읽기 전용 거부, 롤백 시 취소)으로 동작한다. 인자 없이 만들면 지금처럼 바로 쓴다(relay 테스트가 행을 미리 심는 용도). - `IdempotencyStoreContract`의 시간 케이스(renew가 lease를 실제로 늘린다, review 05-02)는 시계를 - 가진 store(실 Redis, PostgreSQL)에서만 의미가 있다. in-memory fake는 만료가 없어 `elapse`가 - no-op이다. PostgreSQL에서 renew가 lease를 늘리지 않게 바꾼 변이로 이 케이스가 실패함을 확인했다. + 가진 store(실 Redis, PostgreSQL)에서만 의미가 있다. 시계를 움직일 수 없는 store(in-memory fake, + command fake 위 Redis)는 `elapse`를 재정의하지 않고, 기본 `elapse`가 그 케이스를 abort해 보고서에 + skipped로 남는다. 예전에는 no-op이라 시간이 흐르지 않은 채 통과했다(review 2026-11-03 RR-01-06). + PostgreSQL에서 renew가 lease를 늘리지 않게 바꾼 변이로 이 케이스가 실패함을 확인했다. +- `OutboxAggregateSequenceContract`는 포트 Javadoc의 세 약속 중 둘을 보지 않았다(RR-01-06). "쓰기 + 트랜잭션 밖 거부"와 "롤백된 번호는 다시 준다"를 넣었다. fake는 `DirectTransactionPort`를 받으면 같은 + 규칙으로 동작한다(인자 없이 만들면 예전처럼 트랜잭션을 보지 않는다 — sample 테스트가 쓰는 방식). + `CacheRegionContract`에는 2^53 위 revision 한 자리 차이 경우를 넣었다. fake가 revision을 double로 + 비교하게 바꾼 변이가 이제 실패한다(전에는 통과). notification 계약 다섯 개(deduplication·request·reconciliation job·admin operation·template, 2026-11-03)는 PostgreSQL 쪽에만 있던 테스트를 계약으로 옮긴 것이다. `ReconciliationJobClaimContractTest`(8건)와 @@ -301,6 +308,36 @@ r3-J: queue가 fence를 보지 않음(같은 worker 이름 재claim 1건), queue `SuppressionSideEffect`가 complaint를 INVALID로 기록함(1건). relay use case가 결과 조회 대신 예외만 보는 `publish`를 부르게 바꾸면 relay 테스트 7건이 실패한다. +### 9.3 application-fileserver 저장 포트 (review 2026-11-03 RR-01-04·RR-07-06) + +`persistence-fileserver`가 구현하는 포트 9개와 소유자 규칙 포트. 계약은 `application-fileserver` +testFixtures(`dev.caskeleton.application.fileserver.testkit`)에 있고, 실제 구현 쪽은 persistence-jpa의 +`postgresqlIntegrationTest` source set(`readiness` 패키지, 공용 기동 `FileserverPostgreSqlStack`)에서 태그 +`jpa-contract`로 `jpaPlatformContractTest` lane이 돌린다. 시간이 걸린 경우는 stack이 움직이는 시계로 본다. + +| 포트 | 계약 | fake | 상속하는 실제 구현 테스트 | +| --- | --- | --- | --- | +| `FileMetadataPort` | `FileMetadataPortContract` | `InMemoryFileMetadataStore` | `PostgreSqlFileserverMetadataStoreIntegrationTest` (`postgresqlFileserverMetadataIntegrationTest`) | +| `CleanupQueuePort` | `CleanupQueuePortContract` | `InMemoryCleanupQueue` | `PostgreSqlFileserverReclamationIntegrationTest` (`postgresqlFileserverReclamationIntegrationTest`) | +| `FileQuotaPort`·`QuotaCommitPort`·`QuotaReclaimPort` | `FileQuotaLedgerContract` — 세 포트가 한 장부라 한 계약 | `FakeFileQuotaService`(장부) + `FakeQuotaCommitGateway`·`RecordingQuotaReclaimGateway`(같은 장부에 쓴다) | `PostgreSqlFileQuotaLedgerContractTest` | +| `UploadSessionPort` | `UploadSessionPortContract` | `InMemoryUploadSessionStore` | `PostgreSqlUploadSessionContractTest` | +| `StagingUploadLocatorPort` | `StagingUploadLocatorPortContract` | `InMemoryUploadSessionStore`(같은 세션) | `PostgreSqlStagingUploadLocatorContractTest` | +| `RecoveryQueuePort` | `RecoveryQueuePortContract` | `InMemoryRecoveryQueue` | `PostgreSqlRecoveryQueueContractTest` | +| `ContentReferenceLedgerPort` | `ContentReferenceLedgerPortContract` | `InMemoryFileMetadataStore`(같은 레코드) | `PostgreSqlContentReferenceLedgerContractTest` | +| `FileAccessPolicyPort` | `FileAccessPolicyPortContract` — 규칙 정본은 `FileAccessSubject.owns` | `FakeFileAccessPolicy` | `RoleBasedFileAccessPolicyContractTest` — fileserver adapter (`test`) | + +계약을 처음 적용할 때 fake가 실제와 다르게 답한 경우와 처리(모두 fake를 실제에 맞췄다): + +- 쿼터: fake에 시계가 없어 만료된 예약도 사용량에 셌고, 해제한 예약의 확정·연장을 조용히 받았다. 확정· + 회수 fake는 기록만 하고 장부를 바꾸지 않았다. 이제 한 장부에 시계가 있고 JPA와 같은 조건으로 거절한다. + 만료됐지만 해제되지 않은 예약의 확정은 JPA SQL이 받으므로(쓴 바이트는 사용량이다) 계약도 받는다 — + `JpaFileQuota` Javadoc의 "만료된 예약은 확정할 수 없다"는 SQL과 달라 고쳤다. +- 업로드 세션: fake가 기한이 지난 업로드에 lease를 줬고, lease가 만료된 writer의 offset 확정을 받았고, + `findExpired`를 기한 순이 아니라 넣은 순서로 줬다. +- 복구 큐: fake가 한도 0을 거절하지 않았다. +- 소유자 규칙: fake와 실제 정책이 같은 규칙을 따로 썼다(principal·tenant 비교 두 벌). 정본 + `FileAccessSubject.owns` 하나를 둘 다 부른다. fake 규칙에서 tenant 비교를 빼는 변이가 계약 2건으로 잡힌다. + ### 9.2 계약이 없는 포트 (의도) | 포트 | 실제 구현 | fake | 계약이 없는 이유 | 다음 단계 | diff --git a/docs/web/http-contract.md b/docs/web/http-contract.md index 7d98487b..3e8c1f9b 100644 --- a/docs/web/http-contract.md +++ b/docs/web/http-contract.md @@ -7,8 +7,11 @@ 코드가 이 문서를 지키는지는 다음 테스트가 확인한다. - 응답 원문의 키 집합: `src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/envelope/EnvelopeWireShapeTest.java` +- 핸들러 밖 오류(`/error` 디스패치)의 실패 Envelope: 실제 컨테이너에서 `WebContractAssertions` + (`TomcatWebContractIT`·Jetty lane), 기본 JWT 체인에서 `JwtChainErrorDispatchIT` - 멱등성 응답의 status·code·retryable·`Retry-After`: `IdempotencyExceptionMappingTest`, - `RetryAfterAdvisorTest`, `OperationalErrorTest`, 키 형식은 `IdempotencyKeyHeaderTest` + `RetryAfterAdvisorTest`, `OperationalErrorTest`, 키 형식은 `IdempotencyKeyHeaderTest`, + 실행 전 거부와 결과 불명의 구분은 `IdempotencyExecutorTest`, lease 하한은 `IdempotencySettingsTest` - 헤더 규칙(3절): `WebMvcRequestIdFilterTest`, `CorsSecurityFilterIntegrationTest` - web 전용 코드(4절): `TransportErrorHandlingTest`, `IdempotencyKeyHeaderTest`, web-advanced `WebAdvancedErrorMappingTest` @@ -51,8 +54,14 @@ JSON 응답은 모두 `Envelope` 하나의 모양이다(소유자 결정 2). ** | `correlationId` | 위와 같음. 값이 없으면 키가 없다 | | `page` | 페이지 목록 응답에만. `number`·`size`·`total`·`sort`(미정렬이면 `sort: null`) | -필터 앞에서 끊긴 응답(필터 체인 밖의 오류)은 id 키가 빠질 수 있다. 클라이언트는 `meta`의 모르는 -키를 무시한다. +컨트롤러에 닿기 전에 거부된 요청(보안 방화벽이 막은 경로, 필터의 `sendError`, 필터가 던진 예외)도 같은 +실패 Envelope다. 컨테이너가 `/error`로 넘긴 요청을 `EnvelopeErrorController`가 status에 맞는 코드로 답한다. +본문에 요청 경로나 시각을 싣지 않는다. 원래 status가 그대로 나가고, 기본 JWT 체인에서도 익명 400이 401로 +바뀌지 않는다(재리뷰 RR-02-02). 다만 요청 id 필터를 지나기 전에 끊긴 응답은 `meta`의 id 키가 빠질 수 있다. +클라이언트는 `meta`의 모르는 키를 무시한다. + +4xx·5xx status로 나가는 본문은 `success:true`로 감싸지 않는다. 실패 Envelope가 아닌 본문을 오류 status로 +돌려주는 핸들러는 그 본문이 그대로 나간다. ### 왜 이렇게 나가는가 @@ -79,21 +88,31 @@ JSON 응답은 모두 `Envelope` 하나의 모양이다(소유자 결정 2). ** 첫 요청이 끝나면 같은 작업이 두 번 적용된다. - **`Retry-After: 2`의 근거.** 기본 processing lease(`app.capabilities.idempotency.processing-lease`, 30s)보다 짧아, 첫 시도가 아직 끝날 수 있는 동안 다시 묻는다. 너무 이르면 409를 한 번 더 받을 뿐이고 - 결과가 바뀌지는 않는다. lease를 2초보다 짧게 설정한 서비스는 이 값이 lease를 넘으므로 값을 다시 - 정한다(`RetryAfterAdvisor`, `docs/registries/error-codes.yaml`). + 결과가 바뀌지는 않는다. 값은 `IdempotencyExecutor.IN_FLIGHT_RETRY_AFTER` 하나이고 `RetryAfterAdvisor`가 + 그 값을 쓴다. `processing-lease`를 이 값(2초) 이하로 설정하면 기동이 실패한다(`IdempotencySettings`, + 재리뷰 RR-02-05). - executor는 기다리지 않는다. 처리 중이면 바로 409를 돌려준다. 키로 결과를 조회하는 엔드포인트는 없다. - **키 문법**: 보이는 ASCII(`0x21`–`0x7E`) 1~256자. 서버는 trim하거나 고치지 않는다. 짝 프론트의 `isValidIdempotencyKey`와 같은 규칙이다. KEYED 엔드포인트는 모두 `IdempotencyKeyHeader.parse`로 헤더를 읽어 같은 코드로 거부한다(리뷰 02-12). 키 원문은 응답에 돌려 쓰지 않는다. 키를 요구하지 않는 엔드포인트는 키 없이 보호 없이 실행된다. -같은 executor가 내는 나머지 두 실패: +같은 executor가 내는 나머지 세 실패. 저장소가 응답하지 않은 경우는 **언제** 멈췄는지로 코드가 갈린다. | 경우 | status | `error.code` | `retryable` | `Retry-After` | 클라이언트 행동 | | --- | --- | --- | --- | --- | --- | -| 멱등성 저장소 응답 없음 | 503 | `IDEMPOTENCY_UNAVAILABLE` | `true` | 없음 | 작업 실행 전에 거부됐다. 같은 키로 나중에 다시 보낸다 | +| 작업 실행 전(claim·시작 단계) 멱등성 저장소 응답 없음 | 503 | `IDEMPOTENCY_UNAVAILABLE` | `true` | 없음 | 작업은 실행되지 않았다. 같은 키로 나중에 다시 보낸다 | +| 작업 실행 뒤 결과 기록(complete 단계) 중 저장소 응답 없음 — **결과 불명** | 503 | `IDEMPOTENCY_COMPLETION_UNKNOWN` | `true` | 없음 | "적용되지 않았다"고 알리지 않는다. **같은 키로** 나중에 다시 보낸다. 새 키 금지 | | 이전 시도의 효과를 증명할 수 없음 | 500 | `IDEMPOTENCY_RECOVERY_REQUIRED` | `false` | 없음 | 재전송도 재실행도 안전하지 않다. 업무 데이터로 결과를 확인한다 | +- **결과 불명이 무엇을 뜻하나.** 작업은 이미 돌았다. 그 작업의 트랜잭션 안 DB 변경은 예외와 함께 + 롤백되지만, 트랜잭션 밖 효과(다른 시스템 호출, 메시지)는 일어났을 수 있다. 그래서 실행 전 거부와 같은 + 503이어도 code가 다르다. 클라이언트는 status가 아니라 `error.code`로 가른다. +- **같은 키로 다시 보내면 서버가 판정한다.** 결과가 실제로는 기록됐으면 재생(첫 요청의 성공 응답), 첫 시도의 + processing lease가 아직 남았으면 409 `IDEMPOTENT_IN_FLIGHT`, lease가 지났으면 500 + `IDEMPOTENCY_RECOVERY_REQUIRED`다. 새 키로 보내면 서버는 다른 요청으로 보고 막지 못한다 + (재리뷰 2026-11-03 README 4절 결정, RR-02-03). + ## 3. 헤더 ### 요청 id·correlation id @@ -127,7 +146,7 @@ CORS가 켜져 있으면(`app.cors.enabled=true`) cross-origin 실제 응답의 | 코드 | status | category | `retryable` | `Retry-After` | 언제 | | --- | --- | --- | --- | --- | --- | -| `REQUEST_TIMEOUT` | 503 | `TRANSIENT_DEPENDENCY` | `true` | 없음 | 비동기 요청(`Callable`·`DeferredResult`·스트리밍)이 시간 안에 끝나지 않았다 | +| `REQUEST_TIMEOUT` | 503 | `TRANSIENT_DEPENDENCY` | `true` | 없음 | 비동기 요청(`Callable`·`DeferredResult`·스트리밍)이 시간 안에 끝나지 않았다. GraphQL도 같은 이름을 쓰지만 그쪽 `category`는 GraphQL 전용 어휘 `TIMEOUT`이다(`src/adapter/inbound/graphql/README.md`) | | `IDEMPOTENCY_KEY_INVALID` | 400 | `VALIDATION` | `false` | 없음 | 2절 | | `RATE_LIMIT_UNAVAILABLE` | 503 | `TRANSIENT_DEPENDENCY` | `true` | 있음 | rate-limit 저장소 응답 없음(`docs/web/runbook.md`) | | `RATE_LIMIT_INCOMPATIBLE` | 503 | `INTERNAL` | `false` | 없음 | rate-limit 제공자가 계약과 맞지 않음 | @@ -143,6 +162,8 @@ CORS가 켜져 있으면(`app.cors.enabled=true`) cross-origin 실제 응답의 - 응답 키를 더하거나 빼면 `EnvelopeWireShapeTest`와 이 문서 1절을 같이 고친다. - 응답 헤더를 새로 쓰면 `CorsExposedHeaders`에 넣고 3절을 고친다. - 멱등성 코드의 status·retryable·`Retry-After`를 바꾸면 `OperationalError`, `RetryAfterAdvisor`, - `docs/registries/error-codes.yaml`, 이 문서 2절을 같은 변경에서 고친다. + `docs/registries/error-codes.yaml`, 이 문서 2절을 같은 변경에서 고친다. 공유 코드의 `retryable`은 + gRPC trailer(`error-retryable`)와 GraphQL extension(`retryable`)에도 그대로 나가므로 그 둘의 README와 + 매핑 테스트(`GrpcStatusMapperTest`, `GraphQlWireErrorMapperTest`)도 같이 본다. - 짝 프론트의 백엔드 계약 문서(`clean-architecture-frontend-template` `docs/architecture/backend-api-and-server-state-contract.md` §4.1·§6)는 이 문서를 가리켜야 한다. diff --git a/infra/redis-sdk/README.md b/infra/redis-sdk/README.md index d6d687c3..832c5860 100644 --- a/infra/redis-sdk/README.md +++ b/infra/redis-sdk/README.md @@ -38,7 +38,7 @@ qualifies nothing, so "no CA configured" is an error rather than a client with v ## The ACL fixture -`acl/all-accounts.acl` provisions the accounts every lane uses. Two things about it matter, and +`acl/lane-fixture-accounts.acl` provisions the accounts every lane uses. Two things about it matter, and neither can be written in the file itself — **Redis refuses to start if an `aclfile` contains a comment line**, so the whole file is directives and the explanation lives here. diff --git a/infra/redis-sdk/acl/README.md b/infra/redis-sdk/acl/README.md index d561a860..0117e9b1 100644 --- a/infra/redis-sdk/acl/README.md +++ b/infra/redis-sdk/acl/README.md @@ -1,4 +1,16 @@ -# ACL accounts +# ACL accounts — test fixtures, not a deployment file + +**Nothing in this directory is loaded by a deployment.** Every file here is a fixture for the local +and test lanes: its passwords are public (`fixture-*`, written in this repository), and +`lane-fixture-accounts.acl` also provisions three accounts that only a lane topology needs — among +them `ca-skeleton-cluster-bootstrap`, which has `~* &* +@all`, every command on every key. Loaded into +a real Redis, that line alone gives anyone who can reach the server and read this repository full +control of it (`CONFIG`, `MODULE`, every key). A deployment writes its own ACL file of the same +shape: the application accounts it actually uses, passwords from its secret store, and none of the +lane-only accounts (review 2026-11-03 RR-07-04). The application enforces its half of that: under +the `prod` runtime profile it refuses to start while any setting or environment variable holds a +value beginning with `fixture-` (`SecretSourceValidator`), so a deployment that copied a lane's +`REDIS_LANE_*` values stops before it connects. One file per `CommandAccess` level. Each account is deliberately narrower than the SDK's own rules: the account is the last enforcement boundary and a permit issued inside the process never widens it, @@ -7,23 +19,21 @@ so a mistake in the SDK is still refused by the server. A Redis ACL file accepts nothing but complete `user` lines — no comments and no line continuations — which is why the rationale lives here instead of inline. -Every account here carries a **fixture** password (`fixture-*`). None of them is a deployment -credential and none is read by anything but the local lanes; a deployment replaces the whole file -with one whose passwords come from its secret store. They are written down rather than left as -`nopass` because an account that accepts any password makes every assertion about authentication -pass for the same reason a typo would have — and because these files are what a fork copies. The -four per-role files carried `nopass` long after `all-accounts.acl`, the file a deployment actually -loads, had been fixed; a fork that copied one of them got an open account and a green lane. +Every account here carries a **fixture** password (`fixture-*`). They are written down rather than +left as `nopass` because an account that accepts any password makes every assertion about +authentication pass for the same reason a typo would have — and because these files are what a fork +copies. The four per-role files carried `nopass` long after the combined lane file had been fixed; +a fork that copied one of them got an open account and a green lane. | File | Account | Grants | | --- | --- | --- | | `application.acl` | `CommandAccess.APPLICATION` | the typed data-structure commands over `prod:*`, with the dangerous and deprecated names denied | | `application-advanced.acl` | `APPLICATION_ADVANCED` | the above plus pub/sub, transactions, and the registered-script path. `EVAL` is absent: only `EVALSHA` of an already loaded digest is reachable. `+time` is granted because the rate-limit and idempotency scripts read the server clock with `redis.call('TIME')`, and `TIME` is in no data category (`@read`/`@write`): without it every such script fails with an ACL error. The startup SCRIPT probe runs `TIME` so a template missing it fails at boot | | `raw-gateway.acl` | `RAW_GATEWAY` | exactly the commands the catalog classifies `RAW_ONLY`, and nothing else | -| `all-accounts.acl` | – | the four accounts concatenated; Redis takes one `aclfile`, so this is what a deployment loads | +| `lane-fixture-accounts.acl` | – | what the lanes load (Redis takes one `aclfile`): `user default off`, the four per-role accounts of this table, and three lane-only accounts — `ca-skeleton-replication` (replica sync) and `ca-skeleton-sentinel` for the replicated and sentinel topologies, `ca-skeleton-cluster-bootstrap` (`+@all`) to create the cluster lane's slots. Never a deployment file | | `admin-readonly.acl` | `ADMIN_READONLY` | read-only diagnostics. Every destructive counterpart is denied here and blocked in the command policy catalog — two independent controls for the same rule | -## Who loads `all-accounts.acl` +## Who loads `lane-fixture-accounts.acl` The Redis topology lanes (`../standalone`, `../sentinel`, `../cluster`, `../tls`) and the application's Compose lanes that switch Redis on (`local-redis`, `all-adapters` in diff --git a/infra/redis-sdk/acl/all-accounts.acl b/infra/redis-sdk/acl/lane-fixture-accounts.acl similarity index 100% rename from infra/redis-sdk/acl/all-accounts.acl rename to infra/redis-sdk/acl/lane-fixture-accounts.acl diff --git a/infra/redis-sdk/cluster/compose.yml b/infra/redis-sdk/cluster/compose.yml index 6ef79fd7..aefe63ff 100644 --- a/infra/redis-sdk/cluster/compose.yml +++ b/infra/redis-sdk/cluster/compose.yml @@ -42,7 +42,7 @@ x-node: &node --min-replicas-max-lag 1 \ --masteruser ca-skeleton-replication \ --masterauth fixture-replication \ - --aclfile /etc/redis/acl/all-accounts.acl + --aclfile /etc/redis/acl/lane-fixture-accounts.acl healthcheck: test: ["CMD-SHELL", "[ \"$$(redis-cli -p $$NODE_PORT --user ca-skeleton-application --pass fixture-application --no-auth-warning ping)\" = PONG ]"] interval: 2s diff --git a/infra/redis-sdk/sentinel/compose.yml b/infra/redis-sdk/sentinel/compose.yml index 49aea446..9aab8f37 100644 --- a/infra/redis-sdk/sentinel/compose.yml +++ b/infra/redis-sdk/sentinel/compose.yml @@ -51,7 +51,7 @@ x-data-node: &data-node --min-replicas-max-lag 1 \ --masteruser ca-skeleton-replication \ --masterauth fixture-replication \ - --aclfile /etc/redis/acl/all-accounts.acl + --aclfile /etc/redis/acl/lane-fixture-accounts.acl healthcheck: test: ["CMD-SHELL", "[ \"$$(redis-cli -p $$NODE_PORT --user ca-skeleton-application --pass fixture-application --no-auth-warning ping)\" = PONG ]"] interval: 2s diff --git a/infra/redis-sdk/standalone/compose.yml b/infra/redis-sdk/standalone/compose.yml index df271ab7..ea8f4be0 100644 --- a/infra/redis-sdk/standalone/compose.yml +++ b/infra/redis-sdk/standalone/compose.yml @@ -14,7 +14,7 @@ services: - --save - "" - --aclfile - - /etc/redis/acl/all-accounts.acl + - /etc/redis/acl/lane-fixture-accounts.acl volumes: - ../acl:/etc/redis/acl:ro ports: diff --git a/infra/redis-sdk/tls/compose.yml b/infra/redis-sdk/tls/compose.yml index 051b0221..218adb92 100644 --- a/infra/redis-sdk/tls/compose.yml +++ b/infra/redis-sdk/tls/compose.yml @@ -61,7 +61,7 @@ services: - --save - "" - --aclfile - - /etc/redis/acl/all-accounts.acl + - /etc/redis/acl/lane-fixture-accounts.acl ports: - "${REDIS_PORT:-6390}:6379" healthcheck: diff --git a/scripts/feature-scaffold/probes/src/domain-core/src/test/java/dev/caskeleton/domain/__feature__/testkit/CallerTransaction__Feature__RepositoryContractTest.java.tmpl b/scripts/feature-scaffold/probes/src/domain-core/src/test/java/dev/caskeleton/domain/__feature__/testkit/CallerTransaction__Feature__RepositoryContractTest.java.tmpl new file mode 100644 index 00000000..e1cc1e27 --- /dev/null +++ b/scripts/feature-scaffold/probes/src/domain-core/src/test/java/dev/caskeleton/domain/__feature__/testkit/CallerTransaction__Feature__RepositoryContractTest.java.tmpl @@ -0,0 +1,51 @@ +package dev.caskeleton.domain.__feature__.testkit; + +import dev.caskeleton.domain.__feature__.__Feature__; +import dev.caskeleton.domain.__feature__.__Feature__Id; +import dev.caskeleton.domain.__feature__.__Feature__Repository; +import java.util.Optional; +import java.util.function.Supplier; + +/** + * Round-trip probe, never generated into a feature: a repository that refuses work outside its + * caller's transaction, as docs/manual/04-persistence.md teaches a real adapter to, inherits the + * generated contract and passes it (review 2026-11-03 RR-01-01). + */ +class CallerTransaction__Feature__RepositoryContractTest extends __Feature__RepositoryContract { + + private final InMemory__Feature__Repository rows = new InMemory__Feature__Repository(); + private boolean transactionActive; + + @Override + protected __Feature__Repository repository() { + return new __Feature__Repository() { + @Override + public __Feature__ save(__Feature__ aggregate) { + requireTransaction(); + return rows.save(aggregate); + } + + @Override + public Optional<__Feature__> findById(__Feature__Id id) { + requireTransaction(); + return rows.findById(id); + } + }; + } + + @Override + protected T inTransaction(Supplier work) { + transactionActive = true; + try { + return work.get(); + } finally { + transactionActive = false; + } + } + + private void requireTransaction() { + if (!transactionActive) { + throw new IllegalStateException("requires an active caller transaction"); + } + } +} diff --git a/scripts/feature-scaffold/templates/src/app-bootstrap/src/main/java/dev/caskeleton/bootstrap/autoconfigure/__feature__/__Feature__Config.java.tmpl b/scripts/feature-scaffold/templates/src/app-bootstrap/src/main/java/dev/caskeleton/bootstrap/autoconfigure/__feature__/__Feature__Config.java.tmpl index 4cc9b7c7..7fe6caf5 100644 --- a/scripts/feature-scaffold/templates/src/app-bootstrap/src/main/java/dev/caskeleton/bootstrap/autoconfigure/__feature__/__Feature__Config.java.tmpl +++ b/scripts/feature-scaffold/templates/src/app-bootstrap/src/main/java/dev/caskeleton/bootstrap/autoconfigure/__feature__/__Feature__Config.java.tmpl @@ -19,15 +19,22 @@ import org.springframework.context.annotation.Import; * *

Turns itself on from what is on the classpath: a {@link __Feature__Repository} adapter and a * {@link TransactionPort}. Until a repository adapter exists the feature is absent rather than half - * present (docs/manual/02-new-feature.md step 5). When the adapter comes from another - * auto-configuration, name it in {@code afterName} so this condition sees its bean. + * present (docs/manual/02-new-feature.md step 5). + * + *

{@code @ConditionalOnBean} only sees beans of auto-configurations processed before this one, + * and nothing orders them by itself: without {@code afterName} the feature would be on or off + * depending on the class names' alphabetical order. {@code TransactionPort} comes from + * persistence-jpa's {@code PersistenceJpaAutoConfiguration}, named here. When the repository adapter + * comes from another auto-configuration, add its name to the list. * *

The use cases are beans, not objects the controller builds, because the method security * advisor proxies beans: {@code @RequiresPermission} on create is enforced by that proxy. Grant * {@code __feature_kebab__:create} to a role under {@code app.authz.role-permissions}; until then * every create is refused (docs/manual/07-authorization.md). */ -@AutoConfiguration +@AutoConfiguration( + afterName = + "dev.caskeleton.adapter.outbound.persistence.autoconfigure.PersistenceJpaAutoConfiguration") @ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.SERVLET) @ConditionalOnBean({__Feature__Repository.class, TransactionPort.class}) @Import(__Feature__Controller.class) diff --git a/scripts/feature-scaffold/templates/src/domain-core/src/testFixtures/java/dev/caskeleton/domain/__feature__/testkit/__Feature__RepositoryContract.java.tmpl b/scripts/feature-scaffold/templates/src/domain-core/src/testFixtures/java/dev/caskeleton/domain/__feature__/testkit/__Feature__RepositoryContract.java.tmpl index cbab07f8..290a1492 100644 --- a/scripts/feature-scaffold/templates/src/domain-core/src/testFixtures/java/dev/caskeleton/domain/__feature__/testkit/__Feature__RepositoryContract.java.tmpl +++ b/scripts/feature-scaffold/templates/src/domain-core/src/testFixtures/java/dev/caskeleton/domain/__feature__/testkit/__Feature__RepositoryContract.java.tmpl @@ -6,12 +6,17 @@ import dev.caskeleton.domain.__feature__.__Feature__; import dev.caskeleton.domain.__feature__.__Feature__Id; import dev.caskeleton.domain.__feature__.__Feature__Repository; import java.util.Optional; +import java.util.function.Supplier; import org.junit.jupiter.api.Test; /** * What every {@link __Feature__Repository} promises. {@link InMemory__Feature__Repository} extends * it in domain-core's tests; a real adapter extends it in a lane that runs against its real store * (docs/manual/04-persistence.md). Add a test here before adding a repository method. + * + *

Every repository call goes through {@link #inTransaction(Supplier)}. A real adapter refuses + * work outside its caller's transaction ({@code CallerTransaction.require}), so its contract test + * opens one there; the fake needs none and keeps the default. */ public abstract class __Feature__RepositoryContract { @@ -25,13 +30,25 @@ public abstract class __Feature__RepositoryContract { */ protected abstract __Feature__Repository repository(); + /** + * Runs one repository call the way a use case does, inside its transaction. The default runs it + * directly; an adapter that requires a caller transaction opens and commits one here. + * + * @param work one call on the repository + * @param what the call returns + * @return what the call returned + */ + protected T inTransaction(Supplier work) { + return work.get(); + } + @Test void aSaved__Feature__IsFoundByItsId() { // given - repository().save(__Feature__.create(FIRST, "first")); + inTransaction(() -> repository().save(__Feature__.create(FIRST, "first"))); // when - Optional<__Feature__> found = repository().findById(FIRST); + Optional<__Feature__> found = inTransaction(() -> repository().findById(FIRST)); // then assertThat(found.map(__Feature__::name)).contains("first"); @@ -39,18 +56,19 @@ public abstract class __Feature__RepositoryContract { @Test void anIdNothingWasSavedUnderIsNotFound() { - assertThat(repository().findById(UNKNOWN)).isEmpty(); + assertThat(inTransaction(() -> repository().findById(UNKNOWN))).isEmpty(); } @Test void savingTheSameIdAgainReplacesTheStoredState() { // given - repository().save(__Feature__.create(FIRST, "first")); + inTransaction(() -> repository().save(__Feature__.create(FIRST, "first"))); // when - repository().save(__Feature__.restore(FIRST, "renamed")); + inTransaction(() -> repository().save(__Feature__.restore(FIRST, "renamed"))); // then - assertThat(repository().findById(FIRST).map(__Feature__::name)).contains("renamed"); + assertThat(inTransaction(() -> repository().findById(FIRST)).map(__Feature__::name)) + .contains("renamed"); } } diff --git a/scripts/verify-feature-scaffold.sh b/scripts/verify-feature-scaffold.sh index 4bcfa754..01573e99 100755 --- a/scripts/verify-feature-scaffold.sh +++ b/scripts/verify-feature-scaffold.sh @@ -12,6 +12,9 @@ # 그 전에 템플릿 패키지 이름(Operation, Notification, Common)을 넣어 거절되는지와 아무것도 쓰지 # 않는지 확인한다. # +# 생성물의 테스트가 모두 실제로 돌았는지 템플릿 목록으로 확인하고, 생성된 저장소 계약을 호출자 +# 트랜잭션을 요구하는 저장소로 한 번 더 돌린다(scripts/feature-scaffold/probes, 기능에는 생성되지 않는다). +# # 루트 Gradle 태스크 `verifyFeatureScaffold`가 이 스크립트를 부르고 mainCheck에 들어 있다. set -euo pipefail @@ -92,18 +95,68 @@ cd "$copy/src" :app-bootstrap:check \ architectureCheck -# 생성된 테스트가 실제로 실행되었는지: 각 leaf의 테스트 결과에 기능 테스트 클래스가 있어야 한다. +# 생성된 테스트가 실제로 실행되었는지: 템플릿의 테스트 클래스 하나하나가 자기 leaf의 결과 XML을 +# 남겨야 한다. 목록을 손으로 적지 않고 템플릿에서 만든다 — 손으로 적은 목록은 8개 중 4개만 보았고, +# 필터·태그가 바뀌어 나머지가 실행되지 않아도 녹색이었다(review 2026-11-03 RR-01-10). +scaffold_templates="$copy/scripts/feature-scaffold/templates/src" +mapfile -t test_templates < <(cd "$scaffold_templates" && find . -path '*/src/test/java/*' -name '*Test.java.tmpl' | sort) +[ "${#test_templates[@]}" -gt 0 ] || { echo "템플릿에 테스트가 없다: $scaffold_templates" >&2; exit 1; } + +# 결과 XML 하나를 확인한다: 있고, 한 건 이상 돌았고, 실패·오류가 없다. +require_test_result() { + local result="$1" what="$2" + [ -f "$result" ] || { echo "$what 실행되지 않았다: $result" >&2; exit 1; } + if grep -q 'tests="0"' "$result"; then + echo "$what 한 건도 돌지 않았다: $result" >&2 + exit 1 + fi + if grep -q '&2 + exit 1 + fi +} + +# 템플릿 경로(/src/test/java/<패키지 경로>/<클래스>.java.tmpl)를 그 leaf의 결과 XML 경로로. +result_of() { + local template="${1#./}" name="$2" package_name="$3" + local leaf="${template%%/src/test/java/*}" + local class_path="${template#*/src/test/java/}" + class_path="${class_path%.java.tmpl}" + local fqcn + fqcn="$(printf '%s' "$class_path" | sed -e "s/__Feature__/$name/g" -e "s/__feature__/$package_name/g" | tr '/' '.')" + printf '%s/build/test-results/test/TEST-%s.xml' "$leaf" "$fqcn" +} + for name in "${names[@]}"; do - for result in \ - "domain-core/build/test-results/test/TEST-dev.caskeleton.domain.$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]').${name}Test.xml" \ - "application-core/build/test-results/test/TEST-dev.caskeleton.application.$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]').Create${name}UseCaseTest.xml" \ - "adapter/inbound/web/build/test-results/test/TEST-dev.caskeleton.adapter.inbound.web.$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]').${name}ControllerTest.xml" \ - "app-bootstrap/build/test-results/test/TEST-dev.caskeleton.bootstrap.autoconfigure.$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]').${name}ConfigTest.xml"; do - [ -f "$result" ] || { echo "생성된 테스트가 실행되지 않았다: $result" >&2; exit 1; } - if grep -q '&2 - exit 1 - fi + package_name="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]')" + for template in "${test_templates[@]}"; do + require_test_result "$(result_of "$template" "$name" "$package_name")" "생성된 테스트가" + done +done + +# 생성된 저장소 계약은 실제 adapter가 상속할 수 있어야 한다. 매뉴얼 04장의 adapter는 호출자 +# 트랜잭션 밖의 호출을 거부하므로, 그렇게 하는 저장소로 계약을 돌려 본다(review 2026-11-03 RR-01-01). +# 탐침은 기능에 생성되지 않는 테스트라 PR 게이트(check)와 따로 test 태스크만 돌린다. +probe_templates="$copy/scripts/feature-scaffold/probes/src" +mapfile -t probes < <(cd "$probe_templates" && find . -type f -name '*.java.tmpl' | sort) +probe_filters=() +for name in "${names[@]}"; do + package_name="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]')" + for probe in "${probes[@]}"; do + target="$(printf '%s' "${probe#./}" | sed -e "s/__Feature__/$name/g" -e "s/__feature__/$package_name/g")" + target="${target%.tmpl}" + mkdir -p "$(dirname "$copy/src/$target")" + sed -e "s/__Feature__/$name/g" -e "s/__feature__/$package_name/g" \ + "$probe_templates/${probe#./}" > "$copy/src/$target" + class_path="${target#*/src/test/java/}" + probe_filters+=(--tests "$(printf '%s' "${class_path%.java}" | tr '/' '.')") + done +done +./gradlew --console=plain :domain-core:test "${probe_filters[@]}" +for name in "${names[@]}"; do + package_name="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]')" + for probe in "${probes[@]}"; do + require_test_result "$(result_of "$probe" "$name" "$package_name")" "저장소 계약 탐침이" done done echo "feature scaffold 왕복 통과: ${names[*]}" diff --git a/src/.env.example b/src/.env.example index 895cc69d..200bb942 100644 --- a/src/.env.example +++ b/src/.env.example @@ -135,6 +135,7 @@ APP_CACHE_CANONICAL_DEFAULT_PROVIDER=disabled APP_REDIS_ENABLED=true APP_CACHE_REDIS_POSITIVE_HARD_TTL=5m APP_CACHE_REDIS_NEGATIVE_TTL=10s +APP_CACHE_REDIS_MAXIMUM_SCRIPT_CONNECTIONS=4 APP_IDEMPOTENCY_REDIS_COMMAND_TIMEOUT=200ms APP_LEASE_REDIS_COMMAND_TIMEOUT=200ms APP_LEASE_REDIS_CONTENTION_RETRY_AFTER=50ms @@ -151,7 +152,7 @@ APP_REDIS_TIMEOUT_SCRIPT=1s # reads the password from and authenticates as . With the application # reference empty and anonymous access not accepted, the application refuses to start while Redis # is on; with the advanced one empty, scripts run as the application account and startup warns. The -# Compose Redis lanes connect as the fixture accounts of infra/redis-sdk/acl/all-accounts.acl (the +# Compose Redis lanes connect as the fixture accounts of infra/redis-sdk/acl/lane-fixture-accounts.acl (the # Redis comment in src/config/runtime/compose-profile-contracts.json). APP_REDIS_AUTHENTICATION_ADVANCED_CREDENTIAL_REFERENCE= APP_REDIS_AUTHENTICATION_ANONYMOUS_ACCESS_ACCEPTED=false @@ -293,6 +294,9 @@ APP_GRAPHQL_PER_PARENT_FETCH_WARNING_THRESHOLD=10 # Optional build-composition: these bind only when :adapter:inbound:websocket is present. # The default bootJar does not contain that leaf. APP_WEBSOCKET_ENABLED=true +# Explicit offs for two more capabilities that are on when present (the Compose lanes set them). +APP_OBJECTSTORAGE_ENABLED=true +APP_GRPC_SERVER_ENABLED=true APP_WEBSOCKET_ENDPOINT=/ws APP_WEBSOCKET_ALLOWED_ORIGINS=http://localhost:3000 APP_WEBSOCKET_BROADCAST_DESTINATION=/topic/events diff --git a/src/adapter/inbound/graphql/README.md b/src/adapter/inbound/graphql/README.md index 69252f30..08d0f9d2 100644 --- a/src/adapter/inbound/graphql/README.md +++ b/src/adapter/inbound/graphql/README.md @@ -77,7 +77,12 @@ instrumentation, data fetcher resolver 세 곳이 같은 bean 을 받는다). - **client 문구**: shared-contract `ClientErrorMessages`(Category 별 한국어 문구 표 하나, 소유자 결정 11). 예외 메시지는 절대 나가지 않는다. - **retryable**: 코드의 `retryable()` 이다(category 에서 계산하지 않는다). 플랫폼 자체 코드만 wire - category 기본값을 쓴다. + category 기본값을 쓴다. 그래서 공유 코드의 값을 바꾸면 이 wire 도 바뀐다: 처리 중 중복 + `IDEMPOTENT_IN_FLIGHT` 는 2026-11-01 리뷰(02-01)부터 `retryable: true` 다 — 같은 키로 다시 보내면 서버가 + 판정한다(`docs/web/http-contract.md` §2). `GraphQlWireErrorMapperTest` 가 고정한다(재리뷰 RR-02-06). +- **`REQUEST_TIMEOUT` 은 web 과 이름만 같다.** 여기의 `category` 는 GraphQL wire 전용 6값 어휘(`TIMEOUT`)이고, + web 의 같은 이름 코드는 HTTP 503·operational category `TRANSIENT_DEPENDENCY` 다 + (`docs/web/http-contract.md` §4). 둘 다 `retryable: true` 다. - **body 크기 초과**: `PAYLOAD_TOO_LARGE`(web 과 같은 코드). 문서 모양 거부(`GRAPHQL_DOCUMENT_SHAPE_REJECTED`)는 별도 코드. diff --git a/src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/context/GraphQlDeadline.java b/src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/context/GraphQlDeadline.java index 3baa3a76..97d0aabf 100644 --- a/src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/context/GraphQlDeadline.java +++ b/src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/context/GraphQlDeadline.java @@ -14,6 +14,9 @@ import java.time.Instant; *

  • {@code GraphQlPlatformWebInterceptor} ends the request with {@code REQUEST_TIMEOUT} when it * passes, and Spring cancels the execution so no further field is dispatched. A resolver * already running is not interrupted. + *
  • {@code GraphQlDeadlineInstrumentation} refuses to start a non-trivial resolver once it has + * passed, failing that field with {@code REQUEST_TIMEOUT}; this is what still holds when the + * whole execution runs on the request thread and the interceptor's timer starts too late. *
  • {@code GraphQlCostBudgetHandler} refuses to start an execution whose deadline has passed. *
  • {@code GraphQlBatchExecutor} clamps each platform-registered DataLoader batch to what is * left, and stops between chunks once it is spent. diff --git a/src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/cost/GraphQlComplexityCalculator.java b/src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/cost/GraphQlComplexityCalculator.java index 90aae3a8..943feffe 100644 --- a/src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/cost/GraphQlComplexityCalculator.java +++ b/src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/cost/GraphQlComplexityCalculator.java @@ -121,11 +121,6 @@ public final class GraphQlComplexityCalculator { } } - /** The effective page size for a connection request. */ - public int effectivePageSize(Integer first, Integer last) { - return first != null ? first : last != null ? last : defaultPageSize; - } - private void requireAllowedPage(Integer requested) { if (requested == null) { return; diff --git a/src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/cost/GraphQlDocumentComplexityScorer.java b/src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/cost/GraphQlDocumentComplexityScorer.java index b67d7366..cd3b109b 100644 --- a/src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/cost/GraphQlDocumentComplexityScorer.java +++ b/src/adapter/inbound/graphql/src/main/java/dev/caskeleton/adapter/inbound/graphql/cost/GraphQlDocumentComplexityScorer.java @@ -15,6 +15,7 @@ import graphql.language.Value; import graphql.language.VariableDefinition; import graphql.language.VariableReference; import graphql.schema.GraphQLArgument; +import graphql.schema.GraphQLCompositeType; import graphql.schema.GraphQLFieldDefinition; import graphql.schema.GraphQLFieldsContainer; import graphql.schema.GraphQLList; @@ -59,6 +60,13 @@ import java.util.Objects; *

    Pricing an omitted argument as one row was the bypass: {@code items(first: 20)} scored 410 * while {@code items} scored 40, and four nested lists added up instead of multiplying. * + *

    A selection set is walked whatever kind of composite type it selects from. A union has no + * fields of its own — only {@code __typename} — so its selections are inline fragments and fragment + * spreads, each priced against its type condition. Treating "not a fields container" as "nothing + * below" was the second bypass (re-review 2026-11-03, RR-02-01): {@code search { ... on Post { + * comments(first: 100000) } }} scored 20, and neither the budget nor the page maximum applied under + * any union-typed field. + * *

    Traversal is bounded, and fragment cycles are cut by tracking the expansion path. This runs on * documents that have passed validation, but the bound stays because the scorer is also used from * the pre-execution path where a hostile document is exactly what it is meant to price. @@ -143,7 +151,7 @@ public final class GraphQlDocumentComplexityScorer { private long selectionSetCost( GraphQLSchema schema, - GraphQLFieldsContainer parent, + GraphQLCompositeType parent, SelectionSet selectionSet, Map fragments, PageVariables variables, @@ -173,10 +181,10 @@ public final class GraphQlDocumentComplexityScorer { expansionPath, listsAreThePage)); } else if (selection instanceof InlineFragment inlineFragment) { - GraphQLFieldsContainer target = + GraphQLCompositeType target = inlineFragment.getTypeCondition() == null ? parent - : fieldsContainer(schema, inlineFragment.getTypeCondition().getName()); + : compositeType(schema, inlineFragment.getTypeCondition().getName()); total = GraphQlComplexityCalculator.saturatedAdd( total, @@ -196,10 +204,10 @@ public final class GraphQlDocumentComplexityScorer { if (fragment == null || expansionPath.contains(spread.getName())) { continue; } - GraphQLFieldsContainer target = + GraphQLCompositeType target = fragment.getTypeCondition() == null ? parent - : fieldsContainer(schema, fragment.getTypeCondition().getName()); + : compositeType(schema, fragment.getTypeCondition().getName()); expansionPath.push(spread.getName()); total = GraphQlComplexityCalculator.saturatedAdd( @@ -221,7 +229,7 @@ public final class GraphQlDocumentComplexityScorer { private long fieldCost( GraphQLSchema schema, - GraphQLFieldsContainer parent, + GraphQLCompositeType parent, Field field, Map fragments, PageVariables variables, @@ -235,8 +243,12 @@ public final class GraphQlDocumentComplexityScorer { return 0; } - GraphQLFieldDefinition definition = parent.getFieldDefinition(field.getName()); String coordinate = parent.getName() + "." + field.getName(); + // A union declares no fields; validation rejects anything but __typename directly under one. + GraphQLFieldDefinition definition = + parent instanceof GraphQLFieldsContainer container + ? container.getFieldDefinition(field.getName()) + : null; if (definition == null) { // Validation rejects an unknown field; price what is there rather than invent a type. return calculator.fieldCost(coordinate, 0).total(); @@ -250,7 +262,7 @@ public final class GraphQlDocumentComplexityScorer { long childCost = selectionSetCost( schema, - fieldsContainer(definition.getType()), + compositeType(definition.getType()), field.getSelectionSet(), fragments, variables, @@ -353,14 +365,14 @@ public final class GraphQlDocumentComplexityScorer { }; } - private static GraphQLFieldsContainer fieldsContainer(GraphQLSchema schema, String typeName) { - GraphQLType type = schema.getType(typeName); - return fieldsContainer(type); + private static GraphQLCompositeType compositeType(GraphQLSchema schema, String typeName) { + return compositeType(schema.getType(typeName)); } - private static GraphQLFieldsContainer fieldsContainer(GraphQLType type) { + /** The object, interface or union a selection set selects from; null for a leaf type. */ + private static GraphQLCompositeType compositeType(GraphQLType type) { GraphQLType unwrapped = unwrap(type); - return unwrapped instanceof GraphQLFieldsContainer container ? container : null; + return unwrapped instanceof GraphQLCompositeType composite ? composite : null; } private static GraphQLType unwrap(GraphQLType type) { diff --git a/src/adapter/inbound/graphql/src/test/java/dev/caskeleton/adapter/inbound/graphql/cost/GraphQlDocumentComplexityScorerTest.java b/src/adapter/inbound/graphql/src/test/java/dev/caskeleton/adapter/inbound/graphql/cost/GraphQlDocumentComplexityScorerTest.java index cc5d8dd7..2e06b0a5 100644 --- a/src/adapter/inbound/graphql/src/test/java/dev/caskeleton/adapter/inbound/graphql/cost/GraphQlDocumentComplexityScorerTest.java +++ b/src/adapter/inbound/graphql/src/test/java/dev/caskeleton/adapter/inbound/graphql/cost/GraphQlDocumentComplexityScorerTest.java @@ -34,11 +34,19 @@ class GraphQlDocumentComplexityScorerTest { pagedWithDefault(first: Int = 50): [Item!]! itemConnection(first: Int): ItemConnection node: Item + allItems: [Item!]! + search: [SearchResult!]! + nodes: [Named!]! } - type Item { + union SearchResult = Item + interface Named { + name: String! + } + type Item implements Named { name: String! children: [Item!]! grid: [[Item!]!]! + paged(first: Int): [Item!]! } type ItemConnection { edges: [ItemEdge!]! @@ -48,10 +56,15 @@ class GraphQlDocumentComplexityScorerTest { } """; + // Pricing never resolves a runtime type; the resolvers only satisfy the schema generator. private static final GraphQLSchema SCHEMA = new SchemaGenerator() .makeExecutableSchema( - new SchemaParser().parse(SDL), RuntimeWiring.newRuntimeWiring().build()); + new SchemaParser().parse(SDL), + RuntimeWiring.newRuntimeWiring() + .type("SearchResult", type -> type.typeResolver(environment -> null)) + .type("Named", type -> type.typeResolver(environment -> null)) + .build()); private static final long BUDGET = 10_000; @@ -246,6 +259,87 @@ class GraphQlDocumentComplexityScorerTest { assertThat(score).isEqualTo(20L + 20L); } + @Test + @DisplayName("union 필드 아래 inline fragment 의 중첩 list 도 일반 list 와 같이 곱한다") + void nestedListsUnderAUnionFieldMultiplyLikeAPlainList() { + // given + String plain = "query Q { allItems { children { children { children { name } } } } }"; + + // when + long score = + score( + "query Q { search { ... on Item { children { children { children { name } } } } } }", + Map.of()); + + // then + assertThat(score).isEqualTo(score(plain, Map.of())); + assertThatThrownBy(() -> calculator.verify(new GraphQlComplexityResult(score), BUDGET)) + .isInstanceOf(GraphQlComplexityRejectedException.class); + } + + @Test + @DisplayName("union 필드 아래 fragment spread 도 그 타입으로 내려가 곱한다") + void aFragmentSpreadUnderAUnionFieldIsPricedAgainstItsTypeCondition() { + // given + String plain = "query Q { allItems { children { children { children { name } } } } }"; + + // when + long score = + score( + "query Q { search { ...F } } " + + "fragment F on Item { children { children { children { name } } } }", + Map.of()); + + // then + assertThat(score).isEqualTo(score(plain, Map.of())); + } + + @Test + @DisplayName("union 필드 아래 page 인자도 상한을 넘으면 page size exceeds maximum 으로 거부된다") + void aPageSizeUnderAUnionFieldIsCheckedAgainstTheMaximum() { + // given + String document = "query Q { search { ... on Item { paged(first: 100000) { name } } } }"; + + // when + Throwable thrown = catchThrowable(() -> score(document, Map.of())); + + // then + assertThat(thrown) + .isInstanceOf(GraphQlComplexityRejectedException.class) + .hasMessageContaining("page size exceeds maximum"); + } + + @Test + @DisplayName("interface 필드 아래 inline fragment 의 중첩 list 도 일반 list 와 같이 곱한다") + void nestedListsUnderAnInterfaceFieldMultiplyLikeAPlainList() { + // given + String plain = "query Q { allItems { children { children { children { name } } } } }"; + + // when + long score = + score( + "query Q { nodes { ... on Item { children { children { children { name } } } } } }", + Map.of()); + + // then + assertThat(score).isEqualTo(score(plain, Map.of())); + } + + @Test + @DisplayName("interface 필드 아래 page 인자도 상한을 넘으면 page size exceeds maximum 으로 거부된다") + void aPageSizeUnderAnInterfaceFieldIsCheckedAgainstTheMaximum() { + // given + String document = "query Q { nodes { ... on Item { paged(first: 100000) { name } } } }"; + + // when + Throwable thrown = catchThrowable(() -> score(document, Map.of())); + + // then + assertThat(thrown) + .isInstanceOf(GraphQlComplexityRejectedException.class) + .hasMessageContaining("page size exceeds maximum"); + } + private long score(String source, Map variables) { return score(calculator, source, variables); } diff --git a/src/adapter/inbound/graphql/src/test/java/dev/caskeleton/adapter/inbound/graphql/error/GraphQlWireErrorMapperTest.java b/src/adapter/inbound/graphql/src/test/java/dev/caskeleton/adapter/inbound/graphql/error/GraphQlWireErrorMapperTest.java index 3a39c97d..c27ccb56 100644 --- a/src/adapter/inbound/graphql/src/test/java/dev/caskeleton/adapter/inbound/graphql/error/GraphQlWireErrorMapperTest.java +++ b/src/adapter/inbound/graphql/src/test/java/dev/caskeleton/adapter/inbound/graphql/error/GraphQlWireErrorMapperTest.java @@ -55,6 +55,29 @@ class GraphQlWireErrorMapperTest { .containsExactly("DB_QUERY_CANCELED", false); } + /** + * The in-flight duplicate's {@code retryable} became {@code true} in the 2026-11-01 review + * (02-01): a resend with the same key is judged by the server, so it is safe. GraphQL carries the + * code's own value, so the change reached this wire too; this pins it (re-review RR-02-06). + */ + @org.junit.jupiter.api.Test + @DisplayName("처리 중 중복(IDEMPOTENT_IN_FLIGHT)은 GraphQL extension 에서도 retryable=true 로 나간다") + void anInFlightDuplicateIsRetryableOnTheGraphQlWire() { + // given + GraphQlWireErrorMapper mapper = GraphQlWireErrorMapper.masking(); + RuntimeException failure = + dev.caskeleton.application.idempotency.IdempotencyInFlightException.forScopeDigest( + "digest"); + + // when + GraphQlWireError error = mapper.map(failure, GraphQlErrorContext.test()); + + // then + assertThat(error.extensions()) + .extracting(extensions -> extensions.get("code"), extensions -> extensions.get("retryable")) + .containsExactly("IDEMPOTENT_IN_FLIGHT", true); + } + @org.junit.jupiter.api.Test void anIllegalArgumentIsABadParameterRequestError() { // given diff --git a/src/adapter/inbound/grpc/README.md b/src/adapter/inbound/grpc/README.md index 9c45f9fd..80201aa2 100644 --- a/src/adapter/inbound/grpc/README.md +++ b/src/adapter/inbound/grpc/README.md @@ -140,7 +140,8 @@ handler throw, listener callback throw, `responseObserver.onError(...)`, raw - **와이어 status(coarse)** 는 `GrpcStatusMapper.toStatus(Category)` 가 결정한다(HTTP status 가 coarse 인 것과 동형). 정확한 `code`/`category`/`retryable` 은 `Status` trailer `Metadata` (`error-code` / `error-category` / `error-retryable`)에 실어 클라이언트가 switch 하게 한다. - `retryable` 은 코드의 값이다. status description 은 shared-contract `ClientErrorMessages` 의 + `retryable` 은 코드의 값이다(처리 중 중복 `IDEMPOTENT_IN_FLIGHT` 는 2026-11-01 리뷰 02-01부터 `ABORTED` + + `error-retryable: true` — 같은 키로 다시 보내면 서버가 판정한다. `GrpcStatusMapperTest` 가 고정, 재리뷰 RR-02-06). status description 은 shared-contract `ClientErrorMessages` 의 Category 별 한국어 문구(web·GraphQL 과 같은 표)다. - **`IllegalArgumentException`** 은 `BAD_PARAMETER`(`INVALID_ARGUMENT`) — 모든 전송이 같다. - **`ApiErrorCode` 추출**: feature 예외는 `ApiErrorCarrier`(이 모듈이 제공하는 전송-중립 hook)를 diff --git a/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcStatusMapperTest.java b/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcStatusMapperTest.java index 57aa1bd4..c996114a 100644 --- a/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcStatusMapperTest.java +++ b/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcStatusMapperTest.java @@ -7,6 +7,7 @@ import dev.caskeleton.shared.error.OperationalError; import io.grpc.Metadata; import io.grpc.Status; import java.util.EnumSet; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -64,4 +65,22 @@ class GrpcStatusMapperTest { // then assertThat(retryable).isEqualTo("false"); } + + /** + * The in-flight duplicate's {@code retryable} became {@code true} in the 2026-11-01 review + * (02-01). The trailer carries the code's own value, so the change reached the gRPC wire as well: + * status {@code ABORTED} (category {@code CONFLICT}) with {@code error-retryable: true} + * (re-review RR-02-06). + */ + @Test + @DisplayName("처리 중 중복(IDEMPOTENT_IN_FLIGHT)은 ABORTED 이고 trailer 의 retryable 은 true 다") + void anInFlightDuplicateIsAbortedAndRetryableOnTheGrpcWire() { + // when + Metadata trailers = mapper.trailersFor(OperationalError.IDEMPOTENT_IN_FLIGHT); + + // then + assertThat(mapper.statusFor(OperationalError.IDEMPOTENT_IN_FLIGHT).getCode()) + .isEqualTo(Status.Code.ABORTED); + assertThat(trailers.get(GrpcStatusMapper.RETRYABLE_KEY)).isEqualTo("true"); + } } diff --git a/src/adapter/inbound/web/README.md b/src/adapter/inbound/web/README.md index b71ce115..0754f787 100644 --- a/src/adapter/inbound/web/README.md +++ b/src/adapter/inbound/web/README.md @@ -241,6 +241,7 @@ commit-aware response wrapper 계약을 구현한다. 또한 이 모듈은 HTML | `PreconditionFailedException` | `PRECONDITION_FAILED` | 412 | D15: `If-Match` 불일치 쓰기 = 낙관적 동시성 충돌 → 412 (raw 409/500 금지) | | `PageValidationException` | `VALIDATION_FAILED` | 400 | D18/D20/D21: 계약 범위 밖 페이지/정렬/필터 파라미터. field + reasonCode 를 details 로 | | `IdempotencyUnavailableException` | `IDEMPOTENCY_UNAVAILABLE` | 503 (retryable=true) | 조정 저장소가 응답하지 않아 action 전에 거부. carrier 분기 | +| `IdempotencyCompletionUnknownException` | `IDEMPOTENCY_COMPLETION_UNKNOWN` | 503 (retryable=true) | action 이 돈 뒤 결과 기록(complete) 중 저장소가 응답하지 않음. 결과 불명 — 같은 키로 재전송, 새 키 금지(`docs/web/http-contract.md` §2, 재리뷰 RR-02-03). carrier 분기 | | `IdempotencyRecoveryRequiredException` | `IDEMPOTENCY_RECOVERY_REQUIRED` | 500 (retryable=false) | 이전 시도의 외부 효과 여부를 판단할 수 없음. carrier 분기 | | `IdempotencyInFlightException` | `IDEMPOTENT_IN_FLIGHT` | 409 (retryable=true, `Retry-After: 2`) | 같은 scope 의 다른 시도가 아직 처리 중(executor 는 기다리지 않는다). 클라이언트 행동: 결과 불명. 같은 `Idempotency-Key`로 `Retry-After` 뒤 재전송. 새 키 금지(`docs/web/http-contract.md` §2). 진단정보(scope digest)는 클라이언트 미도달 | | `IdempotencyRequestMismatchException` | `IDEMPOTENT_REQUEST_MISMATCH` | 422 (retryable=false) | D8: `Idempotency-Key` 를 다른 본문으로 재사용. 다시 보내지 않는다(`docs/web/http-contract.md` §2). fingerprint/scope 노출 금지 | @@ -284,9 +285,26 @@ commit-aware response wrapper 계약을 구현한다. 또한 이 모듈은 HTML advice 가 같은 mapper 로 Envelope 를 문자열로 직렬화해 넘긴다. 텍스트를 요청한 클라이언트는 그대로 문자열을 받는다. ArchUnit 으로 `String` 반환을 금지하는 안은 버렸다: `text/plain` 엔드포인트의 정당한 용도를 막고, 규칙을 돌리지 않는 서비스에는 함정이 그대로 남는다. +- **4xx·5xx 본문은 감싸지 않는다.** `success` 는 클라이언트가 가르는 판별자라, 오류 status 본문을 + `success:true` 로 감싸면 성공으로 읽힌다. Boot `/error` 의 map 이 그렇게 나갔다(재리뷰 RR-02-02). - 위치: sample 모듈이 아니라 adapter-web. 실행 앱은 adapter-web 에 의존하지만 sample-portfolio 에는 의존하지 않으므로, 응답 래핑이 실제로 동작하려면 여기 있어야 한다. +### EnvelopeErrorController +- 컨트롤러에 닿기 전에 생긴 오류(보안 방화벽의 `RequestRejectedException`, 필터의 `sendError`, 필터가 + 던진 예외)는 `GlobalExceptionHandler` 를 지나지 않고 컨테이너의 `/error` 디스패치로 간다. Boot 기본 + `BasicErrorController` 는 거기서 `{timestamp, status, error, path}` 를 썼고, advice 가 그것을 + `success:true` 로 감쌌으며 `path` 는 요청 경로 원문이었다(재리뷰 RR-02-02). 이 컨트롤러가 그 자리를 + 대신해 status 를 `GlobalExceptionHandler` 와 같은 표(`HttpStatusErrorCodes`)로 코드에 매핑하고 실패 + Envelope 를 `EnvelopeResponseWriter` 로 쓴다. 요청에서 가져온 값은 싣지 않는다. +- `WebMvcPlatformAutoConfiguration` 이 `ErrorMvcAutoConfiguration` 보다 먼저 `ErrorController` 가 없을 때만 + 등록한다. 어댑터가 자기 `ErrorController` 를 두면 물러난다. +- `SecurityConfig` 는 오류 **디스패치**만 허용한다(`DispatcherType.ERROR`). 예전에는 `/error` 가 공개 경로가 + 아니라서 익명 요청의 400 이 401 로 바뀌었다. `/error` 를 직접 부른 요청은 여전히 인증이 필요하다. + 단언은 실제 Tomcat 의 `TomcatWebContractIT`(공유 `WebContractAssertions`)와 `JwtChainErrorDispatchIT`. +- 앱이 보안 체인을 직접 정의하면(예: sample 의 데모 체인) 같은 한 줄 + (`dispatcherTypeMatchers(DispatcherType.ERROR).permitAll()`)을 그 체인에도 둬야 원래 status 가 나간다. + ### CacheControlFilter - 스켈레톤 기본 HTTP 캐시 정책. - `Cache-Control: no-store` 는 인증된 API 의 안전한 기본값. `Vary: Accept, Accept-Encoding, Authorization` 로 diff --git a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/auth/SecurityConfig.java b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/auth/SecurityConfig.java index 0ca348d0..9c3a634d 100644 --- a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/auth/SecurityConfig.java +++ b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/auth/SecurityConfig.java @@ -2,6 +2,7 @@ package dev.caskeleton.adapter.inbound.web.auth; import dev.caskeleton.adapter.inbound.web.settings.CorsSettings; import dev.caskeleton.adapter.inbound.web.settings.SecuritySettings; +import jakarta.servlet.DispatcherType; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -83,6 +84,11 @@ public class SecurityConfig { .requestCache(cache -> cache.disable()) .authorizeHttpRequests( auth -> { + // The container's error dispatch for a request that already failed. Authorising it + // again turned an anonymous 400 into a 401 that blamed authentication. Only the + // dispatch type is opened: a request that names /error itself is still a request. + // The error controller writes nothing from the request (re-review RR-02-02). + auth.dispatcherTypeMatchers(DispatcherType.ERROR).permitAll(); if (publicPaths.length > 0) { auth.requestMatchers(publicPaths).permitAll(); } diff --git a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/envelope/EnvelopeBodyAdvice.java b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/envelope/EnvelopeBodyAdvice.java index b5dbec59..76ac5ba9 100644 --- a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/envelope/EnvelopeBodyAdvice.java +++ b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/envelope/EnvelopeBodyAdvice.java @@ -12,6 +12,7 @@ import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.StringHttpMessageConverter; import org.springframework.http.server.ServerHttpRequest; import org.springframework.http.server.ServerHttpResponse; +import org.springframework.http.server.ServletServerHttpResponse; import org.springframework.web.bind.annotation.RestControllerAdvice; import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; import tools.jackson.databind.ObjectMapper; @@ -34,6 +35,13 @@ import tools.jackson.databind.json.JsonMapper; * still sees {@code 200 text/plain}. So when the String converter is about to write JSON, the * envelope is serialised here, with the same mapper and wire rules, and the converter writes that * text (review 2026-11-01 02-06). A text client still gets the plain string. + * + *

    A body sent with a 4xx or 5xx status is never wrapped. {@code success} is the discriminator a + * client branches on, so a {@code success:true} envelope around an error-status body tells it the + * request worked. Boot's {@code /error} map reached clients exactly that way until the error + * dispatch got its own envelope writer ({@code EnvelopeErrorController}, re-review 2026-11-03 + * RR-02-02). Failure envelopes come from the error handlers; a handler that returns some other body + * with an error status gets that body on the wire as it wrote it, not a false success. */ @RestControllerAdvice public final class EnvelopeBodyAdvice implements ResponseBodyAdvice { @@ -74,10 +82,18 @@ public final class EnvelopeBodyAdvice implements ResponseBodyAdvice { if (selectedContentType != null && !MediaType.APPLICATION_JSON.includes(selectedContentType)) { return body; } + if (isErrorStatus(response)) { + return body; + } Envelope envelope = Envelope.ok(body, ResponseMetaFactory.fromMdc()); if (StringHttpMessageConverter.class.isAssignableFrom(selectedConverterType)) { return objectMapper.writeValueAsString(envelope); } return envelope; } + + private static boolean isErrorStatus(ServerHttpResponse response) { + return response instanceof ServletServerHttpResponse servlet + && servlet.getServletResponse().getStatus() >= 400; + } } diff --git a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/envelope/EnvelopeWireFormatConfig.java b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/envelope/EnvelopeWireFormatConfig.java index 4ed63d8f..f75421a4 100644 --- a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/envelope/EnvelopeWireFormatConfig.java +++ b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/envelope/EnvelopeWireFormatConfig.java @@ -29,7 +29,7 @@ public class EnvelopeWireFormatConfig { } /** The module itself, for a writer that has no Boot mapper to take it from. */ - static SimpleModule module() { + public static SimpleModule module() { SimpleModule module = new SimpleModule("EnvelopeWireFormatModule"); module.setMixInAnnotation(Envelope.class, OmitNullFields.class); module.setMixInAnnotation(BulkEnvelope.class, OmitNullFields.class); diff --git a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/EnvelopeErrorController.java b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/EnvelopeErrorController.java new file mode 100644 index 00000000..2b60ccd8 --- /dev/null +++ b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/EnvelopeErrorController.java @@ -0,0 +1,72 @@ +package dev.caskeleton.adapter.inbound.web.error; + +import dev.caskeleton.adapter.inbound.web.core.OwnsAbsoluteWebRoute; +import dev.caskeleton.adapter.inbound.web.http.ApiHeaders; +import dev.caskeleton.adapter.inbound.web.observability.RetryAfterAdvisor; +import dev.caskeleton.shared.error.ApiErrorCode; +import dev.caskeleton.shared.error.OperationalError; +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.Objects; +import org.springframework.boot.webmvc.error.ErrorController; +import org.springframework.http.HttpStatusCode; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import tools.jackson.databind.ObjectMapper; + +/** + * Answers the container's error dispatch with the failure {@code Envelope}. + * + *

    A failure raised outside the MVC handler — the security firewall's {@code + * RequestRejectedException}, a filter's {@code sendError}, an exception thrown by a filter — never + * reaches {@link GlobalExceptionHandler}. The container forwards it to {@code /error} instead, + * where Boot's {@code BasicErrorController} wrote {@code {timestamp, status, error, path}}; the + * success advice then wrapped that map as {@code success:true}, and {@code path} echoed the raw + * request URI back (re-review 2026-11-03, RR-02-02). This replaces that controller: the error + * status is classified by the same table the exception handler uses, and the body is the failure + * envelope, with nothing from the request in it. + * + *

    Written straight to the response by {@link EnvelopeResponseWriter}, like the other refusals + * made outside a converter. An error dispatch has no handler of its own to negotiate for, and a + * browser's {@code Accept: text/html} would otherwise turn the error into a 406 about the error. + * + *

    A request that names {@code /error} itself carries no error status; it is answered as an + * unknown route. The security chain permits only the error dispatch, so such a request still needs + * a token under the JWT chain. + * + *

    An absolute route ({@link OwnsAbsoluteWebRoute}): the container forwards to {@code /error}, + * not to {@code /error} under the presentation prefix. + */ +@Controller +@OwnsAbsoluteWebRoute +@RequestMapping("${server.error.path:${error.path:/error}}") +public final class EnvelopeErrorController implements ErrorController { + + private final ObjectMapper objectMapper; + + /** + * Creates the controller. + * + * @param objectMapper Boot's mapper, which carries the envelope wire rules + */ + public EnvelopeErrorController(ObjectMapper objectMapper) { + this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper"); + } + + /** Writes the failure envelope for the status the error dispatch carries. */ + @RequestMapping + public void error(HttpServletRequest request, HttpServletResponse response) throws IOException { + if (response.isCommitted()) { + return; + } + ApiErrorCode code = + request.getAttribute(RequestDispatcher.ERROR_STATUS_CODE) instanceof Integer status + ? HttpStatusErrorCodes.forStatus(HttpStatusCode.valueOf(status)) + : OperationalError.ROUTE_NOT_FOUND; + RetryAfterAdvisor.retryAfterSeconds(code) + .ifPresent(seconds -> response.setHeader(ApiHeaders.RETRY_AFTER, String.valueOf(seconds))); + EnvelopeResponseWriter.write(response, code, objectMapper); + } +} diff --git a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/GlobalExceptionHandler.java b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/GlobalExceptionHandler.java index 40a19f4e..f1e29196 100644 --- a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/GlobalExceptionHandler.java +++ b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/GlobalExceptionHandler.java @@ -407,7 +407,7 @@ public class GlobalExceptionHandler extends ResponseEntityExceptionHandler { return null; } } - ApiErrorCode code = errorCodeFor(statusCode); + ApiErrorCode code = HttpStatusErrorCodes.forStatus(statusCode); if (code.httpStatus() >= 500) { log.error("spring mvc failure classified as {}", code.code(), ex); spanErrorRecorder.recordException(ex, code.code()); @@ -415,25 +415,4 @@ public class GlobalExceptionHandler extends ResponseEntityExceptionHandler { return new ResponseEntity<>( ErrorResponseFactory.body(code, null), headers, HttpStatusCode.valueOf(code.httpStatus())); } - - private static ApiErrorCode errorCodeFor(HttpStatusCode status) { - return switch (status.value()) { - case 401 -> OperationalError.UNAUTHENTICATED; - case 403 -> OperationalError.FORBIDDEN; - case 404 -> OperationalError.ROUTE_NOT_FOUND; - case 405 -> OperationalError.METHOD_NOT_ALLOWED; - case 406 -> OperationalError.NOT_ACCEPTABLE; - case 412 -> OperationalError.PRECONDITION_FAILED; - case 413 -> OperationalError.PAYLOAD_TOO_LARGE; - case 414 -> OperationalError.URI_TOO_LONG; - case 415 -> OperationalError.UNSUPPORTED_MEDIA_TYPE; - case 429 -> OperationalError.RATE_LIMIT_EXCEEDED; - // Spring answers its async request timeout with 503: transient, not an internal fault. - case 503 -> WebTransportError.REQUEST_TIMEOUT; - default -> - status.is4xxClientError() - ? OperationalError.BAD_PARAMETER - : OperationalError.INTERNAL_ERROR; - }; - } } diff --git a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/HttpStatusErrorCodes.java b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/HttpStatusErrorCodes.java new file mode 100644 index 00000000..e37d6948 --- /dev/null +++ b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/HttpStatusErrorCodes.java @@ -0,0 +1,38 @@ +package dev.caskeleton.adapter.inbound.web.error; + +import dev.caskeleton.shared.error.ApiErrorCode; +import dev.caskeleton.shared.error.OperationalError; +import org.springframework.http.HttpStatusCode; + +/** + * The operational code a bare HTTP status stands for, when a status is all that is known. + * + *

    Two places only have a status to classify: a Spring MVC exception this adapter has no + * dedicated handler for, and the container's error dispatch for a failure raised outside the + * handler. They share this one table, so the same status never reaches a client as two codes. + */ +final class HttpStatusErrorCodes { + + private HttpStatusErrorCodes() {} + + static ApiErrorCode forStatus(HttpStatusCode status) { + return switch (status.value()) { + case 401 -> OperationalError.UNAUTHENTICATED; + case 403 -> OperationalError.FORBIDDEN; + case 404 -> OperationalError.ROUTE_NOT_FOUND; + case 405 -> OperationalError.METHOD_NOT_ALLOWED; + case 406 -> OperationalError.NOT_ACCEPTABLE; + case 412 -> OperationalError.PRECONDITION_FAILED; + case 413 -> OperationalError.PAYLOAD_TOO_LARGE; + case 414 -> OperationalError.URI_TOO_LONG; + case 415 -> OperationalError.UNSUPPORTED_MEDIA_TYPE; + case 429 -> OperationalError.RATE_LIMIT_EXCEEDED; + // Spring answers its async request timeout with 503: transient, not an internal fault. + case 503 -> WebTransportError.REQUEST_TIMEOUT; + default -> + status.is4xxClientError() + ? OperationalError.BAD_PARAMETER + : OperationalError.INTERNAL_ERROR; + }; + } +} diff --git a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/mvc/autoconfigure/WebMvcPlatformAutoConfiguration.java b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/mvc/autoconfigure/WebMvcPlatformAutoConfiguration.java index 5d8328ad..3f7989be 100644 --- a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/mvc/autoconfigure/WebMvcPlatformAutoConfiguration.java +++ b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/mvc/autoconfigure/WebMvcPlatformAutoConfiguration.java @@ -1,5 +1,7 @@ package dev.caskeleton.adapter.inbound.web.mvc.autoconfigure; +import dev.caskeleton.adapter.inbound.web.envelope.EnvelopeWireFormatConfig; +import dev.caskeleton.adapter.inbound.web.error.EnvelopeErrorController; import dev.caskeleton.adapter.inbound.web.error.GlobalExceptionHandler; import dev.caskeleton.adapter.inbound.web.mvc.filter.JsonBodyLimitFilter; import dev.caskeleton.adapter.inbound.web.mvc.filter.WebMvcRequestIdFilter; @@ -12,8 +14,12 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.jackson.autoconfigure.JsonMapperBuilderCustomizer; +import org.springframework.boot.webmvc.autoconfigure.error.ErrorMvcAutoConfiguration; +import org.springframework.boot.webmvc.error.ErrorController; import org.springframework.context.annotation.Bean; import tools.jackson.core.StreamReadFeature; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; /** * Wires the web platform into a servlet application. @@ -30,8 +36,12 @@ import tools.jackson.core.StreamReadFeature; * *

    Every bean is {@link ConditionalOnMissingBean}: the platform supplies the one that would * otherwise be missing rather than the one that must be used. + * + *

    Before {@link ErrorMvcAutoConfiguration}, so the envelope error controller is the {@link + * ErrorController} that configuration finds and Boot's {@code BasicErrorController} backs off. Its + * error-page registration with the container stays: that is what forwards to {@code /error}. */ -@AutoConfiguration +@AutoConfiguration(before = ErrorMvcAutoConfiguration.class) @ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.SERVLET) @ConditionalOnProperty( prefix = "app.web.mvc", @@ -54,6 +64,22 @@ public class WebMvcPlatformAutoConfiguration { return new GlobalExceptionHandler(spanErrorRecorderProvider); } + /** + * Answers the container's error dispatch — failures raised outside the MVC handler — with the + * failure envelope instead of Boot's {@code {timestamp, status, error, path}} map. + * + *

    Boot's mapper when there is one, which already carries the envelope wire rules; otherwise a + * mapper with only those rules, as {@code EnvelopeBodyAdvice} does without Boot. + */ + @Bean + @ConditionalOnMissingBean(ErrorController.class) + public EnvelopeErrorController envelopeErrorController( + ObjectProvider objectMapper) { + return new EnvelopeErrorController( + objectMapper.getIfAvailable( + () -> JsonMapper.builder().addModule(EnvelopeWireFormatConfig.module()).build())); + } + /** * Assigns the correlation identifiers, validating anything a caller proposed. With a tracer in * the composition the trace id is the tracer's: the filter runs inside Boot's server observation diff --git a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/observability/RetryAfterAdvisor.java b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/observability/RetryAfterAdvisor.java index e46ec3d6..8fcae639 100644 --- a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/observability/RetryAfterAdvisor.java +++ b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/observability/RetryAfterAdvisor.java @@ -1,5 +1,6 @@ package dev.caskeleton.adapter.inbound.web.observability; +import dev.caskeleton.application.idempotency.IdempotencyExecutor; import dev.caskeleton.shared.error.ApiErrorCode; import java.util.Map; import java.util.OptionalInt; @@ -13,17 +14,26 @@ public final class RetryAfterAdvisor { /** Per-code {@code Retry-After} values (seconds). */ private static final Map RETRY_AFTER_SECONDS = Map.of( - "RATE_LIMIT_EXCEEDED", 1, - // Below the default processing lease (30s): the resend arrives while the first attempt - // can still finish, and an early resend costs one more 409. docs/web/http-contract.md. - "IDEMPOTENT_IN_FLIGHT", 2, - "AUTH_KID_UNKNOWN", 5, - "AUTH_JWKS_UNAVAILABLE", 30, - "DEPENDENCY_TIMEOUT", 2, - "DEPENDENCY_CONNECT_FAILED", 2, - "DEPENDENCY_DNS_FAILED", 5, - "DEPENDENCY_5XX_SERVER", 2, - "DEPENDENCY_CIRCUIT_OPEN", 10); + "RATE_LIMIT_EXCEEDED", + 1, + // The application's value, which the processing-lease check also reads: the resend + // arrives while the first attempt can still finish. docs/web/http-contract.md §2. + "IDEMPOTENT_IN_FLIGHT", + Math.toIntExact(IdempotencyExecutor.IN_FLIGHT_RETRY_AFTER.toSeconds()), + "AUTH_KID_UNKNOWN", + 5, + "AUTH_JWKS_UNAVAILABLE", + 30, + "DEPENDENCY_TIMEOUT", + 2, + "DEPENDENCY_CONNECT_FAILED", + 2, + "DEPENDENCY_DNS_FAILED", + 5, + "DEPENDENCY_5XX_SERVER", + 2, + "DEPENDENCY_CIRCUIT_OPEN", + 10); private RetryAfterAdvisor() {} diff --git a/src/adapter/inbound/web/src/systemTest/java/dev/caskeleton/adapter/inbound/web/testkit/mvc/JwtChainErrorDispatchIT.java b/src/adapter/inbound/web/src/systemTest/java/dev/caskeleton/adapter/inbound/web/testkit/mvc/JwtChainErrorDispatchIT.java new file mode 100644 index 00000000..3188c65b --- /dev/null +++ b/src/adapter/inbound/web/src/systemTest/java/dev/caskeleton/adapter/inbound/web/testkit/mvc/JwtChainErrorDispatchIT.java @@ -0,0 +1,91 @@ +package dev.caskeleton.adapter.inbound.web.testkit.mvc; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.adapter.inbound.web.envelope.EnvelopeWireFormatConfig; +import dev.caskeleton.adapter.inbound.web.mvc.autoconfigure.WebAdapterAutoConfiguration; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** + * The error dispatch under the production JWT security chain, over a real Tomcat. + * + *

    The chain authorises every dispatch, the container's error dispatch to {@code /error} + * included. {@code /error} is not a public path, so an anonymous request whose original failure was + * a 400 used to come back as 401 — the client was told to authenticate when its request was + * malformed (re-review 2026-11-03, RR-02-02). The error dispatch is now permitted; a request that + * names {@code /error} directly is still a request and still needs a token. + * + *

    The chain is {@code WebSecurityAutoConfiguration} itself. The JWT decoder is lazy, so the + * unreachable issuer below is never contacted: no request here carries a token. + */ +@SpringBootTest( + classes = JwtChainErrorDispatchIT.JwtChainApplication.class, + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = { + "spring.main.banner-mode=off", + "spring.mvc.problemdetails.enabled=false", + "app.web.mvc.enabled=true", + "app.security.auth-mode=jwt", + "app.security.issuer-uri=https://issuer.invalid/realms/ca", + "app.security.audience=ca-skeleton-api", + "app.security.public-paths=/api/v1/fixtures/**", + "app.cors.enabled=false" + }) +class JwtChainErrorDispatchIT { + + private final HttpClient client = + HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); + + @LocalServerPort private int port; + + @Test + @DisplayName("JWT 체인에서 익명 요청이 핸들러 밖에서 400 으로 거부되면 401 이 아니라 400 실패 Envelope 로 나간다") + void anAnonymousRequestRejectedOutsideTheHandlerKeepsItsOwnStatus() throws Exception { + // when + HttpResponse rejected = get(WebContractAssertions.FIREWALL_REJECTED_PATH); + + // then + assertThat(rejected.statusCode()).isEqualTo(400); + WebContractAssertions.assertFailureEnvelope(rejected, "BAD_PARAMETER"); + } + + @Test + @DisplayName("오류 디스패치만 허용한다 — /error 를 직접 부른 익명 요청은 여전히 401 이다") + void aDirectRequestForTheErrorPathStillNeedsAToken() throws Exception { + // when + HttpResponse direct = get("/error"); + + // then + assertThat(direct.statusCode()).isEqualTo(401); + } + + private HttpResponse get(String path) throws Exception { + return client.send( + HttpRequest.newBuilder(URI.create("http://localhost:" + port + path)) + .header("Accept", "application/json") + .GET() + .build(), + HttpResponse.BodyHandlers.ofString()); + } + + /** + * The production web stack minus {@link WebAdapterAutoConfiguration}, whose request-logging + * filter needs the composition root's pseudonymizer, plus that root's envelope wire format; the + * security chain is the shipped one. + */ + @Configuration(proxyBeanMethods = false) + @EnableAutoConfiguration(exclude = WebAdapterAutoConfiguration.class) + @Import({ContractFixtureController.class, EnvelopeWireFormatConfig.class}) + static class JwtChainApplication {} +} diff --git a/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/envelope/EnvelopeBodyAdviceTest.java b/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/envelope/EnvelopeBodyAdviceTest.java index 68a457f6..49028fc1 100644 --- a/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/envelope/EnvelopeBodyAdviceTest.java +++ b/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/envelope/EnvelopeBodyAdviceTest.java @@ -23,6 +23,7 @@ import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; import org.springframework.boot.webmvc.test.autoconfigure.WebMvcTest; import org.springframework.context.annotation.Import; import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.test.json.JsonCompareMode; import org.springframework.test.web.servlet.MockMvc; import org.springframework.web.bind.annotation.GetMapping; @@ -64,6 +65,12 @@ class EnvelopeBodyAdviceTest { return "{\"openapi\":\"3.1.0\"}".getBytes(StandardCharsets.UTF_8); } + /** A failure status with a body that is not an envelope. */ + @GetMapping("/probe/conflict") + public ResponseEntity> conflict() { + return ResponseEntity.status(409).body(Map.of("k", "v")); + } + @GetMapping("/probe/bulk") public BulkEnvelope> bulk() { return BulkEnvelope.allOk(List.of(Map.of("k", "v")), new ResponseMeta("r", "t", "c")); @@ -149,6 +156,25 @@ class EnvelopeBodyAdviceTest { .andExpect(content().json("{\"openapi\":\"3.1.0\"}", JsonCompareMode.STRICT)); } + /** + * A 4xx/5xx body is never a success envelope. {@code success} is the discriminator clients branch + * on (docs/web/http-contract.md §1), so wrapping an error-status body as {@code success:true} + * tells a contract-following client the request worked — Boot's {@code /error} map arrived that + * way (re-review 2026-11-03, RR-02-02). The body is left as the handler wrote it. + */ + @Test + @DisplayName("4xx·5xx 응답 본문은 success:true Envelope 로 감싸지 않는다") + void anErrorStatusBodyIsNotWrappedAsASuccess() throws Exception { + // when + var result = mvc.perform(get("/probe/conflict").accept(MediaType.APPLICATION_JSON)); + + // then + result + .andExpect(status().isConflict()) + .andExpect(jsonPath("$.success").doesNotExist()) + .andExpect(content().json("{\"k\":\"v\"}", JsonCompareMode.STRICT)); + } + @SpringBootConfiguration @EnableAutoConfiguration(exclude = SecurityAutoConfiguration.class) static class TestBootstrap {} diff --git a/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/error/IdempotencyExceptionMappingTest.java b/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/error/IdempotencyExceptionMappingTest.java index 2e1a8412..91ee5f16 100644 --- a/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/error/IdempotencyExceptionMappingTest.java +++ b/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/error/IdempotencyExceptionMappingTest.java @@ -3,6 +3,7 @@ package dev.caskeleton.adapter.inbound.web.error; import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.adapter.inbound.web.http.ApiHeaders; +import dev.caskeleton.application.idempotency.IdempotencyCompletionUnknownException; import dev.caskeleton.application.idempotency.IdempotencyInFlightException; import dev.caskeleton.application.idempotency.IdempotencyRequestMismatchException; import dev.caskeleton.shared.error.Category; @@ -74,4 +75,24 @@ class IdempotencyExceptionMappingTest { false, ClientErrorMessages.of(Category.VALIDATION)); } + + @Test + @DisplayName("complete 단계 결과 불명은 실행 전 거부(IDEMPOTENCY_UNAVAILABLE)와 다른 code 의 retryable 503 이다") + void completionUnknownMapsToItsOwnRetryable503SoTheClientDoesNotReadItAsNotApplied() { + // given + IdempotencyCompletionUnknownException unknown = new IdempotencyCompletionUnknownException(); + + // when + ResponseEntity> response = handler.handleCarrier(unknown); + + // then + assertThat(response) + .extracting( + r -> r.getStatusCode().value(), + r -> r.getHeaders().getFirst(ApiHeaders.RETRY_AFTER), + r -> r.getBody().error().code(), + r -> r.getBody().error().category(), + r -> r.getBody().error().retryable()) + .containsExactly(503, null, "IDEMPOTENCY_COMPLETION_UNKNOWN", "TRANSIENT_DEPENDENCY", true); + } } diff --git a/src/adapter/inbound/web/src/testFixtures/java/dev/caskeleton/adapter/inbound/web/testkit/mvc/WebContractAssertions.java b/src/adapter/inbound/web/src/testFixtures/java/dev/caskeleton/adapter/inbound/web/testkit/mvc/WebContractAssertions.java index 84f47ef5..13cdb94e 100644 --- a/src/adapter/inbound/web/src/testFixtures/java/dev/caskeleton/adapter/inbound/web/testkit/mvc/WebContractAssertions.java +++ b/src/adapter/inbound/web/src/testFixtures/java/dev/caskeleton/adapter/inbound/web/testkit/mvc/WebContractAssertions.java @@ -8,6 +8,8 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.time.Duration; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.json.JsonMapper; /** * The status contract as socket-level assertions, written once and run against every container. @@ -22,6 +24,11 @@ import java.time.Duration; */ public final class WebContractAssertions { + /** A path the servlet firewall refuses before any handler runs ({@code ;} in the URI). */ + public static final String FIREWALL_REJECTED_PATH = "/api/v1/fixtures/f1;x=1"; + + private static final JsonMapper JSON = JsonMapper.builder().build(); + private final HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); @@ -122,6 +129,46 @@ public final class WebContractAssertions { assertThat(post("/api/v1/fixtures", "{\"name\":").statusCode()).isEqualTo(400); } + /** + * An error raised outside the MVC handler is still a failure envelope. + * + *

    The firewall rejects the request before any controller runs, and the container answers it + * with an error dispatch to {@code /error}. Boot's default error controller wrote a map there, + * which the success advice then wrapped as {@code success:true} with the raw request path inside + * (re-review 2026-11-03, RR-02-02). MockMvc performs no error dispatch, so only a container can + * show this. + */ + public void aRequestRejectedOutsideTheHandlerIsAFailureEnvelope() + throws IOException, InterruptedException { + HttpResponse rejected = get(FIREWALL_REJECTED_PATH); + + assertThat(rejected.statusCode()).isEqualTo(400); + assertFailureEnvelope(rejected, "BAD_PARAMETER"); + } + + /** + * Asserts the failure envelope of {@code docs/web/http-contract.md} §1 on an error response. + * + * @param response a response the container wrote through its error dispatch + * @param expectedCode the {@code error.code} the status maps to + */ + public static void assertFailureEnvelope(HttpResponse response, String expectedCode) { + JsonNode body = JSON.readTree(response.body()); + assertThat(body.path("success").isBoolean() && !body.path("success").booleanValue()) + .as( + "a 4xx/5xx body that says success:true is read as a success by a client that follows" + + " the contract: %s", + response.body()) + .isTrue(); + assertThat(body.path("error").path("code").asString()).isEqualTo(expectedCode); + assertThat(body.has("data")).as("a failure envelope has no data key").isFalse(); + assertThat(response.body()) + .as("the error body must not echo the request path back") + .doesNotContain(";x=1") + .doesNotContain("\"path\"") + .doesNotContain("timestamp"); + } + /** Runs every assertion, so a lane cannot silently cover a subset. */ public void assertWholeContract() throws IOException, InterruptedException { createdCarriesALocationHeader(); @@ -131,6 +178,7 @@ public final class WebContractAssertions { anUnknownMethodOnAKnownRouteIsRefused(); anUnsupportedMediaTypeIsRefused(); malformedJsonIsRefused(); + aRequestRejectedOutsideTheHandlerIsAFailureEnvelope(); } private HttpResponse post(String path, String body) @@ -143,6 +191,12 @@ public final class WebContractAssertions { HttpResponse.BodyHandlers.ofString()); } + private HttpResponse get(String path) throws IOException, InterruptedException { + return client.send( + HttpRequest.newBuilder(uri(path)).header("Accept", "application/json").GET().build(), + HttpResponse.BodyHandlers.ofString()); + } + private URI uri(String path) { return URI.create("http://localhost:" + port + path); } diff --git a/src/adapter/inbound/web/src/testFixtures/java/dev/caskeleton/webtestkit/ContractFixtureApplication.java b/src/adapter/inbound/web/src/testFixtures/java/dev/caskeleton/webtestkit/ContractFixtureApplication.java index 3a650c0b..089d298d 100644 --- a/src/adapter/inbound/web/src/testFixtures/java/dev/caskeleton/webtestkit/ContractFixtureApplication.java +++ b/src/adapter/inbound/web/src/testFixtures/java/dev/caskeleton/webtestkit/ContractFixtureApplication.java @@ -1,5 +1,6 @@ package dev.caskeleton.webtestkit; +import dev.caskeleton.adapter.inbound.web.envelope.EnvelopeWireFormatConfig; import dev.caskeleton.adapter.inbound.web.mvc.autoconfigure.WebAdapterAutoConfiguration; import dev.caskeleton.adapter.inbound.web.mvc.autoconfigure.WebSecurityAutoConfiguration; import dev.caskeleton.adapter.inbound.web.testkit.fault.FaultFixtureController; @@ -31,6 +32,10 @@ import org.springframework.security.web.SecurityFilterChain; * are the "sum of the leaf" the paragraph above keeps out. Excluded by class, so a rename breaks * compilation rather than the gate. The third root, {@code WebMethodSecurityAutoConfiguration}, is * left in: no fixture route runs a permissioned use case, so it decides nothing here. + * + *

    {@link EnvelopeWireFormatConfig} is imported by name from the excluded web root: it is the + * wire shape of every envelope this gate reads (a failure body has no {@code data} key), not a + * capability, and without it the gate would certify {@code "data": null} that no deployment sends. */ // Outside `dev.caskeleton.adapter.inbound.web`, which the composition root component-scans. // A fixture application under that root is loaded into the real deployment: its permit-all @@ -40,7 +45,11 @@ import org.springframework.security.web.SecurityFilterChain; @Configuration(proxyBeanMethods = false) @EnableAutoConfiguration( exclude = {WebAdapterAutoConfiguration.class, WebSecurityAutoConfiguration.class}) -@Import({ContractFixtureController.class, FaultFixtureController.class}) +@Import({ + ContractFixtureController.class, + FaultFixtureController.class, + EnvelopeWireFormatConfig.class +}) public class ContractFixtureApplication { /** diff --git a/src/adapter/outbound/cache-redis/CLAUDE.md b/src/adapter/outbound/cache-redis/CLAUDE.md index 9060c016..d3927556 100644 --- a/src/adapter/outbound/cache-redis/CLAUDE.md +++ b/src/adapter/outbound/cache-redis/CLAUDE.md @@ -128,12 +128,15 @@ Focused tests drive the semantic adapters against `InMemorySemanticRedisCommands served through `StubRedisRuntimeClient`. R1/R2 promotion requires a separate real Redis service lane; it may never be silently skipped when selected. -Two real-server lanes. `redisContainerContractTest` (integration category, in `integrationCheck` and -this leaf's `mainCheck`) starts its own Redis with Testcontainers — the `redis` image of -`config/testing/container-images.properties`, loading `infra/redis-sdk/acl/all-accounts.acl` — and -runs `LiveRedisCacheRegionContractTest` and `LiveRedisDistributedLeaseContractTest` (the -`application-core` port contracts the in-memory fakes also pass), `LiveRedisSemanticPortsTest` and -`LiveRedisSemanticScriptAclProbeTest`; a skip fails it (review 2026-11-01 05-16). `RedisTestContainer` +Two real-server lanes. `redisContainerContractTest` (integration category, in `integrationCheck` only — +not in this leaf's `mainCheck` too, which ran it twice per push to main) starts its own Redis with Testcontainers — the `redis` image of +`config/testing/container-images.properties`, loading `infra/redis-sdk/acl/lane-fixture-accounts.acl` — and +runs `LiveRedisCacheRegionContractTest`, `LiveRedisDistributedLeaseContractTest` and +`LiveRedisIdempotencyStoreContractTest` (the `application-core` port contracts the in-memory fakes +also pass), `LiveRedisSemanticPortsTest` and `LiveRedisSemanticScriptAclProbeTest`; a skip fails it +(review 2026-11-01 05-16). The idempotency contract was on the explicit-only topology lane alone +until review 2026-11-03 RR-04-05, so the Lua state machine's stale-handle, renew-refusal and +retry cases reached a real server in no gate. `RedisTestContainer` starts the server only when that lane names the ACL file. `redisTopologyTest` is the external-topology lane, declared through `ca.redis-topology-lane`. This @@ -177,6 +180,15 @@ deployment still gets exactly one client and one event loop. The `SCRIPT` lane i the reason it exists: `SCRIPT LOAD` and `EVALSHA` belong to the advanced account, so the account that reads a cache entry cannot execute a script. +The cache's conditional writes run on the same account but on a lane of their own, +`CacheScriptLane.CACHE_SCRIPT` (`app.capabilities.cache.maximum-script-connections`, contributed by +`RedisCacheConfig` as a `RedisLaneContribution`). Do not move them back to `SCRIPT`: that lane is the +idempotency, rate-limit and lease budget, and a cache under load must not take it (review 2026-11-03 +README §4, RR-04-02). Each lane's wait, refusals and reconnects are meters of `redis-runtime` +(`RedisLaneMetrics`, `redis.lane.*`, tag `lane`), so whether the cache stays off `SCRIPT` is a query, +not an assumption. The `DEGRADED` health order ships with `redis-runtime`, which reports the +status (`config/module-defaults/redis-runtime.yml`, RR-04-10). + A Cluster transaction runs on one node, so the SDK's `RedisTransactionRunner` derives a routing key from the watched keys (or an explicit `RedisSlotTag`) and borrows `RedisSdkLane.TRANSACTION`, which is pinned to its key's owner, on the node that owns that slot. Routed leases are never pooled: a pooled connection is pinned to the previous caller's diff --git a/src/adapter/outbound/cache-redis/README.md b/src/adapter/outbound/cache-redis/README.md index 0be4ad8e..99026fa9 100644 --- a/src/adapter/outbound/cache-redis/README.md +++ b/src/adapter/outbound/cache-redis/README.md @@ -97,7 +97,10 @@ Connection lane limits and command queue limits are different contracts. In part - `app.redis.capacity.maximum-in-flight-commands` bounds the per-connection command queue. - `app.redis.capacity.maximum-script-connections` bounds physical SCRIPT lane connections - (default 8). + (default 8). Idempotency, rate limiting and leases share them. +- `app.capabilities.cache.maximum-script-connections` (`APP_CACHE_REDIS_MAXIMUM_SCRIPT_CONNECTIONS`) + bounds the cache's own script lane (`CacheScriptLane`, default 4), contributed only when the cache + binding is `redis`. See "Why the cache has its own script lane" below. - `app.redis.lifecycle.acquire-timeout` (default 2s) is how long a borrow waits for a returned connection when its lane is full. @@ -153,7 +156,7 @@ Important contracts: Every lookup returns a `CacheWriteCondition` naming the state it saw — `absent`, `vacant:` (an invalidation marker, an entry of an older region generation, an expired entry) or `present:`. A record that carries the condition applies only while the key is still in that -state (registered program, SCRIPT lane). This is what rejects a source load that began before an +state (registered program, cache script lane). This is what rejects a source load that began before an invalidation and finished after it. A hit also returns an opaque SHA-1 observation token over the stored bytes. SHA-1 here is **not a @@ -181,10 +184,43 @@ example one written by `UPSERT` as `rev-1`), another schema version, a generatio writer knows — is `NOT_RECORDED_CONDITION`. Both conditional intents, and any record carrying a write condition, run registered scripts, so -they borrow the SCRIPT lane (the advanced account when one is configured, the application account -otherwise). The application account is not granted `EVALSHA`; on the REGULAR lane every conditional -write came back `INDETERMINATE`. Unconditioned `UPSERT` and `ONLY_IF_ABSENT` stay on the REGULAR -lane. +they run on the scripting account (the advanced account when one is configured, the application +account otherwise), on the cache's own script lane. The application account is not granted +`EVALSHA`; on the REGULAR lane every conditional write came back `INDETERMINATE`. Unconditioned +`UPSERT` and `ONLY_IF_ABSENT` stay on the REGULAR lane. + +### Why the cache has its own script lane + +Recording with the condition a lookup returned is the normal way to use this port, so nearly every +cache fill is a script. On the runtime's SCRIPT lane those fills shared eight connections per pod +with idempotency claims, rate-limit evaluations and lease calls, and a timed-out lease is discarded +rather than pooled, so each timeout also cost a reconnect. A slow Redis or a burst of misses after a +region invalidation filled the lane with cache work; an idempotency claim then waited out its +timeout and answered `UNAVAILABLE`, and a rate-limit call fell to its failure policy. The cache is a +speed feature that may degrade; idempotency, rate limiting and leases are correctness features that +must not (review 2026-11-03 README §4, RR-04-02). + +So `CacheScriptLane.CACHE_SCRIPT` is a lane of its own — the same scripting account, its own +connections, its own ceiling (`app.capabilities.cache.maximum-script-connections`, default 4). A cache +that runs out of them degrades (`DEGRADED_UNAVAILABLE`, nothing sent) without touching the SCRIPT +lane. Alternatives not taken: a per-capability permit cap inside the shared SCRIPT lane (still one +pool of connections and one reconnect cost), and running the cache programs on the REGULAR lane +(grants the application account `EVALSHA`, which the account split exists to withhold). + +Whether the split holds under load is visible per lane. With a `MeterRegistry` in the context, +`redis-runtime` registers these for every lane it pools — `REGULAR`, `SCRIPT`, `CACHE_SCRIPT` and the +typed SDK's — tagged `lane=` (`RedisLaneMetrics`; rows in `docs/registries/metrics.yaml`): + +| Meter | Type | Read it as | +| --- | --- | --- | +| `redis.lane.acquire.wait.seconds` (`outcome` = `admitted`/`refused`) | timer, histogram | how long a borrow queued for a connection; a growing admitted p99 comes before refusals | +| `redis.lane.acquire.refused.total` (`reason` = `saturated`/`closing`/`interrupted`/`connect_failed`) | counter | a borrow that got no connection, nothing sent; `saturated` on `SCRIPT` is the starvation this lane exists to prevent | +| `redis.lane.connections.opened.total` | counter | new connections; after warm-up every one is a reconnect | +| `redis.lane.connections.discarded.total` (`reason` = `invalidated`/`broken`) | counter | connections closed instead of pooled — `invalidated` is a timed-out or failed call (the lease is not reused), and is what the reconnects replace | +| `redis.lane.connections.active`, `redis.lane.connections.pending` | gauge | connections lent out now; borrows waiting now | + +Every counter series exists from startup at zero, so an alert on "refusals above zero" sees the first +one. A lane refusal is also logged, per call, by `RedisSemanticFailureLogger`. ### Key invalidation @@ -300,10 +336,21 @@ Useful focused checks: `redisContainerContractTest` runs the cache-region and lease port contracts (the ones the in-memory fakes pass) and the live semantic-port tests against a Redis it starts itself, under the standalone -topology's ACL file. It is in `integrationCheck` and `mainCheck`; the image is +topology's ACL file. It is in `integrationCheck` (integration-main.yml's `integration-check` job), and +only there: listing it in `mainCheck` as well ran it twice on every push to main. The image is `config/testing/container-images.properties` `redis`. Live Redis/topology qualification remains separate from fast unit tests. A release claim for a specific Redis topology or ACL layout requires the corresponding live qualification lane; the in-memory gateway is for deterministic contract tests, not evidence that an external Redis deployment accepts the same network/TLS/ACL configuration. + +## 아직 내보내지 않는 지표 + +`redis.capability.*` 여섯 개(operations.total, duration.seconds, admission.rejected.total, +inflight.total, readiness.total, lifecycle.drain.total)는 redis-production-capability 설계(Task 16)에 +있었지만 어떤 코드도 내보내지 않았다. 레지스트리(`docs/registries/metrics.yaml`)와 incident runbook이 있는 +것처럼 적고 있어서, 2026-11-03 재리뷰 수정에서 레지스트리에서 빼고 runbook을 지금 내보내는 +`redis.lane.*`(redis-runtime, lane별 대기·거절·연결) 기준으로 고쳤다. capability 단위 지표가 필요해지면 +operation 결과를 아는 semantic adapter(이 leaf)에서 내보내고, 그때 레지스트리에 다시 넣는다. + diff --git a/src/adapter/outbound/cache-redis/build.gradle b/src/adapter/outbound/cache-redis/build.gradle index fd9ac4d6..1719cf67 100644 --- a/src/adapter/outbound/cache-redis/build.gradle +++ b/src/adapter/outbound/cache-redis/build.gradle @@ -48,7 +48,8 @@ dependencies { // command surface the catalog exists to prevent. Zero imports. // io.micrometer:micrometer-core — observation leaves this leaf as RedisObservation through a // Consumer sink; binding it to a meter registry belongs to the composition root, not here. - // Zero imports. + // Zero imports. The connection lanes' meters, CACHE_SCRIPT's included, are redis-runtime's + // (RedisLaneMetrics): the lanes are pooled there, so that is where they are counted. } // Redis topology qualification is opt-in and owned by `ca.redis-topology-lane`: mode/property @@ -69,8 +70,12 @@ redisTopologyLane { // adapters' port contracts reached a real server in no stage, and the fake and the adapter could hide // the same defect (05-02, 05-03). The image is config/testing/container-images.properties' `redis`, // and the ACL file is infra/redis-sdk's own, so the accounts and their rights are the standalone -// topology's. Integration category: integrationCheck runs it, and so does this leaf's mainCheck, like -// the HTTP client's Toxiproxy lane. Sentinel, cluster and TLS remain the topology lane's. +// topology's. Integration category, so integrationCheck runs it — integration-main.yml's +// `integration-check` job, on every push to main and nightly. It is deliberately not in this leaf's +// mainCheck as well: `main-check` and `integration-check` are two jobs of the same workflow run, so a +// lane in both started a second Redis container and ran every live contract twice per push without +// covering anything the first run did not (found while fixing review 2026-11-03). Sentinel, cluster +// and TLS remain the topology lane's. def redisAclDirectory = rootProject.projectDir.parentFile.toPath().resolve('infra/redis-sdk/acl').toFile() strictTestLanes { lane('redisContainerContractTest') { @@ -78,15 +83,16 @@ strictTestLanes { description = 'Runs the Redis semantic adapters\' live contracts against a Testcontainers Redis on the pinned image.' requires 'dev.caskeleton.adapter.outbound.cache.redis.cache.LiveRedisCacheRegionContractTest', 'dev.caskeleton.adapter.outbound.cache.redis.lease.LiveRedisDistributedLeaseContractTest', + // The fake carries a Java copy of the idempotency Lua state machine; only a real + // server proves the two agree (05-02 was the fake copying the Lua's bug). It ran on + // the explicit-only topology lane alone, so no gate ever ran it (RR-04-05). + 'dev.caskeleton.adapter.outbound.cache.redis.idempotency.LiveRedisIdempotencyStoreContractTest', 'dev.caskeleton.adapter.outbound.cache.redis.LiveRedisSemanticPortsTest', 'dev.caskeleton.adapter.outbound.cache.redis.capability.LiveRedisSemanticScriptAclProbeTest' rejectSkipped = true - systemProperty 'redis.container.acl-file', new File(redisAclDirectory, 'all-accounts.acl').absolutePath + systemProperty 'redis.container.acl-file', new File(redisAclDirectory, 'lane-fixture-accounts.acl').absolutePath inputDirectory redisAclDirectory shouldRunAfter 'test' } } -tasks.named('mainCheck') { - dependsOn 'redisContainerContractTest' -} diff --git a/src/adapter/outbound/cache-redis/gradle.lockfile b/src/adapter/outbound/cache-redis/gradle.lockfile index 05009344..2c16a9d3 100644 --- a/src/adapter/outbound/cache-redis/gradle.lockfile +++ b/src/adapter/outbound/cache-redis/gradle.lockfile @@ -50,7 +50,7 @@ io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnota io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor io.lettuce:lettuce-core:6.8.2.RELEASE=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath io.micrometer:micrometer-commons:1.16.7=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -io.micrometer:micrometer-core:1.16.7=testCompileClasspath,testRuntimeClasspath +io.micrometer:micrometer-core:1.16.7=runtimeClasspath,testCompileClasspath,testRuntimeClasspath io.micrometer:micrometer-jakarta9:1.16.7=testCompileClasspath,testRuntimeClasspath io.micrometer:micrometer-observation:1.16.7=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath io.netty:netty-buffer:4.2.17.Final=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath @@ -103,7 +103,7 @@ org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle org.codehaus.plexus:plexus-utils:3.3.0=checkstyle org.dom4j:dom4j:2.2.0=spotbugs org.hamcrest:hamcrest:3.0=testCompileClasspath,testRuntimeClasspath -org.hdrhistogram:HdrHistogram:2.2.2=testRuntimeClasspath +org.hdrhistogram:HdrHistogram:2.2.2=runtimeClasspath,testRuntimeClasspath org.javassist:javassist:3.28.0-GA=checkstyle org.jetbrains:annotations:17.0.0=testCompileClasspath,testRuntimeClasspath org.jspecify:jspecify:1.0.1=annotationProcessor,checkstyle,compileClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath @@ -116,7 +116,7 @@ org.junit.platform:junit-platform-engine:6.0.3=testRuntimeClasspath org.junit.platform:junit-platform-launcher:6.0.3=testRuntimeClasspath org.junit:junit-bom:6.0.3=testCompileClasspath,testRuntimeClasspath org.junit:junit-bom:6.1.0=spotbugs -org.latencyutils:LatencyUtils:2.0.3=testRuntimeClasspath +org.latencyutils:LatencyUtils:2.0.3=runtimeClasspath,testRuntimeClasspath org.mockito:mockito-core:5.20.0=mockitoAgent,testCompileClasspath,testRuntimeClasspath org.mockito:mockito-junit-jupiter:5.20.0=testCompileClasspath,testRuntimeClasspath org.objenesis:objenesis:3.3=testRuntimeClasspath diff --git a/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/cache/CacheScriptLane.java b/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/cache/CacheScriptLane.java new file mode 100644 index 00000000..152d45ee --- /dev/null +++ b/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/cache/CacheScriptLane.java @@ -0,0 +1,34 @@ +package dev.caskeleton.adapter.outbound.cache.redis.cache; + +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisAccount; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLane; + +/** + * The cache's own script lane, so a cache's load cannot starve a correctness capability (review + * 2026-11-03 README §4, RR-04-02). + * + *

    Every conditional cache write is a registered program, and a registered program needs the + * scripting account. On the runtime's SCRIPT lane those writes shared eight connections with + * idempotency claims, rate-limit evaluations and lease calls — and the normal way to use the cache + * (record with the condition the lookup returned) makes every cache fill a script. A slow Redis or + * a burst of misses after a region invalidation then held those connections with cache work while + * an idempotency claim waited out its timeout and failed. The cache is a speed feature and may + * degrade; idempotency and leases are correctness features and must not. They get separate + * connection budgets. + * + *

    Same account as the SCRIPT lane: the grant is the same, only the connections and their ceiling + * ({@code app.capabilities.cache.maximum-script-connections}) are the cache's. Alternatives that + * were not taken: a per-capability permit cap inside the shared SCRIPT lane would still share the + * connections and their reconnect cost after a discarded lease; running cache programs on the + * REGULAR lane would grant the application account {@code EVALSHA}, which the account split exists + * to withhold. + */ +public enum CacheScriptLane implements RedisLane { + /** The cache region's conditional and revision-ordered writes. */ + CACHE_SCRIPT; + + @Override + public RedisAccount account() { + return RedisAccount.ADVANCED; + } +} diff --git a/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/cache/RedisCacheRegionAdapter.java b/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/cache/RedisCacheRegionAdapter.java index 95b3a6e3..7cd93fbf 100644 --- a/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/cache/RedisCacheRegionAdapter.java +++ b/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/cache/RedisCacheRegionAdapter.java @@ -68,8 +68,10 @@ public final class RedisCacheRegionAdapter implements CacheRegionPort implements CacheRegionPortThe runtime pools a contributed lane like its own SCRIPT lane, on the same scripting + * account, but with connections and a ceiling of its own: the cache's conditional writes no + * longer compete with idempotency claims, rate-limit evaluations and lease calls for the SCRIPT + * lane (review 2026-11-03 README §4, RR-04-02). + * + * @param capabilities the capability settings + * @return the lane contribution + */ + @Bean + @ConditionalOnProperty(name = CapabilitySelectors.CACHE_DEFAULT_BINDING, havingValue = "redis") + RedisLaneContribution redisCacheScriptLane(RedisCapabilitySettings capabilities) { + RedisCapabilitySettings.Cache cache = capabilities.getCache(); + cache.validate(); + return new RedisLaneContribution( + CacheScriptLane.CACHE_SCRIPT, cache.getMaximumScriptConnections()); + } } diff --git a/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisCapabilitySettings.java b/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisCapabilitySettings.java index c10d57be..5303e105 100644 --- a/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisCapabilitySettings.java +++ b/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisCapabilitySettings.java @@ -86,6 +86,7 @@ public class RedisCapabilitySettings { private Duration positiveHardTtl = Duration.ofMinutes(5); private Duration negativeTtl = Duration.ofSeconds(10); private Duration minimumHardTtl = Duration.ofSeconds(1); + private int maximumScriptConnections = 4; /** * Reports whether the default binding selected Redis. @@ -119,6 +120,10 @@ public class RedisCapabilitySettings { if (keyVersion < 1) { throw new CapabilityStartupValidationException("the cache key version must be positive"); } + if (maximumScriptConnections < 1) { + throw new CapabilityStartupValidationException( + "the cache script lane needs at least one connection"); + } } public Map getBindings() { @@ -192,6 +197,23 @@ public class RedisCapabilitySettings { public void setMinimumHardTtl(Duration minimumHardTtl) { this.minimumHardTtl = minimumHardTtl; } + + /** + * The ceiling on the cache's own script-lane connections. + * + *

    Separate from {@code app.redis.capacity.maximum-script-connections}, which idempotency, + * rate limiting and leases share: a cache is a speed feature and may run out of connections and + * degrade; it must not take them from the correctness features (review 2026-11-03 README §4). + * + * @return the ceiling, at least 1 + */ + public int getMaximumScriptConnections() { + return maximumScriptConnections; + } + + public void setMaximumScriptConnections(int maximumScriptConnections) { + this.maximumScriptConnections = maximumScriptConnections; + } } /** Edge rate-limit provider composition. */ diff --git a/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisSemanticScriptAclProbe.java b/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisSemanticScriptAclProbe.java index dd88d831..431246aa 100644 --- a/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisSemanticScriptAclProbe.java +++ b/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisSemanticScriptAclProbe.java @@ -28,11 +28,12 @@ import org.springframework.beans.factory.SmartInitializingSingleton; * {@code {1}} only when the value it wrote reads back. * *

    A cache-only deployment is probed too: its conditional writes and revision-ordered writes are - * registered programs on the same lane, and without the permission every one of them would come + * registered programs on the same account, and without the permission every one of them would come * back degraded while plain reads looked healthy. Its probe runs only what the cache programs call - * ({@code GET}, {@code SET}), and a Redis it cannot reach at startup is logged, not fatal — a cache - * outage degrades, it never stops a pod from starting. A server that answers and refuses the - * scripts is a configuration error and fails startup as it does for the correctness roles. + * ({@code GET}, {@code SET}). Only a server that answers and refuses the scripts — an ACL refusal, + * {@code NOSCRIPT}, or a reply other than the value the probe wrote — is a configuration error and + * fails startup as it does for the correctness roles. Every other failure is a cache outage and is + * logged, not fatal: a cache outage degrades, it never stops a pod from starting. */ final class RedisSemanticScriptAclProbe implements SmartInitializingSingleton { @@ -89,7 +90,7 @@ final class RedisSemanticScriptAclProbe implements SmartInitializingSingleton { try { probe(CACHE_PROBE_SCRIPT); } catch (CapabilityStartupValidationException failure) { - if (!unreachable(failure.getCause())) { + if (refusedByTheServer(failure.getCause())) { throw new CapabilityStartupValidationException( "Redis cache is selected but its account cannot run the cache's conditional-write" + " scripts (SCRIPT LOAD/EVALSHA, GET, SET); every conditional record would" @@ -98,15 +99,49 @@ final class RedisSemanticScriptAclProbe implements SmartInitializingSingleton { failure.getCause()); } LOG.warn( - "Redis cache is selected but Redis was unreachable at startup; the cache script" - + " permission is unverified and the cache runs degraded until Redis answers", + "Redis cache is selected but the startup script probe could not complete; the cache" + + " script permission is unverified and the cache runs degraded until Redis answers", failure.getCause()); } } } - private static boolean unreachable(Throwable cause) { - return cause instanceof RedisLaneUnavailableException || cause instanceof TimeoutException; + /** + * Whether the cache probe failed because the server refused the cache's scripts. + * + *

    "Refused" is a statement about the account, and only three answers make it: an ACL refusal + * ({@code NOPERM}, or a script's inner command denied), {@code NOSCRIPT} for the program the same + * connection had just loaded, and a reply other than the value the probe wrote. The check used to + * run the other way — only a lane refusal or a timeout counted as "unreachable" and everything + * else was a configuration error — so a connection that dropped mid-probe, a restarting Redis + * answering {@code LOADING}, or a busy one answering {@code BUSY} stopped a cache-only pod from + * starting, against this leaf's promise that a cache outage never does (review 2026-11-03 + * RR-04-07). + * + * @param cause the probe's failure + * @return {@code true} when the account, not the server's availability, is at fault + */ + private static boolean refusedByTheServer(Throwable cause) { + for (Throwable current = cause; + current != null; + current = current.getCause() == current ? null : current.getCause()) { + if (current instanceof CapabilityStartupValidationException) { + // The probe ran and read back something other than what it wrote. + return true; + } + if (current instanceof RedisLaneUnavailableException || current instanceof TimeoutException) { + return false; + } + String message = current.getMessage(); + if (message != null + && (message.startsWith("NOPERM") + || message.startsWith("NOSCRIPT") + || message.contains("no permissions") + || message.contains("ACL failure"))) { + return true; + } + } + return false; } private void probe(byte[] script) { diff --git a/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/idempotency/IdempotencyScripts.java b/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/idempotency/IdempotencyScripts.java index b0ddddda..47e32400 100644 --- a/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/idempotency/IdempotencyScripts.java +++ b/src/adapter/outbound/cache-redis/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/idempotency/IdempotencyScripts.java @@ -141,6 +141,12 @@ public final class IdempotencyScripts { * it started from and the lease it asked for. A retry from that revision with the same lease is * the same renewal; with a different lease it is a different request, because confirming it would * leave the caller believing it holds the record for a lease the record does not carry. + * + *

    The key's own TTL is stretched to cover the new lease. Claim sets it to the replay TTL, + * which may be shorter than a lease renewed past it; the key then expired while the holder was + * still executing, and the next request got a fresh claim and ran the side effect again (review + * 2026-11-03 RR-04-08). The TTL is only ever lengthened here — completion sets the replay TTL + * again. */ private static final String RENEW = """ @@ -177,6 +183,10 @@ public final class IdempotencyScripts { redis.call('HSET', KEYS[1], 'leaseUntil', nowMillis + tonumber(ARGV[4]), 'rev', rev + 1, 'renewedFromRev', ARGV[2], 'renewedLease', ARGV[4]) + local ttl = redis.call('PTTL', KEYS[1]) + if ttl >= 0 and ttl < tonumber(ARGV[4]) then + redis.call('PEXPIRE', KEYS[1], ARGV[4]) + end return {'APPLIED', attempt, rev + 1, owner, '', ''} """; diff --git a/src/adapter/outbound/cache-redis/src/main/resources/config/module-defaults/cache-redis.yml b/src/adapter/outbound/cache-redis/src/main/resources/config/module-defaults/cache-redis.yml index eab2668f..8dc4b8f2 100644 --- a/src/adapter/outbound/cache-redis/src/main/resources/config/module-defaults/cache-redis.yml +++ b/src/adapter/outbound/cache-redis/src/main/resources/config/module-defaults/cache-redis.yml @@ -39,6 +39,10 @@ app: # Floor on the hard TTL. Below it, entries expire faster than the round trip that wrote them: # every read misses and every miss writes. minimum-hard-ttl: ${APP_CACHE_REDIS_MINIMUM_HARD_TTL:1s} + # The cache's own script-lane connections, apart from the SCRIPT lane that idempotency, rate + # limiting and leases share (app.redis.capacity.maximum-script-connections). A cache under + # load runs out of these and degrades; it cannot take the correctness features' connections. + maximum-script-connections: ${APP_CACHE_REDIS_MAXIMUM_SCRIPT_CONNECTIONS:4} idempotency: # Redis store tuning only; the provider selector and lifecycle policy stay with the # application's idempotency executor. @@ -75,15 +79,8 @@ app: maximum-cost: ${APP_RATE_LIMIT_MAXIMUM_COST:10} cleanup-grace: ${APP_RATE_LIMIT_CLEANUP_GRACE:5s} -# redis-runtime 의 redisOptional 은 Redis 에 닿지 못하면 DEGRADED 를 보고한다. Boot 기본 순서 -# (DOWN, OUT_OF_SERVICE, UP, UNKNOWN)에 없는 상태는 DOWN 보다 앞에 정렬돼, 다른 contributor 가 DOWN -# 이어도 전체 health 가 HTTP 200 DEGRADED 로 보였다(리뷰 05-23e). DEGRADED 를 DOWN 뒤, UP 앞에 둔다. -# 이 키를 정하는 module-defaults 는 이 파일 하나뿐이다. 서비스가 순서를 바꾸면 DEGRADED 를 넣어야 한다. -management: - endpoint: - health: - status: - order: DOWN,OUT_OF_SERVICE,DEGRADED,UP,UNKNOWN +# DEGRADED 상태 순서(management.endpoint.health.status.order)는 그 상태를 내는 redis-runtime 이 +# 싣는다(config/module-defaults/redis-runtime.yml, 리뷰 2026-11-03 RR-04-10). # /actuator/info 에 싣는 이 모듈의 상태(management.info.env). compose smoke 가 lane 이 실제로 켠 # capability 를 이 값으로 확인한다. 위 스위치를 해석한 값이고, 기본값은 RedisCapabilityConfig 의 diff --git a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/LiveRedisRuntime.java b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/LiveRedisRuntime.java index 654c0bc0..6ec0796c 100644 --- a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/LiveRedisRuntime.java +++ b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/LiveRedisRuntime.java @@ -1,5 +1,6 @@ package dev.caskeleton.adapter.outbound.cache.redis; +import dev.caskeleton.adapter.outbound.cache.redis.cache.CacheScriptLane; import dev.caskeleton.adapter.outbound.cache.redis.runtime.RedisTopologyEndpoint; import dev.caskeleton.adapter.outbound.cache.redis.runtime.config.RedisCredentialResolver.RedisCredentials; import dev.caskeleton.adapter.outbound.cache.redis.runtime.config.RedisRuntimeSettings; @@ -71,6 +72,9 @@ public final class LiveRedisRuntime { for (RedisConnectionKind kind : RedisConnectionKind.values()) { limits.put(kind, 4); } + // The cache region's own script lane, which the composition root contributes when the cache + // binding is redis (RedisCacheConfig). + limits.put(CacheScriptLane.CACHE_SCRIPT, 4); return new RedisRuntimeOwner( new RedisTopologyClientFactory( settings, diff --git a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/RedisTestContainer.java b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/RedisTestContainer.java index a23f444b..65ef2da7 100644 --- a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/RedisTestContainer.java +++ b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/RedisTestContainer.java @@ -28,7 +28,7 @@ final class RedisTestContainer { static final String ACL_FILE_PROPERTY = "redis.container.acl-file"; private static final String IMAGE_PROPERTY = "ca.test.image.redis"; - private static final String ACL_IN_CONTAINER = "/etc/redis/acl/all-accounts.acl"; + private static final String ACL_IN_CONTAINER = "/etc/redis/acl/lane-fixture-accounts.acl"; private static final int PORT = 6379; private static GenericContainer server; diff --git a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/cache/RedisCacheRegionAdapterTest.java b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/cache/RedisCacheRegionAdapterTest.java index e83d72d9..25fc90db 100644 --- a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/cache/RedisCacheRegionAdapterTest.java +++ b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/cache/RedisCacheRegionAdapterTest.java @@ -60,8 +60,17 @@ class RedisCacheRegionAdapterTest { for (RedisConnectionKind kind : RedisConnectionKind.values()) { limits.put(kind, 4); } - return new RedisCacheRegionAdapter<>( + limits.put(CacheScriptLane.CACHE_SCRIPT, 4); + return cache( + now, new RedisRuntimeOwner(client, limits, Duration.ofSeconds(1), Duration.ofSeconds(1)), + cacheKeys); + } + + private RedisCacheRegionAdapter cache( + Instant now, RedisRuntimeOwner owner, RedisCacheRegionAdapter.CacheKeys cacheKeys) { + return new RedisCacheRegionAdapter<>( + owner, cacheKeys, key -> "digest-" + key, value -> value.getBytes(StandardCharsets.UTF_8), @@ -428,6 +437,31 @@ class RedisCacheRegionAdapterTest { .hasMessageContaining("ONLY_IF_SOURCE_REVISION_NEWER"); } + @Test + @DisplayName("멱등성·lease가 SCRIPT lane을 모두 잡고 있어도 캐시 조건부 기록은 캐시 전용 lane에서 기록된다") + void aConditionalCacheWriteDoesNotWaitForTheCorrectnessScriptLane() { + // given + Map limits = new java.util.LinkedHashMap<>(); + limits.put(RedisConnectionKind.REGULAR, 4); + limits.put(RedisConnectionKind.SCRIPT, 1); + limits.put(CacheScriptLane.CACHE_SCRIPT, 1); + try (RedisRuntimeOwner owner = + new RedisRuntimeOwner( + StubRedisRuntimeClient.serving(gateway), + limits, + Duration.ofMillis(100), + Duration.ofSeconds(1)); + var heldByAnIdempotencyClaim = owner.borrow(RedisConnectionKind.SCRIPT)) { + RedisCacheRegionAdapter cache = cache(T0, owner, keys); + + // when + CacheRecordOutcome outcome = cache.record("order-1", "payload", newerThan("2")); + + // then + assertThat(outcome).isEqualTo(CacheRecordOutcome.RECORDED); + } + } + @Test @DisplayName("an unreachable Redis degrades: the cache is bypassed, nothing fails") void anUnreachableRedisDegrades() { diff --git a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/cache/RedisCacheRegionContractTest.java b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/cache/RedisCacheRegionContractTest.java index d3a996dc..8044176a 100644 --- a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/cache/RedisCacheRegionContractTest.java +++ b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/cache/RedisCacheRegionContractTest.java @@ -23,6 +23,7 @@ class RedisCacheRegionContractTest extends CacheRegionContract { for (RedisConnectionKind kind : RedisConnectionKind.values()) { limits.put(kind, 4); } + limits.put(CacheScriptLane.CACHE_SCRIPT, 4); return new RedisCacheRegionAdapter<>( new RedisRuntimeOwner( StubRedisRuntimeClient.serving(new InMemorySemanticRedisCommands()), diff --git a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisCapabilityCompositionTest.java b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisCapabilityCompositionTest.java index 16e01054..d81e6d13 100644 --- a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisCapabilityCompositionTest.java +++ b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisCapabilityCompositionTest.java @@ -3,6 +3,9 @@ package dev.caskeleton.adapter.outbound.cache.redis.capability; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import dev.caskeleton.adapter.outbound.cache.redis.cache.CacheScriptLane; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.config.RedisLaneContribution; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.config.RedisLaneMetrics; import dev.caskeleton.adapter.outbound.cache.redis.runtime.config.RedisRuntimeAutoConfiguration; import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisRuntimeOwner; import dev.caskeleton.adapter.outbound.support.startup.StartupFailureException; @@ -11,6 +14,8 @@ import dev.caskeleton.application.idempotency.IdempotencyStorePort; import dev.caskeleton.application.lease.DistributedLeasePort; import dev.caskeleton.application.secret.SecretSourcePort; import dev.caskeleton.shared.ratelimit.EdgeRateLimitPort; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import java.time.Clock; import java.time.Duration; import java.util.Arrays; @@ -94,6 +99,66 @@ class RedisCapabilityCompositionTest { }); } + @Test + @DisplayName("Redis 캐시 binding은 설정한 상한으로 캐시 전용 script lane을 SCRIPT lane과 따로 기여한다") + void theCacheBindingContributesItsOwnScriptLane() { + runner + .withPropertyValues( + "app.capabilities.cache.bindings.default=redis", + "app.capabilities.cache.maximum-script-connections=3") + .run( + context -> { + // when + List contributions = + List.copyOf(context.getBeansOfType(RedisLaneContribution.class).values()); + + // then + assertThat(context).hasNotFailed(); + assertThat(contributions) + .containsExactly(new RedisLaneContribution(CacheScriptLane.CACHE_SCRIPT, 3)); + }); + } + + @Test + @DisplayName("Redis 캐시 binding의 캐시 전용 lane은 SCRIPT lane과 따로 대기·거절·재연결 지표를 낸다") + void theCacheScriptLaneHasMetersOfItsOwn() { + runner + .withPropertyValues("app.capabilities.cache.bindings.default=redis") + .withBean(MeterRegistry.class, SimpleMeterRegistry::new) + .run( + context -> { + // when + MeterRegistry registry = context.getBean(MeterRegistry.class); + + // then + assertThat(context).hasNotFailed(); + for (String lane : List.of(CacheScriptLane.CACHE_SCRIPT.name(), "SCRIPT")) { + assertThat( + registry + .find(RedisLaneMetrics.CONNECTIONS_PENDING) + .tag("lane", lane) + .gauge()) + .as("waiting borrows of lane %s", lane) + .isNotNull(); + assertThat( + registry + .find(RedisLaneMetrics.ACQUIRE_REFUSED) + .tag("lane", lane) + .tag("reason", "saturated") + .counter()) + .as("refusals of lane %s", lane) + .isNotNull(); + assertThat( + registry + .find(RedisLaneMetrics.CONNECTIONS_OPENED) + .tag("lane", lane) + .counter()) + .as("connections opened on lane %s", lane) + .isNotNull(); + } + }); + } + @Test @DisplayName("a rate limiter with no configured policy is refused at startup") void aRateLimiterWithoutPoliciesIsRefused() { diff --git a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisSemanticScriptAclProbeTest.java b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisSemanticScriptAclProbeTest.java index c1909150..71a846dc 100644 --- a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisSemanticScriptAclProbeTest.java +++ b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/capability/RedisSemanticScriptAclProbeTest.java @@ -191,6 +191,85 @@ class RedisSemanticScriptAclProbeTest { assertThat(failure).isNull(); } + @ParameterizedTest + @ValueSource( + strings = { + "LOADING Redis is loading the dataset in memory", + "Connection closed", + "BUSY Redis is busy running a script" + }) + @DisplayName("cache만 고른 배포는 probe 도중 연결이 끊기거나 Redis가 아직 준비 중이어도 권한 오류로 보지 않고 기동한다") + void aCacheOnlyDeploymentStartsWhenTheProbeIsInterruptedByATransientFailure(String reply) { + // given + RedisCapabilitySettings capabilities = new RedisCapabilitySettings(); + capabilities.getCache().getBindings().put("default", "redis"); + RedisSemanticScriptAclProbe probe = + new RedisSemanticScriptAclProbe( + owner(gatewayFailingLoadWith(reply), new AtomicInteger()), + capabilities, + "prod:svc:shared:probe".getBytes(StandardCharsets.UTF_8), + Duration.ofSeconds(1)); + + // when + Throwable failure = catchThrowable(probe::afterSingletonsInstantiated); + + // then + assertThat(failure).isNull(); + } + + @Test + @DisplayName("cache만 고른 배포도 probe가 기대와 다른 답을 받으면 기동에서 실패한다") + void aCacheOnlyDeploymentWhoseProbeReadsBackTheWrongValueFailsStartup() { + // given + RedisCapabilitySettings capabilities = new RedisCapabilitySettings(); + capabilities.getCache().getBindings().put("default", "redis"); + RedisSemanticScriptAclProbe probe = + new RedisSemanticScriptAclProbe( + owner(gatewayAnswering(List.of(0L)), new AtomicInteger()), + capabilities, + "prod:svc:shared:probe".getBytes(StandardCharsets.UTF_8), + Duration.ofSeconds(1)); + + // when + Throwable failure = catchThrowable(probe::afterSingletonsInstantiated); + + // then + assertThat(failure) + .isInstanceOf(CapabilityStartupValidationException.class) + .hasMessageContaining("cache"); + } + + private static SemanticRedisCommands gatewayFailingLoadWith(String reply) { + return (SemanticRedisCommands) + Proxy.newProxyInstance( + SemanticRedisCommands.class.getClassLoader(), + new Class[] {SemanticRedisCommands.class}, + (proxy, method, arguments) -> + switch (method.getName()) { + case "loadScript" -> CompletableFuture.failedFuture(new RuntimeException(reply)); + case "toString" -> "RedisSemanticScriptAclProbeTestGateway"; + default -> + throw new UnsupportedOperationException( + "unexpected gateway method: " + method.getName()); + }); + } + + private static SemanticRedisCommands gatewayAnswering(List reply) { + return (SemanticRedisCommands) + Proxy.newProxyInstance( + SemanticRedisCommands.class.getClassLoader(), + new Class[] {SemanticRedisCommands.class}, + (proxy, method, arguments) -> + switch (method.getName()) { + case "loadScript" -> CompletableFuture.completedFuture("probe-digest"); + case "evaluateRegisteredForList" -> CompletableFuture.completedFuture(reply); + case "toString" -> "RedisSemanticScriptAclProbeTestGateway"; + default -> + throw new UnsupportedOperationException( + "unexpected gateway method: " + method.getName()); + }); + } + private static RedisRuntimeOwner owner(SemanticRedisCommands gateway, AtomicInteger opened) { RedisRuntimeClient client = new RedisRuntimeClient() { diff --git a/src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/security/RoleBasedFileAccessPolicy.java b/src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/security/RoleBasedFileAccessPolicy.java index 3405c544..fbab6023 100644 --- a/src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/security/RoleBasedFileAccessPolicy.java +++ b/src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/security/RoleBasedFileAccessPolicy.java @@ -71,26 +71,15 @@ public final class RoleBasedFileAccessPolicy implements FileAccessPolicyPort { if (permitted.stream().noneMatch(subject.roles()::contains)) { throw denied(FileserverErrorCode.ACCESS_DENIED, "subject does not hold a permitted role"); } - if (descriptor.isPresent() && !isOwnedBy(descriptor.get(), subject)) { + // Ownership is FileAccessSubject.owns, the rule the port owns (owner decision 14): a file with + // no recorded owner predates ownership and belongs to nobody on the data plane. An admin role + // does not bypass it — acting on somebody else's file is what the audited management plane is + // for, and its operations carry no descriptor. + if (descriptor.isPresent() && !subject.owns(descriptor.get())) { throw denied(FileserverErrorCode.ACCESS_DENIED, "subject does not own the file"); } } - /** - * Whether the file was created by this caller, in this caller's tenant. - * - *

    A file without a recorded owner predates ownership and belongs to nobody on the data plane. - * An admin role does not bypass this: acting on somebody else's file is what the audited - * management plane is for, and its operations carry no descriptor. - */ - private static boolean isOwnedBy(FileDescriptor descriptor, FileAccessSubject subject) { - return descriptor - .owner() - .filter(owner -> owner.principalId().equals(subject.principalId())) - .filter(owner -> owner.tenantId().equals(subject.tenant())) - .isPresent(); - } - private static boolean isAnonymous(FileAccessSubject subject) { return subject.principalId().equals(FileAccessSubject.anonymous().principalId()); } diff --git a/src/adapter/outbound/fileserver/src/test/java/dev/caskeleton/adapter/outbound/fileserver/platform/security/RoleBasedFileAccessPolicyContractTest.java b/src/adapter/outbound/fileserver/src/test/java/dev/caskeleton/adapter/outbound/fileserver/platform/security/RoleBasedFileAccessPolicyContractTest.java new file mode 100644 index 00000000..f192d9ac --- /dev/null +++ b/src/adapter/outbound/fileserver/src/test/java/dev/caskeleton/adapter/outbound/fileserver/platform/security/RoleBasedFileAccessPolicyContractTest.java @@ -0,0 +1,31 @@ +package dev.caskeleton.adapter.outbound.fileserver.platform.security; + +import dev.caskeleton.application.fileserver.api.security.FileAccessPolicyPort; +import dev.caskeleton.application.fileserver.api.security.FileAccessSubject; +import dev.caskeleton.application.fileserver.testkit.FileAccessPolicyPortContract; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** The production policy held to the ownership contract its fake is held to (RR-07-06). */ +class RoleBasedFileAccessPolicyContractTest extends FileAccessPolicyPortContract { + + private static final String READ = "ROLE_FILE_READ"; + private static final String WRITE = "ROLE_FILE_WRITE"; + + private final RoleBasedFileAccessPolicy policy = + new RoleBasedFileAccessPolicy(Set.of(READ), Set.of(WRITE), Set.of("ROLE_FILE_ADMIN"), false); + + @Override + protected FileAccessPolicyPort policy() { + return policy; + } + + @Override + protected FileAccessSubject permittedSubject(String principalId, Optional tenant) { + return new FileAccessSubject( + principalId, + Set.of(READ, WRITE), + tenant.map(value -> Map.of(FileAccessSubject.TENANT_ATTRIBUTE, value)).orElse(Map.of())); + } +} diff --git a/src/adapter/outbound/httpclient/README.md b/src/adapter/outbound/httpclient/README.md index faa4f02a..b80f46d8 100644 --- a/src/adapter/outbound/httpclient/README.md +++ b/src/adapter/outbound/httpclient/README.md @@ -41,7 +41,11 @@ no reactive (`WebClient`/Reactor Netty) surface; it was removed on 2026-09-25 (d cancels the request at the deadline, the JDK transport bounds its exchange timeout by the time left, the experimental Jetty HTTP/3 transport aborts the request, and the caller gets `HttpDeadlineExceededException`. A streaming download's setup (up to the response headers) runs - inside the same deadline; after hand-over each body gap is bounded by `read-idle`. + inside the same deadline; after hand-over each body gap is bounded by `read-idle` and nothing else, + on both transports — the JDK transport arms its own per-read timer for this, because the JDK client + has no socket read timeout. A body that fails after hand-over throws the same stable exceptions as + a buffered call (`HttpResponseTimeoutException` for a stalled body, `HttpResponseTruncatedException` + for a cut one). - **Evidence.** Each attempt reports `NOT_SENT`, `SENT_NO_RESPONSE`, `RESPONSE_RECEIVED`, or `PARTIAL_RESPONSE`. `NOT_SENT` is only claimed when a stage failure proves it. - **Retry.** Never from the HTTP method alone. Idempotency, body replayability, evidence, deadline, @@ -53,7 +57,9 @@ no reactive (`WebClient`/Reactor Netty) surface; it was removed on 2026-09-25 (d whether the call throws them or returns them as a result (`RETURN_RESULT`). Any other 4xx (404, 409, 422 …) is a healthy answer, even when it reaches the caller as `HttpRemoteErrorException`. Window, threshold and open-state wait are per profile under `clients[N].circuit-breaker.*` - (defaults are Resilience4j's; see `docs/httpclient/configuration-reference.md`). + (defaults are Resilience4j's; see `docs/httpclient/configuration-reference.md`). An attempt that + the caller's own deadline (`HttpOperation.deadline()`) ended is not counted at all: the caller ran + out of time, the upstream did not fail. An attempt the profile's `total-call` ended still counts. - **Resources.** Connections and buffers are reclaimed after success, error, decode failure, size rejection, and cancellation alike. - **Secrets.** No URL, query value, path variable, token, cookie, or idempotency key reaches a log, diff --git a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/DeadlineAwareJdkRequestFactory.java b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/DeadlineAwareJdkRequestFactory.java index ce9ed51a..6b28ad8f 100644 --- a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/DeadlineAwareJdkRequestFactory.java +++ b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/DeadlineAwareJdkRequestFactory.java @@ -21,22 +21,35 @@ import org.springframework.http.client.JdkClientHttpRequestFactory; * timeout is fixed per factory, so a factory is made per request with {@code min(response-header * timeout, remaining deadline)} from {@link AttemptDeadline}. The factory only holds references; * the client and the write executor are shared. + * + *

    A streaming download cannot use that request. Its body is handed to the caller and may run for + * minutes, but Spring's timer only stops when the body stream is closed, so every JDK download + * longer than the response-header timeout was cut mid-body (review 2026-11-03 RR-04-03). Inside a + * streaming setup scope the factory makes a {@link JdkStreamingClientHttpRequest} instead: the + * JDK's own request timeout bounds the wait for headers, the attempt deadline cancels the exchange + * up to hand-over, and the body is bounded per gap by {@code read-idle}. */ final class DeadlineAwareJdkRequestFactory implements ClientHttpRequestFactory { private final HttpClient client; private final Executor writeExecutor; private final Duration exchangeTimeout; + private final Duration readIdle; - DeadlineAwareJdkRequestFactory(HttpClient client, Duration exchangeTimeout) { + DeadlineAwareJdkRequestFactory(HttpClient client, Duration exchangeTimeout, Duration readIdle) { this.client = Objects.requireNonNull(client, "jdk http client"); this.exchangeTimeout = Objects.requireNonNull(exchangeTimeout, "exchange timeout"); + this.readIdle = Objects.requireNonNull(readIdle, "read idle timeout"); // The executor Spring would pick itself, chosen once instead of once per request. this.writeExecutor = client.executor().orElseGet(SimpleAsyncTaskExecutor::new); } @Override public ClientHttpRequest createRequest(URI uri, HttpMethod httpMethod) throws IOException { + if (AttemptDeadline.boundsSetupOnly()) { + return new JdkStreamingClientHttpRequest( + client, uri, httpMethod, AttemptDeadline.bound(exchangeTimeout), readIdle); + } JdkClientHttpRequestFactory delegate = new JdkClientHttpRequestFactory(client, writeExecutor); delegate.setReadTimeout(AttemptDeadline.bound(exchangeTimeout)); return delegate.createRequest(uri, httpMethod); diff --git a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/JdkBlockingTransportProvider.java b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/JdkBlockingTransportProvider.java index 64fab27f..4453f4b4 100644 --- a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/JdkBlockingTransportProvider.java +++ b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/JdkBlockingTransportProvider.java @@ -62,7 +62,8 @@ public final class JdkBlockingTransportProvider implements BlockingTransportProv HttpClient client = clientFactory.create(profile, tlsMaterialResolver.apply(profile)); clients.put(new TransportResourceKey(profile.name(), generation), client); ClientHttpRequestFactory factory = - new DeadlineAwareJdkRequestFactory(client, profile.timeout().responseHeader()); + new DeadlineAwareJdkRequestFactory( + client, profile.timeout().responseHeader(), profile.timeout().readIdle()); listener.onRuntimeCreated(profile.name(), ID); return factory; } diff --git a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/JdkStreamingClientHttpRequest.java b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/JdkStreamingClientHttpRequest.java new file mode 100644 index 00000000..f38f1459 --- /dev/null +++ b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/JdkStreamingClientHttpRequest.java @@ -0,0 +1,128 @@ +package dev.caskeleton.adapter.outbound.httpclient.jdk; + +import dev.caskeleton.adapter.outbound.httpclient.transport.AttemptDeadline; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.client.AbstractBufferingClientHttpRequest; +import org.springframework.http.client.ClientHttpResponse; + +/** + * A JDK request for a streaming download, whose timeouts end at the response headers (review + * 2026-11-03 RR-04-03). + * + *

    Three limits, each where the JDK can enforce it: + * + *

      + *
    • the JDK's own request timeout bounds the wait for the response headers and stops when they + * arrive; + *
    • the attempt deadline cancels the pending exchange, so connect, send and the header wait + * stay inside the call's total budget; + *
    • after hand-over, {@link ReadIdleInputStream} bounds each gap of the body by {@code + * read-idle} — the JDK client has no socket read timeout of its own. + *
    + * + *

    The request body is buffered: a streaming download sends none or a small one, and the JDK + * needs a body publisher before the exchange starts. + */ +final class JdkStreamingClientHttpRequest extends AbstractBufferingClientHttpRequest { + + /** Headers the JDK client refuses to set, as Spring's own JDK request lists them. */ + private static final Set RESTRICTED_HEADERS = restrictedHeaders(); + + private final HttpClient client; + private final URI uri; + private final HttpMethod method; + private final Duration responseHeaderTimeout; + private final Duration readIdle; + + JdkStreamingClientHttpRequest( + HttpClient client, + URI uri, + HttpMethod method, + Duration responseHeaderTimeout, + Duration readIdle) { + this.client = Objects.requireNonNull(client, "jdk http client"); + this.uri = Objects.requireNonNull(uri, "uri"); + this.method = Objects.requireNonNull(method, "method"); + this.responseHeaderTimeout = + Objects.requireNonNull(responseHeaderTimeout, "response header timeout"); + this.readIdle = Objects.requireNonNull(readIdle, "read idle timeout"); + } + + @Override + public HttpMethod getMethod() { + return method; + } + + @Override + public URI getURI() { + return uri; + } + + @Override + protected ClientHttpResponse executeInternal(HttpHeaders headers, byte[] body) + throws IOException { + HttpRequest.Builder request = + HttpRequest.newBuilder(uri) + .timeout(responseHeaderTimeout) + .method( + method.name(), + body.length == 0 + ? HttpRequest.BodyPublishers.noBody() + : HttpRequest.BodyPublishers.ofByteArray(body)); + headers.forEach( + (name, values) -> { + if (!RESTRICTED_HEADERS.contains(name)) { + values.forEach(value -> request.header(name, value)); + } + }); + CompletableFuture> pending = + client.sendAsync(request.build(), HttpResponse.BodyHandlers.ofInputStream()); + AttemptDeadline.abortOnExpiry(() -> pending.cancel(true)); + HttpResponse response = await(pending); + return new JdkStreamingClientHttpResponse( + response, new ReadIdleInputStream(response.body(), readIdle)); + } + + private static HttpResponse await( + CompletableFuture> pending) throws IOException { + try { + return pending.get(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + pending.cancel(true); + InterruptedIOException failure = new InterruptedIOException("streaming setup interrupted"); + failure.initCause(interrupted); + throw failure; + } catch (CancellationException cancelled) { + // Only the attempt deadline cancels this future; the gateway names the deadline. + throw new IOException("streaming setup cancelled at the attempt deadline", cancelled); + } catch (ExecutionException failed) { + Throwable cause = failed.getCause(); + if (cause instanceof IOException io) { + throw io; + } + throw new IOException("streaming setup failed", cause); + } + } + + private static Set restrictedHeaders() { + Set restricted = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + restricted.addAll(Set.of("connection", "content-length", "expect", "host", "upgrade")); + return restricted; + } +} diff --git a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/JdkStreamingClientHttpResponse.java b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/JdkStreamingClientHttpResponse.java new file mode 100644 index 00000000..7a6f8ffa --- /dev/null +++ b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/JdkStreamingClientHttpResponse.java @@ -0,0 +1,63 @@ +package dev.caskeleton.adapter.outbound.httpclient.jdk; + +import java.io.IOException; +import java.io.InputStream; +import java.net.http.HttpResponse; +import java.util.Objects; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.HttpStatusCode; +import org.springframework.http.client.ClientHttpResponse; + +/** The response of a {@link JdkStreamingClientHttpRequest}, body bounded per gap. */ +final class JdkStreamingClientHttpResponse implements ClientHttpResponse { + + private final HttpResponse response; + private final InputStream body; + private final HttpHeaders headers = new HttpHeaders(); + + JdkStreamingClientHttpResponse(HttpResponse response, InputStream body) { + this.response = Objects.requireNonNull(response, "jdk response"); + this.body = Objects.requireNonNull(body, "body stream"); + // HTTP/2 pseudo-headers (":status") are protocol framing, not response headers. + response + .headers() + .map() + .forEach( + (name, values) -> { + if (!name.startsWith(":")) { + headers.addAll(name, values); + } + }); + } + + @Override + public HttpStatusCode getStatusCode() { + return HttpStatusCode.valueOf(response.statusCode()); + } + + @Override + public String getStatusText() { + HttpStatus known = HttpStatus.resolve(response.statusCode()); + return known == null ? "" : known.getReasonPhrase(); + } + + @Override + public HttpHeaders getHeaders() { + return headers; + } + + @Override + public InputStream getBody() { + return body; + } + + @Override + public void close() { + try { + body.close(); + } catch (IOException ignored) { + // Closing releases the exchange; a failure to close has nothing left to release. + } + } +} diff --git a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/ReadIdleInputStream.java b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/ReadIdleInputStream.java new file mode 100644 index 00000000..02643336 --- /dev/null +++ b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/jdk/ReadIdleInputStream.java @@ -0,0 +1,90 @@ +package dev.caskeleton.adapter.outbound.httpclient.jdk; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.SocketTimeoutException; +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A JDK response body whose every read is bounded by {@code read-idle} (review 2026-11-03 + * RR-04-03). + * + *

    The JDK client has no socket read timeout, so a streamed body that stopped arriving held the + * caller's thread and the connection forever. Each read arms a timer on the JDK's shared delay + * scheduler; when it fires the stream is closed, which wakes the blocked read, and the read reports + * a {@link SocketTimeoutException} — the same exception Apache's socket timeout raises, so one + * classifier reading names both. A read that returns first disarms its timer. + */ +final class ReadIdleInputStream extends FilterInputStream { + + @FunctionalInterface + private interface Read { + long run() throws IOException; + } + + private final Duration readIdle; + private final AtomicBoolean idled = new AtomicBoolean(); + + ReadIdleInputStream(InputStream delegate, Duration readIdle) { + super(Objects.requireNonNull(delegate, "body stream")); + this.readIdle = Objects.requireNonNull(readIdle, "read idle timeout"); + } + + @Override + public int read() throws IOException { + return (int) guarded(super::read); + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + return (int) guarded(() -> super.read(buffer, offset, length)); + } + + @Override + public long skip(long count) throws IOException { + return guarded(() -> super.skip(count)); + } + + private long guarded(Read read) throws IOException { + requireNotIdled(); + CompletableFuture timer = new CompletableFuture<>(); + CompletableFuture expiry = + timer.completeOnTimeout(null, readIdle.toNanos(), TimeUnit.NANOSECONDS).thenRun(this::idle); + try { + long result = read.run(); + requireNotIdled(); + return result; + } catch (IOException failure) { + requireNotIdled(); + throw failure; + } finally { + // Completing first skips the idle action and lets the scheduler drop the pending task. + timer.cancel(false); + expiry.cancel(false); + } + } + + private void idle() { + idled.set(true); + try { + in.close(); + } catch (IOException ignored) { + // The read this wakes reports the timeout; a failed close has nothing more to say. + } + } + + /** + * After the timer fired, whatever the read returned — data, end of stream or "closed" — is not + * the rest of the body. + */ + private void requireNotIdled() throws SocketTimeoutException { + if (idled.get()) { + throw new SocketTimeoutException("no response body bytes within the read-idle timeout"); + } + } +} diff --git a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/AttemptCircuitBreaker.java b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/AttemptCircuitBreaker.java index 26b571da..88f89eef 100644 --- a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/AttemptCircuitBreaker.java +++ b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/AttemptCircuitBreaker.java @@ -17,6 +17,13 @@ public interface AttemptCircuitBreaker { void onError(long durationNanos, Throwable failure); + /** + * Gives back a permission without recording an outcome, for an attempt that says nothing about + * the upstream (an attempt its caller's own deadline ended). Half-open trial slots are returned; + * the window is left as it was. + */ + void releasePermission(); + String state(); static AttemptCircuitBreaker alwaysClosed() { @@ -36,6 +43,11 @@ public interface AttemptCircuitBreaker { // Nothing to record. } + @Override + public void releasePermission() { + // Nothing was taken. + } + @Override public String state() { return "CLOSED"; @@ -60,6 +72,11 @@ public interface AttemptCircuitBreaker { delegate.onError(durationNanos, TimeUnit.NANOSECONDS, failure); } + @Override + public void releasePermission() { + delegate.releasePermission(); + } + @Override public String state() { return delegate.getState().name(); diff --git a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/AttemptResiliencePipeline.java b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/AttemptResiliencePipeline.java index d306c441..df8c946e 100644 --- a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/AttemptResiliencePipeline.java +++ b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/AttemptResiliencePipeline.java @@ -7,6 +7,7 @@ import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpFailureMetadata; import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpRateLimitRejectedException; import java.util.Objects; import java.util.Optional; +import java.util.function.BooleanSupplier; import java.util.function.Function; import java.util.function.Supplier; @@ -21,6 +22,11 @@ import java.util.function.Supplier; *

    A local rejection (rate limiter or bulkhead) is deliberately not recorded as a * circuit error: the upstream never saw the request, and counting our own back-pressure as upstream * failure would open the breaker on a healthy dependency. + * + *

    An attempt its caller abandoned is not recorded either, for the same reason: an attempt that a + * caller's own deadline ended while the upstream was still answering within the profile's limits is + * the caller running out of time, not the upstream failing (review 2026-11-03 RR-04-01). Its + * permission is given back instead. */ public final class AttemptResiliencePipeline { @@ -79,7 +85,24 @@ public final class AttemptResiliencePipeline { * @return the attempt's value, whether or not it represents a remote failure */ public T execute(AttemptCall call, Function> remoteFailure) { + return execute(call, remoteFailure, () -> false); + } + + /** + * Runs one attempt, recording nothing against the upstream when its caller abandoned it. + * + * @param call the attempt, including any redirect hops it follows + * @param remoteFailure returns the failure to record when the value represents an upstream error + * @param abandonedByCaller read after the attempt failed: whether the caller's own deadline ended + * it, in which case the failure is not the upstream's and the breaker records nothing + * @return the attempt's value, whether or not it represents a remote failure + */ + public T execute( + AttemptCall call, + Function> remoteFailure, + BooleanSupplier abandonedByCaller) { Objects.requireNonNull(remoteFailure, "remote failure classifier"); + Objects.requireNonNull(abandonedByCaller, "abandoned-by-caller signal"); if (!circuitBreaker.tryAcquirePermission()) { rejections.circuitOpen(); throw new HttpCircuitOpenException( @@ -110,7 +133,11 @@ public final class AttemptResiliencePipeline { return result; } catch (Throwable failure) { releaseAttemptPermits(); - circuitBreaker.onError(System.nanoTime() - started, failure); + if (abandonedByCaller.getAsBoolean()) { + circuitBreaker.releasePermission(); + } else { + circuitBreaker.onError(System.nanoTime() - started, failure); + } throw translate(failure); } } diff --git a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/Deadline.java b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/Deadline.java index 3ee96a73..f29f6c9a 100644 --- a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/Deadline.java +++ b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/Deadline.java @@ -10,11 +10,31 @@ import java.util.Objects; * *

    Everything — pool acquire, DNS, connect, TLS, request write, response read, and every retry * backoff — is spent inside this single budget. + * + * @param at the instant the call ends + * @param source who set that instant, which decides what an attempt it ends says about the upstream */ -public record Deadline(Instant at) { +public record Deadline(Instant at, Source source) { + + /** + * Who set the instant a call ends at. + * + *

    The two say different things when they end an attempt. The profile's total-call timeout is + * the operator's statement of how long a healthy upstream may take, so an attempt it ends is the + * upstream's failure. A caller's own deadline is a statement about the caller — it had only that + * much time left — so an attempt it ends says nothing about the upstream (review 2026-11-03 + * RR-04-01). + */ + public enum Source { + /** The profile's {@code timeout.total-call}. */ + PROFILE_TOTAL_CALL, + /** The caller's {@code HttpOperation.deadline()}, when it is earlier than the profile's. */ + CALLER + } public Deadline { Objects.requireNonNull(at, "deadline instant"); + Objects.requireNonNull(source, "deadline source"); } public Duration remaining(Clock clock) { @@ -26,6 +46,11 @@ public record Deadline(Instant at) { return !clock.instant().isBefore(at); } + /** Whether the instant is the caller's own budget rather than the profile's total-call. */ + public boolean setByCaller() { + return source == Source.CALLER; + } + /** The earlier of two deadlines; a parent budget can only ever shrink a child's. */ public Deadline earliest(Deadline other) { return at.isBefore(other.at) ? this : other; diff --git a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/DeadlineCalculator.java b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/DeadlineCalculator.java index 04e7d007..a356fb25 100644 --- a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/DeadlineCalculator.java +++ b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/resilience/DeadlineCalculator.java @@ -9,6 +9,9 @@ import java.util.Optional; /** * Derives the effective deadline from the caller's parent budget and the profile total-call timeout * (design §15.2): {@code effectiveDeadline = min(parentDeadline, now + totalCallTimeout)}. + * + *

    The result also records which of the two won. A tie goes to the profile: the profile's limit + * would have ended the call at that instant anyway. */ public final class DeadlineCalculator { @@ -17,6 +20,9 @@ public final class DeadlineCalculator { Objects.requireNonNull(totalCall, "total call timeout"); Objects.requireNonNull(clock, "clock"); Instant local = clock.instant().plus(totalCall); - return new Deadline(parent.map(value -> value.isBefore(local) ? value : local).orElse(local)); + return parent + .filter(value -> value.isBefore(local)) + .map(value -> new Deadline(value, Deadline.Source.CALLER)) + .orElseGet(() -> new Deadline(local, Deadline.Source.PROFILE_TOTAL_CALL)); } } diff --git a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/restclient/BlockingAttemptExecutor.java b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/restclient/BlockingAttemptExecutor.java index 3b6df73f..70c80312 100644 --- a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/restclient/BlockingAttemptExecutor.java +++ b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/restclient/BlockingAttemptExecutor.java @@ -63,6 +63,7 @@ public final class BlockingAttemptExecutor { try (AttemptDeadline.Scope scope = AttemptDeadline.open(deadline.remaining(support.clock()))) { return executeWithin( scope, + deadline, runtime, prepared, responseType, @@ -76,6 +77,7 @@ public final class BlockingAttemptExecutor { private AttemptOutcome executeWithin( AttemptDeadline.Scope scope, + Deadline deadline, BlockingClientRuntime runtime, PreparedOperation prepared, ResponseType responseType, @@ -146,7 +148,12 @@ public final class BlockingAttemptExecutor { tracker.firstByteDelivered()); } }, - outcome -> upstreamFailure(outcome, baseMetadata)); + outcome -> upstreamFailure(outcome, baseMetadata), + // The caller's own deadline ending the attempt is not the upstream failing. The + // profile's total-call ending it is: that limit is the operator's statement of + // how long a healthy upstream may take, and a slow-drip upstream that never trips + // a per-gap timeout is seen by the breaker only through it (RR-04-01). + () -> deadline.setByCaller() && scope.expired()); return finish(observation, scope, attemptOutcome); } catch (HttpClientException stable) { return finish( diff --git a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/restclient/BlockingStreamingGateway.java b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/restclient/BlockingStreamingGateway.java index 9fabdca1..4f21342e 100644 --- a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/restclient/BlockingStreamingGateway.java +++ b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/restclient/BlockingStreamingGateway.java @@ -5,6 +5,7 @@ import dev.caskeleton.adapter.outbound.httpclient.api.HttpStatus; import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpDeadlineExceededException; import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpFailureMetadata; import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpRemoteErrorException; +import dev.caskeleton.adapter.outbound.httpclient.api.operation.AttemptStage; import dev.caskeleton.adapter.outbound.httpclient.api.operation.ExecutionEvidence; import dev.caskeleton.adapter.outbound.httpclient.api.operation.HttpOperation; import dev.caskeleton.adapter.outbound.httpclient.api.result.BlockingStreamingResponse; @@ -78,9 +79,11 @@ public final class BlockingStreamingGateway { // which deregisters the transport's abort, so the body is then bounded by read-idle per gap // and a long download is not cut at an arbitrary total. Without the scope a setup was bounded // only by per-gap stage timeouts, and an upstream that held or dripped its headers kept the - // caller past its deadline (finding 05-06). + // caller past its deadline (finding 05-06). It is a setup scope, not an attempt scope: the + // JDK transport arms an exchange-wide timeout for an attempt, which cut every download + // longer than the response-header timeout (RR-04-03). AttemptDeadline.Scope setupDeadline = - AttemptDeadline.open(setup.deadline().remaining(runtime.support().clock())); + AttemptDeadline.openStreamSetup(setup.deadline().remaining(runtime.support().clock())); try (setupDeadline) { response = spec.exchange( @@ -102,9 +105,24 @@ public final class BlockingStreamingGateway { .withStatus(new HttpStatus(status)) .withEvidence(ExecutionEvidence.RESPONSE_RECEIVED)); } + // A body that fails after hand-over reaches the caller as the same stable + // exceptions a buffered call throws, classified by the transport's own + // classifier at the body stage — not as the engine's bare IOException, which + // carried no category, no evidence and nothing to log (RR-04-03). InputStream bounded = - new CountingBoundedInputStream( - rawResponse.getBody(), limiter::recordWireBytes); + new ClassifiedBodyInputStream( + new CountingBoundedInputStream( + rawResponse.getBody(), limiter::recordWireBytes), + bodyFailure -> + runtime + .support() + .exceptionMapper() + .map( + runtime + .failureClassifier() + .classify(bodyFailure, AttemptStage.RESPONSE_BODY), + metadata, + bodyFailure)); return new DefaultBlockingStreamingResponse( new HttpStatus(status), responseHeaders, bounded, rawResponse::close); }, diff --git a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/restclient/ClassifiedBodyInputStream.java b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/restclient/ClassifiedBodyInputStream.java new file mode 100644 index 00000000..f2597bda --- /dev/null +++ b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/restclient/ClassifiedBodyInputStream.java @@ -0,0 +1,64 @@ +package dev.caskeleton.adapter.outbound.httpclient.restclient; + +import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpClientException; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; +import java.util.function.Function; + +/** + * A handed-over response body whose read failures reach the caller as stable exceptions (review + * 2026-11-03 RR-04-03). + * + *

    A streamed body is read after the gateway has returned, so the gateway's own failure mapping + * never sees it. Without this a read-idle timeout or a cut connection reached the caller as the + * engine's bare {@code IOException} — no category, no evidence, nothing a retry policy or a log + * could branch on. Closing is the caller's own act and is passed through unclassified. + */ +final class ClassifiedBodyInputStream extends FilterInputStream { + + private final Function classifier; + + ClassifiedBodyInputStream( + InputStream delegate, Function classifier) { + super(Objects.requireNonNull(delegate, "body stream")); + this.classifier = Objects.requireNonNull(classifier, "body failure classifier"); + } + + @Override + public int read() throws IOException { + try { + return super.read(); + } catch (IOException failure) { + throw classifier.apply(failure); + } + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + try { + return super.read(buffer, offset, length); + } catch (IOException failure) { + throw classifier.apply(failure); + } + } + + @Override + public long skip(long count) throws IOException { + try { + return super.skip(count); + } catch (IOException failure) { + throw classifier.apply(failure); + } + } + + @Override + public int available() throws IOException { + try { + return super.available(); + } catch (IOException failure) { + throw classifier.apply(failure); + } + } +} diff --git a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/transport/AttemptDeadline.java b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/transport/AttemptDeadline.java index bde2d067..36d072bb 100644 --- a/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/transport/AttemptDeadline.java +++ b/src/adapter/outbound/httpclient/src/main/java/dev/caskeleton/adapter/outbound/httpclient/transport/AttemptDeadline.java @@ -28,6 +28,11 @@ import java.util.concurrent.TimeUnit; *

    The scope is thread-confined because a blocking attempt runs on the caller's thread from * request creation to the last body byte. When no scope is open — a transport used outside the * gateway — both methods leave the transport's own behaviour unchanged. + * + *

    A streaming download opens a different scope ({@link #openStreamSetup(Duration)}): it bounds + * the exchange only up to the response headers and is closed when the body is handed over. A + * transport whose engine timeout would otherwise keep running over the body reads {@link + * #boundsSetupOnly()} and keeps that timeout off it (review 2026-11-03 RR-04-03). */ public final class AttemptDeadline { @@ -41,8 +46,34 @@ public final class AttemptDeadline { * @param remaining the time left before the logical call's effective deadline */ public static Scope open(Duration remaining) { + return open(remaining, false); + } + + /** + * Opens the deadline scope for a streaming download's setup on the current thread. + * + *

    The caller closes it when the response is handed over, so the deadline never reaches the + * body: a download may legitimately outlast any fixed total (design §15.3). + * + * @param remaining the time left before the logical call's effective deadline + */ + public static Scope openStreamSetup(Duration remaining) { + return open(remaining, true); + } + + /** + * Whether the current scope bounds only a streaming setup, so a timeout the transport arms must + * not outlive the response headers. + */ + public static boolean boundsSetupOnly() { + Scope scope = CURRENT.get(); + return scope != null && scope.setupOnly; + } + + private static Scope open(Duration remaining, boolean setupOnly) { Objects.requireNonNull(remaining, "remaining deadline"); - Scope scope = new Scope(remaining.isNegative() ? Duration.ZERO : remaining, CURRENT.get()); + Scope scope = + new Scope(remaining.isNegative() ? Duration.ZERO : remaining, setupOnly, CURRENT.get()); CURRENT.set(scope); return scope; } @@ -78,6 +109,7 @@ public final class AttemptDeadline { public static final class Scope implements AutoCloseable { private final long endNanos; + private final boolean setupOnly; private final Scope outer; private final CompletableFuture timer = new CompletableFuture<>(); private final CompletableFuture expiry; @@ -85,8 +117,9 @@ public final class AttemptDeadline { private boolean fired; private boolean closed; - private Scope(Duration remaining, Scope outer) { + private Scope(Duration remaining, boolean setupOnly, Scope outer) { this.endNanos = System.nanoTime() + remaining.toNanos(); + this.setupOnly = setupOnly; this.outer = outer; // The JDK's shared delay scheduler, the same one Spring's JDK request uses for its timeout. // Completing the future on close() also cancels the scheduled task, so a finished attempt diff --git a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/resilience/DeadlineCalculatorTest.java b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/resilience/DeadlineCalculatorTest.java index 143e9f13..6d54534f 100644 --- a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/resilience/DeadlineCalculatorTest.java +++ b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/resilience/DeadlineCalculatorTest.java @@ -34,7 +34,8 @@ class DeadlineCalculatorTest { @Test void refusesAttemptWhenBackoffConsumesRemainingBudget() { - Deadline deadline = new Deadline(Instant.parse("2026-08-08T00:00:01Z")); + Deadline deadline = + new Deadline(Instant.parse("2026-08-08T00:00:01Z"), Deadline.Source.PROFILE_TOTAL_CALL); Optional result = new AttemptBudgetCalculator(clock) .nextAttempt( @@ -44,7 +45,8 @@ class DeadlineCalculatorTest { @Test void grantsAttemptWhenBudgetCoversBackoffMinimumAndCleanup() { - Deadline deadline = new Deadline(Instant.parse("2026-08-08T00:00:02Z")); + Deadline deadline = + new Deadline(Instant.parse("2026-08-08T00:00:02Z"), Deadline.Source.PROFILE_TOTAL_CALL); Optional result = new AttemptBudgetCalculator(clock) .nextAttempt( @@ -60,7 +62,8 @@ class DeadlineCalculatorTest { assertThatThrownBy( () -> guard.requireTimeRemaining( - new Deadline(Instant.parse("2026-08-07T23:59:59Z")), + new Deadline( + Instant.parse("2026-08-07T23:59:59Z"), Deadline.Source.PROFILE_TOTAL_CALL), CoreFixtures.notSentMetadata())) .isInstanceOf(HttpDeadlineExceededException.class); assertThatThrownBy( diff --git a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/resilience/RetryCoordinatorTest.java b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/resilience/RetryCoordinatorTest.java index 82b1e4b4..d36ab666 100644 --- a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/resilience/RetryCoordinatorTest.java +++ b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/resilience/RetryCoordinatorTest.java @@ -147,7 +147,7 @@ class RetryCoordinatorTest { @Override public Deadline deadline() { - return new Deadline(clock.instant().plusSeconds(5)); + return new Deadline(clock.instant().plusSeconds(5), Deadline.Source.PROFILE_TOTAL_CALL); } @Override @@ -202,7 +202,7 @@ class RetryCoordinatorTest { @Override public Deadline deadline() { - return new Deadline(clock.instant().plusSeconds(5)); + return new Deadline(clock.instant().plusSeconds(5), Deadline.Source.PROFILE_TOTAL_CALL); } @Override diff --git a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/CircuitBreakerStatusAccountingTest.java b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/CircuitBreakerStatusAccountingTest.java index adb33f86..95ca1839 100644 --- a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/CircuitBreakerStatusAccountingTest.java +++ b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/CircuitBreakerStatusAccountingTest.java @@ -5,20 +5,28 @@ import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.httpclient.api.OperationName; import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpCircuitOpenException; +import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpDeadlineExceededException; import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpRemoteErrorException; import dev.caskeleton.adapter.outbound.httpclient.api.operation.HttpOperation; import dev.caskeleton.adapter.outbound.httpclient.api.result.HttpCallResult; import dev.caskeleton.adapter.outbound.httpclient.api.result.ResponseType; import dev.caskeleton.adapter.outbound.httpclient.profile.CircuitBreakerSettings; +import dev.caskeleton.adapter.outbound.httpclient.profile.TimeoutSettings; +import dev.caskeleton.adapter.outbound.httpclient.profile.TransportType; import dev.caskeleton.adapter.outbound.httpclient.testkit.ClientProfiles; import dev.caskeleton.adapter.outbound.httpclient.testkit.MockHttpServer; import dev.caskeleton.adapter.outbound.httpclient.testkit.TestGateways; +import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; /** * What the per-profile circuit breaker counts as an upstream failure, end to end (finding 05-05). @@ -31,6 +39,10 @@ class CircuitBreakerStatusAccountingTest { private static final int CALLS = 200; private static final int LIBRARY_DEFAULT_WINDOW = 100; private static final int SMALL_WINDOW = 5; + private static final int TINY_WINDOW = 2; + private static final Duration HEALTHY_ANSWER_TIME = Duration.ofMillis(600); + private static final Duration CALLER_BUDGET = Duration.ofMillis(200); + private static final Duration PROFILE_TOTAL_CALL = Duration.ofSeconds(1); @Test @DisplayName("정상 응답인 404가 200번 이어져도 circuit이 열리지 않고 모든 호출이 상대 서버에 닿는다") @@ -158,6 +170,113 @@ class CircuitBreakerStatusAccountingTest { } } + @ParameterizedTest(name = "{0}") + @EnumSource( + value = TransportType.class, + names = {"APACHE", "JDK"}) + @DisplayName("호출자 deadline으로 끊은 시도는 상대 장애로 세지 않아 창을 채워도 다음 호출이 상대 서버에 닿는다") + void attemptsCutShortByTheCallersOwnDeadlineDoNotOpenTheCircuit(TransportType transport) + throws Exception { + try (MockHttpServer server = MockHttpServer.start(); + TestGateways.Harness harness = + TestGateways.forProfile( + ClientProfiles.builder("caller-deadline") + .baseUrl(server.uri("/")) + .transport(transport) + .circuitBreaker( + new CircuitBreakerSettings( + 50, SMALL_WINDOW, SMALL_WINDOW, Duration.ofMinutes(1), 1)) + .build())) { + // given + for (int i = 0; i <= SMALL_WINDOW; i++) { + server.enqueueDelayedHeaders( + 200, + "application/json", + "\"ok\"".getBytes(StandardCharsets.UTF_8), + HEALTHY_ANSWER_TIME); + } + List windowFailures = new ArrayList<>(); + for (int i = 0; i < SMALL_WINDOW; i++) { + windowFailures.add( + catchThrowable( + () -> getUserWithin(harness, Optional.of(Instant.now().plus(CALLER_BUDGET))))); + } + assertThat(windowFailures) + .allSatisfy(f -> assertThat(f).isInstanceOf(HttpDeadlineExceededException.class)); + + // when + Throwable next = catchThrowable(() -> getUserWithin(harness, Optional.empty())); + + // then + assertThat(next).isNull(); + assertThat(server.requestCount()).isEqualTo(SMALL_WINDOW + 1); + } + } + + @ParameterizedTest(name = "{0}") + @EnumSource( + value = TransportType.class, + names = {"APACHE", "JDK"}) + @DisplayName("프로필 total-call 시간으로 끊긴 시도는 상대 장애로 세어 창을 채우면 circuit이 열린다") + void attemptsCutShortByTheProfileTotalCallStillOpenTheCircuit(TransportType transport) + throws Exception { + TimeoutSettings base = ClientProfiles.timeouts(); + try (MockHttpServer server = MockHttpServer.start(); + TestGateways.Harness harness = + TestGateways.forProfile( + ClientProfiles.builder("profile-deadline") + .baseUrl(server.uri("/")) + .transport(transport) + .timeout( + new TimeoutSettings( + base.dns(), + base.connect(), + base.tlsHandshake(), + base.responseHeader(), + base.readIdle(), + PROFILE_TOTAL_CALL)) + .circuitBreaker( + new CircuitBreakerSettings( + 50, TINY_WINDOW, TINY_WINDOW, Duration.ofMinutes(1), 1)) + .build())) { + // given + List windowFailures = new ArrayList<>(); + for (int i = 0; i < TINY_WINDOW; i++) { + server.enqueueSlowDripBody(200, 100, Duration.ofMillis(50)); + windowFailures.add(catchThrowable(() -> getUserWithin(harness, Optional.empty()))); + } + assertThat(windowFailures) + .allSatisfy(f -> assertThat(f).isInstanceOf(HttpDeadlineExceededException.class)); + + // when + Throwable next = catchThrowable(() -> getUserWithin(harness, Optional.empty())); + + // then + assertThat(next).isInstanceOf(HttpCircuitOpenException.class); + assertThat(server.requestCount()).isEqualTo(TINY_WINDOW); + } + } + + private static HttpCallResult getUserWithin( + TestGateways.Harness harness, Optional deadline) { + HttpOperation get = HttpOperation.get(new OperationName("get-user"), "/users/1", Map.of()); + return harness + .gateway() + .exchange( + harness.profile().name(), + new HttpOperation( + get.operationName(), + get.method(), + get.uriTemplate(), + get.uriVariables(), + get.headers(), + get.body(), + get.idempotency(), + get.idempotencyKey(), + deadline), + ResponseType.of(String.class)); + } + private static HttpCallResult getUserAsResult(TestGateways.Harness harness) { return harness .gateway() diff --git a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/StreamingBodyLifetimeTest.java b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/StreamingBodyLifetimeTest.java new file mode 100644 index 00000000..a19d3c6f --- /dev/null +++ b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/restclient/StreamingBodyLifetimeTest.java @@ -0,0 +1,124 @@ +package dev.caskeleton.adapter.outbound.httpclient.restclient; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.adapter.outbound.httpclient.api.OperationName; +import dev.caskeleton.adapter.outbound.httpclient.api.error.HttpResponseTimeoutException; +import dev.caskeleton.adapter.outbound.httpclient.api.operation.AttemptStage; +import dev.caskeleton.adapter.outbound.httpclient.api.operation.ExecutionEvidence; +import dev.caskeleton.adapter.outbound.httpclient.api.operation.HttpOperation; +import dev.caskeleton.adapter.outbound.httpclient.api.result.BlockingStreamingResponse; +import dev.caskeleton.adapter.outbound.httpclient.profile.ClientProfile; +import dev.caskeleton.adapter.outbound.httpclient.profile.ResponseLimits; +import dev.caskeleton.adapter.outbound.httpclient.profile.TimeoutSettings; +import dev.caskeleton.adapter.outbound.httpclient.profile.TransportType; +import dev.caskeleton.adapter.outbound.httpclient.testkit.ClientProfiles; +import dev.caskeleton.adapter.outbound.httpclient.testkit.MockHttpServer; +import dev.caskeleton.adapter.outbound.httpclient.testkit.TestGateways; +import java.time.Duration; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +/** + * What bounds a streaming download's body once it has been handed over (review 2026-11-03 RR-04-03, + * design §15.3). + * + *

    The documented contract is transport-independent: the total budget and the response-header + * timeout bound the setup, and after hand-over each gap of the body is bounded by {@code read-idle} + * alone, so a long download is not cut at an arbitrary total. The JDK transport used to break it: + * Spring's JDK request enforces its read timeout over the whole exchange and closes the body stream + * when it fires, so every JDK download longer than the response-header timeout was cut, and the + * caller got an unclassified {@code IOException: closed}. + * + *

    The profile's response-header timeout (500 ms) and total budget (1 s) are both shorter than + * the body (about 2 s), and the read-idle timeout (1 s) is longer than every gap of it. + */ +class StreamingBodyLifetimeTest { + + private static final int BODY_BYTES = 20; + private static final Duration DRIP_GAP = Duration.ofMillis(100); + private static final Duration STALL_GAP = Duration.ofMillis(2500); + + @ParameterizedTest(name = "{0}") + @EnumSource( + value = TransportType.class, + names = {"APACHE", "JDK"}) + @DisplayName("넘겨받은 스트리밍 본문은 응답 헤더 시간과 전체 예산을 넘겨도 끝까지 읽힌다") + void aStreamedBodyOutlivesTheResponseHeaderTimeoutAndTheTotalBudget(TransportType transport) + throws Exception { + try (MockHttpServer server = MockHttpServer.start(); + TestGateways.Harness harness = TestGateways.forProfile(profile(server, transport))) { + // given + server.enqueueSlowDripBody(200, BODY_BYTES, DRIP_GAP); + BlockingStreamingGateway gateway = new BlockingStreamingGateway(harness.registry()); + + // when + byte[] body; + try (BlockingStreamingResponse response = gateway.download(harness.profile().name(), get())) { + body = response.body().readAllBytes(); + } + + // then + assertThat(body).hasSize(BODY_BYTES); + } + } + + @ParameterizedTest(name = "{0}") + @EnumSource( + value = TransportType.class, + names = {"APACHE", "JDK"}) + @DisplayName("본문 사이 공백이 read-idle을 넘으면 분류된 응답 시간 초과로 끝난다") + void aBodyGapLongerThanReadIdleEndsWithAClassifiedTimeout(TransportType transport) + throws Exception { + try (MockHttpServer server = MockHttpServer.start(); + TestGateways.Harness harness = TestGateways.forProfile(profile(server, transport))) { + // given + server.enqueueSlowDripBody(200, BODY_BYTES, STALL_GAP); + BlockingStreamingGateway gateway = new BlockingStreamingGateway(harness.registry()); + long started = System.nanoTime(); + + // when + Throwable failure = + catchThrowable( + () -> { + try (BlockingStreamingResponse response = + gateway.download(harness.profile().name(), get())) { + response.body().readAllBytes(); + } + }); + + // then + Duration elapsed = Duration.ofNanos(System.nanoTime() - started); + assertThat(failure).isInstanceOf(HttpResponseTimeoutException.class); + assertThat(elapsed).isLessThan(STALL_GAP); + HttpResponseTimeoutException timeout = (HttpResponseTimeoutException) failure; + assertThat(timeout.metadata().stage()).isEqualTo(AttemptStage.RESPONSE_BODY); + assertThat(timeout.metadata().evidence()).isEqualTo(ExecutionEvidence.PARTIAL_RESPONSE); + } + } + + private static HttpOperation get() { + return HttpOperation.get(new OperationName("download"), "/download", Map.of()); + } + + private static ClientProfile profile(MockHttpServer server, TransportType transport) { + TimeoutSettings base = ClientProfiles.timeouts(); + return ClientProfiles.builder("streaming-body") + .baseUrl(server.uri("/")) + .transport(transport) + .timeout( + new TimeoutSettings( + base.dns(), + base.connect(), + base.tlsHandshake(), + Duration.ofMillis(500), + Duration.ofSeconds(1), + Duration.ofSeconds(1))) + .response(new ResponseLimits(1024, 1024, Set.of("text/plain"))) + .build(); + } +} diff --git a/src/adapter/outbound/httpclient/src/testFixtures/java/dev/caskeleton/adapter/outbound/httpclient/testkit/RecordingResilienceComponents.java b/src/adapter/outbound/httpclient/src/testFixtures/java/dev/caskeleton/adapter/outbound/httpclient/testkit/RecordingResilienceComponents.java index d013194b..0c8f600d 100644 --- a/src/adapter/outbound/httpclient/src/testFixtures/java/dev/caskeleton/adapter/outbound/httpclient/testkit/RecordingResilienceComponents.java +++ b/src/adapter/outbound/httpclient/src/testFixtures/java/dev/caskeleton/adapter/outbound/httpclient/testkit/RecordingResilienceComponents.java @@ -68,6 +68,11 @@ public final class RecordingResilienceComponents { events.add("circuit-error"); } + @Override + public void releasePermission() { + events.add("circuit-release"); + } + @Override public String state() { return circuitOpen ? "OPEN" : "CLOSED"; diff --git a/src/adapter/outbound/identifier/CLAUDE.md b/src/adapter/outbound/identifier/CLAUDE.md index 0c1a7c11..85a1c08c 100644 --- a/src/adapter/outbound/identifier/CLAUDE.md +++ b/src/adapter/outbound/identifier/CLAUDE.md @@ -20,16 +20,22 @@ build choices) live in [README.md](README.md). - `UuidV7DurableOperationIdFactory` — `DurableOperationIdFactory` implementation (UUIDv7 via `uuid-creator`). The operation id is also the submission event's id, and the messaging platform's message id accepts only UUIDv7 (review 2026-11-01 06-06). Wired by `composition:durable-operation`. -- The leaf name says "identifier"; general identifier generation, clock and random sources are not here. +- The leaf name says "identifier"; general-purpose clock and random sources are not here. The name is kept because Gradle paths and consumers use it (README, review 05-21). - Wiring is `composition/service-runtime` (`PseudonymizationConfig`), not this leaf. ## Allowed -- `:application-core` (declared in [build.gradle](build.gradle)); the registry entry is the SSOT. +Declared in [build.gradle](build.gradle); the registry entry is the SSOT. + +- `:application-core` — the ports both classes implement. - `:shared-contract`, for `HmacSha256` only (review 2026-11-01 01-11). The pseudonym is the hex HMAC of the principal; `HmacUserPrincipalPseudonymizerVectorTest` pins it to a vector computed outside the JVM, because logs written before and after a release are joined on it. +- `:domain-core` — a feature's server-assigned id adapter implements its domain `*IdFactory` port + here (registry edge `adapter:outbound:identifier -> domain-core`; `docs/manual/02-new-feature.md`). +- `com.github.f4b6a3:uuid-creator` (`libs.uuid.creator`) — UUIDv7 for + `UuidV7DurableOperationIdFactory`, the generator the sample's id adapters use. ## Forbidden @@ -42,8 +48,9 @@ build choices) live in [README.md](README.md). ## Test -Pure unit tests, no Spring context (`HmacUserPrincipalPseudonymizerTest`, including the RFC 4231 -known-answer vector). +Pure unit tests, no Spring context: `HmacUserPrincipalPseudonymizerTest` (including the RFC 4231 +known-answer vector), `HmacUserPrincipalPseudonymizerVectorTest` and +`UuidV7DurableOperationIdFactoryTest`. ```bash cd src diff --git a/src/adapter/outbound/identifier/README.md b/src/adapter/outbound/identifier/README.md index f63eca2f..f740eb57 100644 --- a/src/adapter/outbound/identifier/README.md +++ b/src/adapter/outbound/identifier/README.md @@ -9,7 +9,8 @@ ## 이름과 내용이 다른 이유 (review 05-21) leaf 이름은 `identifier`지만 범용 식별자 생성기·clock·random 소스는 여기 없다. 식별자 생성은 -durable operation id(UUIDv7) 하나뿐이다(아래). 예전 문서가 범용 생성기 계획을 현재 내용처럼 적었다. 이름은 바꾸지 않았다: Gradle 경로·registry id·소비자 +durable operation id(UUIDv7) 하나뿐이고(아래), 기능이 서버에서 id를 정하면 그 기능의 domain +`*IdFactory` 구현도 여기 둔다(`:domain-core` 의존이 그 때문이다). 예전 문서가 범용 생성기 계획을 현재 내용처럼 적었다. 이름은 바꾸지 않았다: Gradle 경로·registry id·소비자 (`composition:service-runtime`, `app-bootstrap`, `sample-portfolio`)의 의존 선언이 모두 이 이름을 쓰고, 바꿔서 얻는 것은 이름 하나뿐이다. 외부 연동이 없는 crypto 능력을 새로 둘 곳이 필요하면 이 leaf에 둔다. 그때 이 절을 고친다. @@ -25,10 +26,10 @@ leaf에 둔다. 그때 이 절을 고친다. 대소문자를 바꾸면 테스트가 실패한다. - Salt: `APP_PRIVACY_PSEUDONYMIZATION_SALT`(secret, 90일 회전)에서 온다. 이 클래스는 salt를 스스로 읽지 않고 생성자로 받는다. 받은 배열은 복사해 둔다. -- Thread safety: `Mac`은 thread-safe하지 않아서 호출마다 새로 만든다. 그래서 싱글톤 bean으로 - 공유해도 된다. `HmacSHA256`은 JDK 필수 알고리즘이라 `NoSuchAlgorithmException`· - `InvalidKeyException`은 사실상 나지 않는다. checked exception을 호출부에 남기지 않으려고 - `IllegalStateException`으로 감싼다. +- 서명: 이 클래스는 `Mac`을 직접 다루지 않는다. shared-contract의 `HmacSha256.sign`에 맡긴다(리뷰 + 2026-11-01 01-11). `HmacSha256`이 호출마다 새 `Mac`을 만들고(`Mac`은 thread-safe하지 않다), + JDK 필수 알고리즘이라 사실상 나지 않는 checked exception을 `IllegalStateException`으로 감싼다. + 그래서 이 클래스는 싱글톤 bean으로 공유해도 된다. ## UuidV7DurableOperationIdFactory @@ -46,6 +47,10 @@ Spring 어노테이션이 없다. bean은 ## 빌드 결정 +의존은 넷이다: `:application-core`(두 클래스가 구현하는 port), `:shared-contract`(`HmacSha256`), +`:domain-core`(기능의 `*IdFactory` 구현 자리), `uuid-creator`(UUIDv7). 목록과 이유는 +[CLAUDE.md](CLAUDE.md) Allowed 절이 기준이다. + `:application-core`를 `implementation`으로 받는다. 이 leaf의 public 타입 시그니처에 application-core 타입이 드러나지만, 조립하는 쪽이 application-core를 이미 classpath에 갖고 있다. 허용 edge는 `src/config/architecture/modules.json`이 기준이다. diff --git a/src/adapter/outbound/identifier/src/main/java/dev/caskeleton/adapter/outbound/identifier/package-info.java b/src/adapter/outbound/identifier/src/main/java/dev/caskeleton/adapter/outbound/identifier/package-info.java index 5f6c7d82..481742d4 100644 --- a/src/adapter/outbound/identifier/src/main/java/dev/caskeleton/adapter/outbound/identifier/package-info.java +++ b/src/adapter/outbound/identifier/src/main/java/dev/caskeleton/adapter/outbound/identifier/package-info.java @@ -1,5 +1,7 @@ /** - * The HMAC-SHA-256 user-principal pseudonymizer. Despite the leaf name, no identifier generation, - * clock or random source lives here; see README for why the name was kept. + * Two identifier adapters with no external IO: the HMAC-SHA-256 user-principal pseudonymizer and + * the UUIDv7 durable operation id factory. A feature's server-assigned id adapter for a domain + * {@code *IdFactory} port belongs here too. General clock and random sources do not live here; see + * README for why the leaf name was kept. */ package dev.caskeleton.adapter.outbound.identifier; diff --git a/src/adapter/outbound/messaging-kafka/CLAUDE.md b/src/adapter/outbound/messaging-kafka/CLAUDE.md new file mode 100644 index 00000000..293b83c7 --- /dev/null +++ b/src/adapter/outbound/messaging-kafka/CLAUDE.md @@ -0,0 +1,14 @@ +# adapter:outbound:messaging-kafka + +- Module ID `adapter-outbound-messaging-kafka`, Gradle path `:adapter:outbound:messaging-kafka`. + Registry SSOT: `src/config/architecture/modules.json`. +- Responsibility: the Kafka publishing capability as one `runtimeOnly` line. It has no production + code: it bundles `adapter:outbound:messaging` (the application's bridge) with + `messaging:messaging-spring-boot-starter-kafka` (the vendored platform and its Kafka transport) and + ships the module defaults in `config/module-defaults/messaging.yml`. +- Boundaries: add no code here. A publish rule belongs to the bridge (`adapter:outbound:messaging`), + a Kafka rule to the vendored product (`src/messaging/CLAUDE.md`); a class here would be a third + place. Allowed edges come only from the registry entry. +- Tests: activation tests (`./gradlew :adapter:outbound:messaging-kafka:test`) and the real-Kafka + lane `messagingKafkaRelayIntegrationTest` (relay → bridge → broker record bytes; `:integrationCheck`). +- Design: `README.md`. diff --git a/src/adapter/outbound/messaging-kafka/src/integrationTest/java/dev/caskeleton/adapter/outbound/messaging/kafka/relay/ApplicationRelayKafkaIntegrationTest.java b/src/adapter/outbound/messaging-kafka/src/integrationTest/java/dev/caskeleton/adapter/outbound/messaging/kafka/relay/ApplicationRelayKafkaIntegrationTest.java index bd113251..2957e18d 100644 --- a/src/adapter/outbound/messaging-kafka/src/integrationTest/java/dev/caskeleton/adapter/outbound/messaging/kafka/relay/ApplicationRelayKafkaIntegrationTest.java +++ b/src/adapter/outbound/messaging-kafka/src/integrationTest/java/dev/caskeleton/adapter/outbound/messaging/kafka/relay/ApplicationRelayKafkaIntegrationTest.java @@ -8,6 +8,7 @@ import dev.caskeleton.application.outbox.OutboxAppendPort; import dev.caskeleton.application.outbox.OutboxCompletion; import dev.caskeleton.application.outbox.OutboxEventStatus; import dev.caskeleton.application.outbox.OutboxLease; +import dev.caskeleton.application.outbox.OutboxLongRetries; import dev.caskeleton.application.outbox.OutboxStorePort; import dev.caskeleton.application.outbox.testkit.InMemoryOutboxStore; import dev.caskeleton.application.outbox.testkit.OutboxEvents; @@ -282,13 +283,15 @@ class ApplicationRelayKafkaIntegrationTest { } @Override - public synchronized OutboxCompletion markFailed(OutboxLease lease, Instant nextAttemptAt) { - return delegate.markFailed(lease, nextAttemptAt); + public synchronized OutboxCompletion markFailed( + OutboxLease lease, Instant nextAttemptAt, String failureCode) { + return delegate.markFailed(lease, nextAttemptAt, failureCode); } @Override - public synchronized OutboxCompletion markDeferred(OutboxLease lease, Instant nextAttemptAt) { - return delegate.markDeferred(lease, nextAttemptAt); + public synchronized OutboxCompletion markDeferred( + OutboxLease lease, Instant nextAttemptAt, String failureCode) { + return delegate.markDeferred(lease, nextAttemptAt, failureCode); } @Override @@ -297,8 +300,13 @@ class ApplicationRelayKafkaIntegrationTest { } @Override - public synchronized OutboxCompletion markDead(OutboxLease lease) { - return delegate.markDead(lease); + public synchronized OutboxCompletion markDead(OutboxLease lease, String failureCode) { + return delegate.markDead(lease, failureCode); + } + + @Override + public synchronized OutboxLongRetries longRetries(int minDeferredClaims, Instant now) { + return delegate.longRetries(minDeferredClaims, now); } @Override diff --git a/src/adapter/outbound/messaging/CLAUDE.md b/src/adapter/outbound/messaging/CLAUDE.md index 025499f2..6cdb166a 100644 --- a/src/adapter/outbound/messaging/CLAUDE.md +++ b/src/adapter/outbound/messaging/CLAUDE.md @@ -5,7 +5,7 @@ - Module ID: `adapter-outbound-messaging` - Gradle path: `:adapter:outbound:messaging` - Focused test (derived from Gradle path): `./gradlew :adapter:outbound:messaging:test --console=plain` -- Runtime baseline: Java 21; repository framework baseline: Spring Boot 4.0.0. +- Runtime baseline: Java 21; repository framework baseline: Spring Boot 4.0.8. - Registry SSOT: `src/config/architecture/modules.json`. Package root: `dev.caskeleton.adapter.outbound.messaging`. diff --git a/src/adapter/outbound/messaging/README.md b/src/adapter/outbound/messaging/README.md index 63e6ff03..10323d26 100644 --- a/src/adapter/outbound/messaging/README.md +++ b/src/adapter/outbound/messaging/README.md @@ -42,13 +42,33 @@ platform `REJECTED` 중 `FailureDescriptor.retryable()`이 true인 것(종료 멈춘다(2026-11-01 리뷰 06-01). `AMBIGUOUS`로 번역하지 않은 이유: 동작은 같지만 "전송됐을 수 있음"이라는 거짓 정보가 남아 중복 경고를 오염시킨다. +### 실패 원인 코드 + +결과(outcome)는 relay가 무엇을 할지만 정한다. 종료 중 거부, 닫힌 transport, 거부된 credential은 모두 +`REJECTED_RETRYABLE` 하나다. 그런데 이 셋은 끝이 없는 재시도다(시도 예산을 쓰지 않는다). 그래서 bridge는 +platform `FailureDescriptor.code()`를 버리지 않고 `OutboxPublishAttempt.failureCode()`로 함께 넘긴다 +(재리뷰 2026-11-03 RR-05-03). relay는 그 코드를 행의 `last_failure_code`(V22)와 실패 로그의 +`error.reason`에 남긴다. 운영자는 이 값으로 "credential이 진짜 틀렸다"와 "broker가 이 레코드만 받지 +않는다"를 가른다(`docs/runbooks/outbox-publish-failed.md` §4.1). + +- platform code는 이미 정제된 이름이다(`KAFKA_AUTHENTICATIONEXCEPTION`, `PUBLISH_DEADLINE_EXCEEDED` 등). + driver 메시지가 아니므로 그대로 옮긴다. application 코드 문법(`[A-Za-z0-9][A-Za-z0-9._:-]*`, 120자)에 + 맞지 않는 code는 그 실패의 `FailureCategory` 이름으로 옮긴다. +- platform으로 옮길 수 없는 event(아래 id 규칙)는 bridge가 `EVENT_NOT_REPRESENTABLE`로 거부한다. +- 버린 대안: relay가 platform 타입을 직접 보는 것(경계가 깨진다), outcome 값을 원인별로 늘리는 것(relay가 + 할 일은 같은데 분기만 는다). + ### event id → message id application `EventId`는 문자열이고 platform `MessageId`는 **UUIDv7**만 받는다. bridge는 event id가 정규 소문자 UUIDv7 문자열일 때만 같은 값으로 옮기고, 그 밖(대문자·축약 표기·v4 등 다른 버전)은 다른 문자열의 message id로 바꿔 보내지 않고 `REJECTED_BEFORE_SEND`로 거부한다(relay는 DEAD). 소비자 중복 -제거 키가 event id라서 값을 바꿔 보내면 중복 제거가 깨지기 때문이다(리뷰 2026-11-01 06-06). 이 거부는 -append 시점(`EventId` 문법)에서 먼저 일어나야 한다 — application 쪽 문법 일원화는 별도 작업이다. +제거 키가 event id라서 값을 바꿔 보내면 중복 제거가 깨지기 때문이다(리뷰 2026-11-01 06-06). + +같은 규칙이 append 쪽에 먼저 있다. `IntegrationEventDraft`가 정규 소문자 UUIDv7이 아닌 event id와 +causation id를 거부하므로(`IntegrationIdentity.isCanonicalUuidV7`), 그런 id는 업무 트랜잭션 안에서 실패한다 +(재리뷰 2026-11-03 RR-05-04). bridge의 거부는 이 규칙 전에 쓰인 행(저장 문법 `EventId`는 그대로 넓다)을 위한 +마지막 방어선이다. ## outbox 발행 포트에는 대역이 없다 @@ -70,9 +90,10 @@ event가 다시 publish될 수 있다. 같은 event id로 다시 나가므로 co 있으면 `messaging-platform`이다. 이 adapter는 confirmed FAILED/DEAD report 하나를 SLF4J 2 fluent ERROR 하나로 렌더링한다. -공통 field는 `error.code`, `error.category`, `dependency_name`, +공통 field는 `error.code`, `error.category`, `error.cause_type`, `error.reason`, `dependency_name`, `dependency_type=messaging`, `outcome`, `event_id`, `event_type`, `aggregate_id`, `correlation_id`, `attempt_count`, `runbook_link`다. retry report만 `next_attempt_at`을 추가한다. +`error.reason`은 위 실패 원인 코드다. payload/envelope/exception-derived field는 받거나 렌더링하지 않는다. logging 내부 `RuntimeException`은 adapter와 use case 양쪽에서 방어하므로 persisted FAILED/DEAD outcome을 바꾸지 않는다. @@ -85,10 +106,8 @@ payload/envelope/exception-derived field는 받거나 렌더링하지 않는다. `envelope`·`realtime`)은 2026-09-25에 삭제됐고 복구 경로는 `docs/capability-status.md` 제거 표에 있다. -공통 build evidence manifest 검증은 build-tools로 옮겼다. 루트 `verifyMessagingContracts` -(`VerifyMessagingEvidenceTask`)가 application-core·shared-contract 두 qualification 레인 결과로 -`build/messaging-evidence/contracts-schema/manifest.json`을 쓰고, 쓴 바이트를 바로 common schema로 -검증한다. 이 leaf는 그 검증에 관여하지 않는다. +integration event 계약 검증은 루트 `verifyMessagingContracts`(`src/build.gradle`)가 한다. application-core· +shared-contract·sample-portfolio의 qualification lane 셋을 묶은 task다. 이 leaf는 그 검증에 관여하지 않는다. canonical path는 platform `PublishResult`의 confirmation/transmission evidence를 application -`OutboxPublishOutcome`으로 명시적으로 변환한다. +`OutboxPublishOutcome`으로 명시적으로 변환하고, 실패 code를 `OutboxPublishAttempt`에 함께 싣는다. diff --git a/src/adapter/outbound/messaging/src/main/java/dev/caskeleton/adapter/outbound/messaging/outbox/OutboxMessagePublishAdapter.java b/src/adapter/outbound/messaging/src/main/java/dev/caskeleton/adapter/outbound/messaging/outbox/OutboxMessagePublishAdapter.java index 91528ef2..3a7d209c 100644 --- a/src/adapter/outbound/messaging/src/main/java/dev/caskeleton/adapter/outbound/messaging/outbox/OutboxMessagePublishAdapter.java +++ b/src/adapter/outbound/messaging/src/main/java/dev/caskeleton/adapter/outbound/messaging/outbox/OutboxMessagePublishAdapter.java @@ -4,17 +4,21 @@ import dev.caskeleton.application.messaging.event.IntegrationEventPublishPort; import dev.caskeleton.application.outbox.ClaimedOutboxEvent; import dev.caskeleton.application.outbox.OutboxMessagePublishPort; import dev.caskeleton.application.outbox.OutboxPublishAmbiguousException; +import dev.caskeleton.application.outbox.OutboxPublishAttempt; import dev.caskeleton.application.outbox.OutboxPublishOutcome; import dev.caskeleton.application.outbox.OutboxPublishRefusedException; +import java.time.Duration; import java.util.Objects; +import java.util.Optional; /** * The relay's publish port: a claimed row's validated event, delegated unchanged to the canonical * {@link IntegrationEventPublishPort}. * *

    The event is the one the append validated, restored from its exact envelope bytes; nothing - * here re-renders it. The platform's four-valued outcome is passed through, so the relay can tell a - * possibly-stored message (ambiguous) from a refused one. + * here re-renders it. The canonical publisher's attempt — its outcome and its failure code — is + * passed through unchanged, so the relay can tell a possibly-stored message (ambiguous) from a + * refused one, and a refused credential from a closed transport. */ public final class OutboxMessagePublishAdapter implements OutboxMessagePublishPort { @@ -27,7 +31,7 @@ public final class OutboxMessagePublishAdapter implements OutboxMessagePublishPo @Override public void publish(ClaimedOutboxEvent event) { - OutboxPublishOutcome outcome = publishForOutcome(event); + OutboxPublishOutcome outcome = publishForOutcome(event).outcome(); if (outcome == OutboxPublishOutcome.CONFIRMED) { return; } @@ -37,8 +41,14 @@ public final class OutboxMessagePublishAdapter implements OutboxMessagePublishPo throw new OutboxPublishRefusedException(outcome); } + /** The canonical publisher's, unchanged: this adapter adds no wait of its own. */ @Override - public OutboxPublishOutcome publishForOutcome(ClaimedOutboxEvent event) { + public Optional longestAttempt() { + return canonicalPublisher.longestAttempt(); + } + + @Override + public OutboxPublishAttempt publishForOutcome(ClaimedOutboxEvent event) { Objects.requireNonNull(event, "event must not be null"); return canonicalPublisher.publish(event.event()).toCompletableFuture().join(); } diff --git a/src/adapter/outbound/messaging/src/main/java/dev/caskeleton/adapter/outbound/messaging/outbox/Slf4jOutboxRelayFailureReportAdapter.java b/src/adapter/outbound/messaging/src/main/java/dev/caskeleton/adapter/outbound/messaging/outbox/Slf4jOutboxRelayFailureReportAdapter.java index 9c6d897a..588a82bb 100644 --- a/src/adapter/outbound/messaging/src/main/java/dev/caskeleton/adapter/outbound/messaging/outbox/Slf4jOutboxRelayFailureReportAdapter.java +++ b/src/adapter/outbound/messaging/src/main/java/dev/caskeleton/adapter/outbound/messaging/outbox/Slf4jOutboxRelayFailureReportAdapter.java @@ -48,6 +48,10 @@ public final class Slf4jOutboxRelayFailureReportAdapter implements OutboxRelayFa // occasionally credentials. The class name is a type; the code is bounded. .addKeyValue("error.code", report.code().code()) .addKeyValue("error.cause_type", report.causeType()) + // Why, in the publisher's words: the cause type of a returned outcome names only + // which + // outcome it was, and a refused credential and a closed transport are one outcome. + .addKeyValue("error.reason", report.failureCode()) .addKeyValue("error.category", report.code().category().name()) .addKeyValue("dependency_name", dependencyName) .addKeyValue("dependency_type", DEPENDENCY_TYPE) diff --git a/src/adapter/outbound/messaging/src/main/java/dev/caskeleton/adapter/outbound/messaging/platformbridge/PlatformIntegrationEventPublishAdapter.java b/src/adapter/outbound/messaging/src/main/java/dev/caskeleton/adapter/outbound/messaging/platformbridge/PlatformIntegrationEventPublishAdapter.java index 49e84155..8b81e8ae 100644 --- a/src/adapter/outbound/messaging/src/main/java/dev/caskeleton/adapter/outbound/messaging/platformbridge/PlatformIntegrationEventPublishAdapter.java +++ b/src/adapter/outbound/messaging/src/main/java/dev/caskeleton/adapter/outbound/messaging/platformbridge/PlatformIntegrationEventPublishAdapter.java @@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.messaging.platformbridge; import dev.caskeleton.application.messaging.event.IntegrationEventPublishPort; import dev.caskeleton.application.messaging.event.ValidatedIntegrationEvent; +import dev.caskeleton.application.outbox.OutboxPublishAttempt; import dev.caskeleton.application.outbox.OutboxPublishOutcome; import dev.caskeleton.messaging.api.CausationId; import dev.caskeleton.messaging.api.ContentType; @@ -26,6 +27,7 @@ import dev.caskeleton.messaging.api.publish.TransmissionEvidence; import dev.caskeleton.messaging.schema.EncodedMessage; import dev.caskeleton.messaging.schema.EncodedMessagePublisher; import dev.caskeleton.messaging.schema.SchemaReference; +import java.time.Duration; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -45,6 +47,12 @@ import java.util.concurrent.CompletionStage; */ public final class PlatformIntegrationEventPublishAdapter implements IntegrationEventPublishPort { + /** + * The failure code of an event this bridge refuses before the platform sees it: a value — an id + * above all — the platform's types cannot carry. + */ + public static final String NOT_REPRESENTABLE = "EVENT_NOT_REPRESENTABLE"; + private final EncodedMessagePublisher publisher; private final ProducerId producer; @@ -55,7 +63,7 @@ public final class PlatformIntegrationEventPublishAdapter implements Integration } @Override - public CompletionStage publish(ValidatedIntegrationEvent event) { + public CompletionStage publish(ValidatedIntegrationEvent event) { Objects.requireNonNull(event, "event must not be null"); final PreparedPublish prepared; try { @@ -64,12 +72,27 @@ public final class PlatformIntegrationEventPublishAdapter implements Integration // Only a value the platform cannot represent is a definite refusal: the same payload is // refused again, so the relay dead-letters it at once. Any other failure is not evidence of // that and propagates, so the relay treats it as a retryable failure with its real cause. - return CompletableFuture.completedFuture(OutboxPublishOutcome.REJECTED_BEFORE_SEND); + return CompletableFuture.completedFuture( + OutboxPublishAttempt.failed( + OutboxPublishOutcome.REJECTED_BEFORE_SEND, NOT_REPRESENTABLE)); } return publisher .publishEncoded(prepared.destination(), prepared.envelope(), PublishOptions.defaults()) - .thenApply(PlatformIntegrationEventPublishAdapter::mapOutcome); + .thenApply(PlatformIntegrationEventPublishAdapter::translate); + } + + /** + * Twice the publish deadline this bridge asks for ({@link PublishOptions#defaults()}). + * + *

    The platform measures the deadline from the call, but a transport may block inside its send + * before the deadline is applied to the stage it returns — Kafka's {@code max.block.ms} is set to + * the same default for exactly that wait — so an attempt can take the deadline twice over. The + * relay sizes its batch against this (review 2026-11-01 06-12). + */ + @Override + public Optional longestAttempt() { + return Optional.of(PublishOptions.defaults().timeout().multipliedBy(2)); } private PreparedPublish prepare(ValidatedIntegrationEvent event) { @@ -146,8 +169,10 @@ public final class PlatformIntegrationEventPublishAdapter implements Integration * * *

    Either refusal is {@code REJECTED_BEFORE_SEND}: the same id is refused again, so the relay - * dead-letters it. The append side's {@code EventId} grammar is where such an id should fail - * first, in the writer's transaction; until it does, this is where the mismatch is caught. + * dead-letters it. A new event fails earlier, in the writer's transaction: {@code + * IntegrationEventDraft} refuses an id that is not a canonical lower-case UUIDv7 (re-review + * 2026-11-03 RR-05-04). This check stays for rows appended before that rule — the stored {@code + * EventId} grammar is wider, so they can still be read back. * * @throws IllegalArgumentException when the value is not a canonical lower-case UUIDv7 */ @@ -177,8 +202,37 @@ public final class PlatformIntegrationEventPublishAdapter implements Integration * committed event, and its aggregate behind it, for a cause that clears on its own. It gets its * own value instead. */ - private static OutboxPublishOutcome mapOutcome(PublishResult result) { + private static OutboxPublishAttempt translate(PublishResult result) { Objects.requireNonNull(result, "publish result must not be null"); + OutboxPublishOutcome outcome = mapOutcome(result); + if (outcome == OutboxPublishOutcome.CONFIRMED) { + return OutboxPublishAttempt.confirmed(); + } + return result + .failure() + .map(PlatformIntegrationEventPublishAdapter::failureCode) + .map(code -> OutboxPublishAttempt.failed(outcome, code)) + .orElseGet(() -> OutboxPublishAttempt.of(outcome)); + } + + /** + * The platform's failure code, carried as the application's failure code (re-review 2026-11-03 + * RR-05-03). + * + *

    The outcome is all the relay acts on, and it loses the reason on purpose: a shutdown, a + * closed transport and a refused credential are one {@code REJECTED_RETRYABLE}. The code keeps it + * for the operator, on the row and in the failure log. The platform's code is already sanitized — + * a stable name, never a driver message — and passes as it is; one outside the application's code + * grammar (the platform only bounds its length) is carried as the failure's category instead, a + * name the platform owns too. + */ + private static String failureCode(FailureDescriptor failure) { + return OutboxPublishAttempt.isFailureCode(failure.code()) + ? failure.code() + : failure.category().name(); + } + + private static OutboxPublishOutcome mapOutcome(PublishResult result) { if (result.completion() == PublishCompletion.CONFIRMED) { return OutboxPublishOutcome.CONFIRMED; } diff --git a/src/adapter/outbound/messaging/src/test/java/dev/caskeleton/adapter/outbound/messaging/MessagingConfigTest.java b/src/adapter/outbound/messaging/src/test/java/dev/caskeleton/adapter/outbound/messaging/MessagingConfigTest.java index 54ea4f2d..9923ddab 100644 --- a/src/adapter/outbound/messaging/src/test/java/dev/caskeleton/adapter/outbound/messaging/MessagingConfigTest.java +++ b/src/adapter/outbound/messaging/src/test/java/dev/caskeleton/adapter/outbound/messaging/MessagingConfigTest.java @@ -5,7 +5,7 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.adapter.outbound.messaging.outbox.OutboxMessagePublishAdapter; import dev.caskeleton.application.messaging.event.IntegrationEventPublishPort; import dev.caskeleton.application.outbox.OutboxMessagePublishPort; -import dev.caskeleton.application.outbox.OutboxPublishOutcome; +import dev.caskeleton.application.outbox.OutboxPublishAttempt; import java.util.concurrent.CompletableFuture; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.runner.ApplicationContextRunner; @@ -32,7 +32,7 @@ class MessagingConfigTest { runner .withBean( IntegrationEventPublishPort.class, - () -> event -> CompletableFuture.completedFuture(OutboxPublishOutcome.CONFIRMED)) + () -> event -> CompletableFuture.completedFuture(OutboxPublishAttempt.confirmed())) .run( context -> assertThat(context.getBean(OutboxMessagePublishPort.class)) diff --git a/src/adapter/outbound/messaging/src/test/java/dev/caskeleton/adapter/outbound/messaging/outbox/Slf4jOutboxRelayFailureReportAdapterTest.java b/src/adapter/outbound/messaging/src/test/java/dev/caskeleton/adapter/outbound/messaging/outbox/Slf4jOutboxRelayFailureReportAdapterTest.java index 85322ffb..5c7e7cd8 100644 --- a/src/adapter/outbound/messaging/src/test/java/dev/caskeleton/adapter/outbound/messaging/outbox/Slf4jOutboxRelayFailureReportAdapterTest.java +++ b/src/adapter/outbound/messaging/src/test/java/dev/caskeleton/adapter/outbound/messaging/outbox/Slf4jOutboxRelayFailureReportAdapterTest.java @@ -51,7 +51,14 @@ class Slf4jOutboxRelayFailureReportAdapterTest { adapter.report( OutboxRelayFailureReport.retryableFailure( - "evt-1", "WorkLogReserved", "agg-1", "corr-1", 2, nextAttemptAt, cause)); + "evt-1", + "WorkLogReserved", + "agg-1", + "corr-1", + 2, + nextAttemptAt, + cause, + "KAFKA_AUTHENTICATIONEXCEPTION")); assertThat(appender.list).hasSize(1); ILoggingEvent event = appender.list.getFirst(); @@ -71,6 +78,7 @@ class Slf4jOutboxRelayFailureReportAdapterTest { Map.entry("correlation_id", "corr-1"), Map.entry("attempt_count", 2), Map.entry("error.cause_type", "java.lang.RuntimeException"), + Map.entry("error.reason", "KAFKA_AUTHENTICATIONEXCEPTION"), Map.entry("runbook_link", "runbook://outbox/publish-failed"), Map.entry("next_attempt_at", "2026-07-25T01:02:03Z"))); assertThat(event.getThrowableProxy()) @@ -89,7 +97,7 @@ class Slf4jOutboxRelayFailureReportAdapterTest { adapter.report( OutboxRelayFailureReport.deadLetter( - "evt-2", "WorkLogReserved", "agg-2", "corr-2", 3, cause)); + "evt-2", "WorkLogReserved", "agg-2", "corr-2", 3, cause, "REJECTED_BEFORE_SEND")); assertThat(appender.list).hasSize(1); ILoggingEvent event = appender.list.getFirst(); @@ -142,7 +150,8 @@ class Slf4jOutboxRelayFailureReportAdapterTest { "agg-3", "corr-3", 3, - new RuntimeException("broker down")))) + new RuntimeException("broker down"), + "RuntimeException"))) .doesNotThrowAnyException(); } diff --git a/src/adapter/outbound/messaging/src/test/java/dev/caskeleton/adapter/outbound/messaging/platformbridge/PlatformIntegrationEventPublishAdapterTest.java b/src/adapter/outbound/messaging/src/test/java/dev/caskeleton/adapter/outbound/messaging/platformbridge/PlatformIntegrationEventPublishAdapterTest.java index 41564585..d1564cd4 100644 --- a/src/adapter/outbound/messaging/src/test/java/dev/caskeleton/adapter/outbound/messaging/platformbridge/PlatformIntegrationEventPublishAdapterTest.java +++ b/src/adapter/outbound/messaging/src/test/java/dev/caskeleton/adapter/outbound/messaging/platformbridge/PlatformIntegrationEventPublishAdapterTest.java @@ -12,6 +12,7 @@ import dev.caskeleton.application.messaging.event.IntegrationEventPublishPort; import dev.caskeleton.application.messaging.event.ValidatedIntegrationEvent; import dev.caskeleton.application.messaging.event.testkit.BrokerAnswer; import dev.caskeleton.application.messaging.event.testkit.IntegrationEventPublishContract; +import dev.caskeleton.application.outbox.OutboxPublishAttempt; import dev.caskeleton.application.outbox.OutboxPublishOutcome; import dev.caskeleton.messaging.api.ContentType; import dev.caskeleton.messaging.api.MessageEnvelope; @@ -59,7 +60,7 @@ class PlatformIntegrationEventPublishAdapterTest extends IntegrationEventPublish new PlatformIntegrationEventPublishAdapter(publisher, "portfolio-service"); ValidatedIntegrationEvent event = event(EVENT_ID, Optional.of(CAUSATION_ID)); - assertThat(bridge.publish(event).toCompletableFuture().join()) + assertThat(bridge.publish(event).toCompletableFuture().join().outcome()) .isEqualTo(OutboxPublishOutcome.CONFIRMED); MessageDestination destination = publisher.destination; @@ -99,19 +100,19 @@ class PlatformIntegrationEventPublishAdapterTest extends IntegrationEventPublish new PlatformIntegrationEventPublishAdapter(publisher, "portfolio-service"); ValidatedIntegrationEvent event = event(EVENT_ID, Optional.empty()); - assertThat(bridge.publish(event).toCompletableFuture().join()) + assertThat(bridge.publish(event).toCompletableFuture().join().outcome()) .isEqualTo(OutboxPublishOutcome.CONFIRMED); publisher.result = ambiguous(); - assertThat(bridge.publish(event).toCompletableFuture().join()) + assertThat(bridge.publish(event).toCompletableFuture().join().outcome()) .isEqualTo(OutboxPublishOutcome.AMBIGUOUS); publisher.result = rejected(PublishEvidence.notTransmitted()); - assertThat(bridge.publish(event).toCompletableFuture().join()) + assertThat(bridge.publish(event).toCompletableFuture().join().outcome()) .isEqualTo(OutboxPublishOutcome.REJECTED_BEFORE_SEND); publisher.result = rejected( new PublishEvidence( true, TransmissionEvidence.TRANSMITTED, false, ConfirmationLevel.NONE)); - assertThat(bridge.publish(event).toCompletableFuture().join()) + assertThat(bridge.publish(event).toCompletableFuture().join().outcome()) .isEqualTo(OutboxPublishOutcome.REJECTED_AFTER_BROKER); } @@ -132,12 +133,101 @@ class PlatformIntegrationEventPublishAdapterTest extends IntegrationEventPublish // when OutboxPublishOutcome outcome = - bridge.publish(event(EVENT_ID, Optional.empty())).toCompletableFuture().join(); + bridge.publish(event(EVENT_ID, Optional.empty())).toCompletableFuture().join().outcome(); // then assertThat(outcome).isEqualTo(OutboxPublishOutcome.REJECTED_RETRYABLE); } + @Test + @DisplayName("platform이 붙인 실패 코드는 원인 코드로 그대로 옮긴다") + void thePlatformsFailureCodeIsCarriedAsTheFailureCode() { + // given: the broker refused this producer's credentials + RecordingPublisher publisher = + new RecordingPublisher( + PublishResult.rejected( + Duration.ZERO, + new FailureDescriptor( + FailureCategory.AUTHENTICATION, + "KAFKA_AUTHENTICATIONEXCEPTION", + true, + "kafka publish failed with AuthenticationException", + Optional.of("AuthenticationException")))); + PlatformIntegrationEventPublishAdapter bridge = + new PlatformIntegrationEventPublishAdapter(publisher, "portfolio-service"); + + // when + OutboxPublishAttempt attempt = + bridge.publish(event(EVENT_ID, Optional.empty())).toCompletableFuture().join(); + + // then + assertThat(attempt) + .isEqualTo( + OutboxPublishAttempt.failed( + OutboxPublishOutcome.REJECTED_RETRYABLE, "KAFKA_AUTHENTICATIONEXCEPTION")); + } + + @Test + @DisplayName("코드 문법에 맞지 않는 platform 실패 코드는 그 실패의 분류 이름으로 옮긴다") + void aPlatformCodeOutsideTheCodeGrammarIsCarriedAsItsCategory() { + // given + RecordingPublisher publisher = + new RecordingPublisher( + PublishResult.ambiguous( + Duration.ZERO, + FailureDescriptor.of( + FailureCategory.AMBIGUOUS, "timed out after 5000 ms", "no answer"))); + PlatformIntegrationEventPublishAdapter bridge = + new PlatformIntegrationEventPublishAdapter(publisher, "portfolio-service"); + + // when + OutboxPublishAttempt attempt = + bridge.publish(event(EVENT_ID, Optional.empty())).toCompletableFuture().join(); + + // then + assertThat(attempt.failureCode()).contains("AMBIGUOUS"); + } + + @Test + @DisplayName("platform으로 옮길 수 없는 event는 그 이유를 원인 코드로 알린다") + void anEventThePlatformCannotRepresentIsRefusedWithItsOwnFailureCode() { + // given + RecordingPublisher publisher = new RecordingPublisher(confirmed()); + PlatformIntegrationEventPublishAdapter bridge = + new PlatformIntegrationEventPublishAdapter(publisher, "portfolio-service"); + + // when + OutboxPublishAttempt attempt = + bridge.publish(event("event-1", Optional.empty())).toCompletableFuture().join(); + + // then + assertThat(attempt) + .isEqualTo( + OutboxPublishAttempt.failed( + OutboxPublishOutcome.REJECTED_BEFORE_SEND, + PlatformIntegrationEventPublishAdapter.NOT_REPRESENTABLE)); + } + + /** + * The relay sizes its batch against this (review 2026-11-01 06-12). The platform's deadline is + * measured from the call, but a transport may block in its send before the deadline is applied — + * Kafka's {@code max.block.ms} is set to the same default — so an attempt can take twice it. + */ + @Test + @DisplayName("발행 하나의 최악 시간은 platform 기본 발행 기한의 두 배다") + void theLongestAttemptIsTwiceThePlatformsDefaultPublishDeadline() { + // given + PlatformIntegrationEventPublishAdapter bridge = + new PlatformIntegrationEventPublishAdapter( + new RecordingPublisher(confirmed()), "portfolio-service"); + + // when + Optional longest = bridge.longestAttempt(); + + // then + assertThat(longest).contains(PublishOptions.defaults().timeout().multipliedBy(2)); + } + @Test @DisplayName("다시 시도해도 같은 결과인 거부는 지금처럼 확정 거부로 번역한다") void aRefusalThePlatformMarksFinalStaysADefiniteRejection() { @@ -153,7 +243,7 @@ class PlatformIntegrationEventPublishAdapterTest extends IntegrationEventPublish // when OutboxPublishOutcome outcome = - bridge.publish(event(EVENT_ID, Optional.empty())).toCompletableFuture().join(); + bridge.publish(event(EVENT_ID, Optional.empty())).toCompletableFuture().join().outcome(); // then assertThat(outcome).isEqualTo(OutboxPublishOutcome.REJECTED_BEFORE_SEND); @@ -165,7 +255,12 @@ class PlatformIntegrationEventPublishAdapterTest extends IntegrationEventPublish PlatformIntegrationEventPublishAdapter bridge = new PlatformIntegrationEventPublishAdapter(publisher, "portfolio-service"); - assertThat(bridge.publish(event("event-1", Optional.empty())).toCompletableFuture().join()) + assertThat( + bridge + .publish(event("event-1", Optional.empty())) + .toCompletableFuture() + .join() + .outcome()) .isEqualTo(OutboxPublishOutcome.REJECTED_BEFORE_SEND); assertThat(publisher.calls).isZero(); } @@ -192,7 +287,7 @@ class PlatformIntegrationEventPublishAdapterTest extends IntegrationEventPublish // when OutboxPublishOutcome outcome = - bridge.publish(event(eventId, Optional.empty())).toCompletableFuture().join(); + bridge.publish(event(eventId, Optional.empty())).toCompletableFuture().join().outcome(); // then assertThat(outcome).isEqualTo(OutboxPublishOutcome.REJECTED_BEFORE_SEND); diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationCallbackProviderConfig.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationCallbackProviderConfig.java index 97cb53a8..95e2ef4f 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationCallbackProviderConfig.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/NotificationCallbackProviderConfig.java @@ -64,17 +64,39 @@ class NotificationCallbackProviderConfig { return cachingCertificates(uri -> fetchCertificate(client, uri)); } - private static byte[] fetchCertificate(HttpClient client, URI uri) { + /** + * Fetches the certificate document at a host-checked SNS URL. + * + *

    The status decides what a failed fetch says. The URL already passed the SNS host check, so + * the host answering is SNS. A 4xx is SNS saying it has no such certificate: a real SNS message + * names one that exists, so the envelope is forged — {@link IllegalArgumentException}, which the + * verifier turns into an invalid signature that is refused and audited. It used to be "cannot + * verify now" like every other non-2xx, so a forged envelope pointing at a certificate SNS never + * issued got a 503, was never audited, and was logged as a provider hiccup (review 2026-11-03 + * RR-07-03). A 5xx, a 408 or 429 (slow or throttling host), or a network failure says nothing + * about the certificate and stays "cannot verify now". + * + * @param client the no-redirect client + * @param uri the host-checked certificate URL + * @return the certificate document, at most {@link #MAX_SNS_CERTIFICATE_BYTES} + * @throws IllegalArgumentException when SNS answers that the certificate does not exist + * @throws CallbackVerificationUnavailableException when the certificate could not be fetched now + */ + static byte[] fetchCertificate(HttpClient client, URI uri) { HttpRequest request = HttpRequest.newBuilder(uri).timeout(SNS_CERTIFICATE_TIMEOUT).GET().build(); try { HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); try (InputStream body = response.body()) { - if (response.statusCode() / 100 != 2) { + int status = response.statusCode(); + if (status / 100 == 4 && status != 408 && status != 429) { + throw new IllegalArgumentException( + "SNS has no signing certificate at the URL the message names (HTTP " + status + ")"); + } + if (status / 100 != 2) { throw certificateUnavailable( - new IOException( - "SNS signing certificate fetch failed with HTTP " + response.statusCode())); + new IOException("SNS signing certificate fetch failed with HTTP " + status)); } return body.readNBytes(MAX_SNS_CERTIFICATE_BYTES); } diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/provider/ses/SesCallbackAdapter.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/provider/ses/SesCallbackAdapter.java index 78e583ae..3aa9373a 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/provider/ses/SesCallbackAdapter.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/provider/ses/SesCallbackAdapter.java @@ -16,6 +16,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import tools.jackson.databind.JsonNode; @@ -36,8 +37,12 @@ import tools.jackson.databind.JsonNode; * signed string), for up to {@link #SNS_HTTP_RETRY_HORIZON}. A window of five minutes threw away * every bounce and complaint that arrived after an outage longer than that, and with them the * suppressions they should have created (review 05-20). Accepting a replay inside the window costs - * nothing: the provider event ledger is unique on {@code (provider_profile_id, provider_event_id)}, - * so a second delivery of the same event is a duplicate, not a second event. + * nothing: the provider event ledger is unique on {@code (provider_profile_id, provider_event_id)} + * whenever an event id is present, and this adapter gives every SES event the signed SNS {@code + * MessageId} as that id, so a second delivery of the same message is a duplicate, not a second + * event. It used to give none, and the only identity left was the {@code (provider_profile_id, + * event_fingerprint)} index — a fingerprint keyed by the active HMAC key, so a redelivery after a + * key rotation inside the window was recorded twice (review 2026-11-03 RR-07-07). */ public final class SesCallbackAdapter implements ProviderCallbackAdapter { @@ -118,7 +123,10 @@ public final class SesCallbackAdapter implements ProviderCallbackAdapter { return List.of(); } String message = envelope.getOrDefault("Message", "{}"); - return normalizer.normalize(message); + // The SNS MessageId is in the signed string and survives every redelivery, so it — not a + // fingerprint under the current HMAC key — is what the ledger recognises a replay by. + return normalizer.normalize( + message, Optional.ofNullable(envelope.get("MessageId")).filter(id -> !id.isBlank())); } private boolean withinReplayWindow(String timestamp, Instant receivedAt) { diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/provider/ses/SesEventNormalizer.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/provider/ses/SesEventNormalizer.java index 24d5d5ff..f35bd5b5 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/provider/ses/SesEventNormalizer.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/provider/ses/SesEventNormalizer.java @@ -7,6 +7,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import tools.jackson.databind.JsonNode; @@ -19,8 +20,16 @@ import tools.jackson.databind.JsonNode; */ public final class SesEventNormalizer { - /** Normalize one SES notification payload. */ - public List normalize(String payload) { + /** + * Normalize one SES notification payload. + * + * @param payload the SES event JSON (the SNS envelope's {@code Message}) + * @param providerEventId the identity of the delivery that carried it — the signed SNS {@code + * MessageId}, which SNS keeps across redeliveries — or empty when the envelope had none + * @return the provider events, each carrying {@code providerEventId} + */ + public List normalize(String payload, Optional providerEventId) { + Objects.requireNonNull(providerEventId, "providerEventId"); JsonNode root = NotificationJsonMapper.mapper().readTree(payload); String type = text(root, "eventType").orElse(text(root, "notificationType").orElse("Unknown")); Optional messageId = @@ -30,24 +39,26 @@ public final class SesEventNormalizer { List events = new ArrayList<>(1); events.add( switch (type) { - case "Send" -> - event(NormalizedEventType.PROVIDER_ACCEPTED, type, messageId, occurredAt, Map.of()); + case "Send" -> event(NormalizedEventType.PROVIDER_ACCEPTED, type, messageId, occurredAt); case "Delivery" -> - event(NormalizedEventType.DELIVERY_CONFIRMED, type, messageId, occurredAt, Map.of()); + event(NormalizedEventType.DELIVERY_CONFIRMED, type, messageId, occurredAt); case "DeliveryDelay" -> - event(NormalizedEventType.DELIVERY_DELAYED, type, messageId, occurredAt, Map.of()); + event(NormalizedEventType.DELIVERY_DELAYED, type, messageId, occurredAt); case "Bounce" -> bounce(root, type, messageId, occurredAt); - case "Complaint" -> - event(NormalizedEventType.COMPLAINT, type, messageId, occurredAt, Map.of()); + case "Complaint" -> event(NormalizedEventType.COMPLAINT, type, messageId, occurredAt); case "Reject" -> - event(NormalizedEventType.PROVIDER_REJECTED, type, messageId, occurredAt, Map.of()); + event(NormalizedEventType.PROVIDER_REJECTED, type, messageId, occurredAt); case "RenderingFailure" -> - event(NormalizedEventType.TEMPLATE_FAILURE, type, messageId, occurredAt, Map.of()); - case "Open" -> event(NormalizedEventType.OPENED, type, messageId, occurredAt, Map.of()); - case "Click" -> event(NormalizedEventType.CLICKED, type, messageId, occurredAt, Map.of()); - default -> event(NormalizedEventType.UNKNOWN, type, messageId, occurredAt, Map.of()); + event(NormalizedEventType.TEMPLATE_FAILURE, type, messageId, occurredAt); + case "Open" -> event(NormalizedEventType.OPENED, type, messageId, occurredAt); + case "Click" -> event(NormalizedEventType.CLICKED, type, messageId, occurredAt); + default -> event(NormalizedEventType.UNKNOWN, type, messageId, occurredAt); }); - return List.copyOf(events); + // One SNS delivery carries one SES event, so the delivery's id is the event's. It is what makes + // a redelivery a duplicate in the ledger: the other identity, the event fingerprint, is keyed + // by + // the active HMAC key and changes when the key rotates (review 2026-11-03 RR-07-07). + return events.stream().map(event -> withEventId(event, providerEventId)).toList(); } private static NormalizedProviderEvent bounce( @@ -68,6 +79,14 @@ public final class SesEventNormalizer { Map.of("bounceType", bounceType)); } + private static NormalizedProviderEvent event( + NormalizedEventType type, + String nativeType, + Optional messageId, + Optional occurredAt) { + return event(type, nativeType, messageId, occurredAt, Map.of()); + } + private static NormalizedProviderEvent event( NormalizedEventType type, String nativeType, @@ -78,6 +97,17 @@ public final class SesEventNormalizer { type, nativeType, Optional.empty(), messageId, occurredAt, attributes); } + private static NormalizedProviderEvent withEventId( + NormalizedProviderEvent event, Optional providerEventId) { + return new NormalizedProviderEvent( + event.type(), + event.providerNativeType(), + providerEventId, + event.providerRequestId(), + event.providerOccurredAt(), + event.attributes()); + } + private static Optional text(JsonNode node, String field) { JsonNode value = node.get(field); return value == null || value.isNull() ? Optional.empty() : Optional.of(value.asString()); diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/provider/smtp/JavaMailSenderSmtpDispatch.java b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/provider/smtp/JavaMailSenderSmtpDispatch.java index b19ef84a..ca659a9f 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/provider/smtp/JavaMailSenderSmtpDispatch.java +++ b/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/provider/smtp/JavaMailSenderSmtpDispatch.java @@ -112,7 +112,12 @@ public final class JavaMailSenderSmtpDispatch implements SmtpDispatch { } else if (cause instanceof SMTPSendFailedException send) { code = send.getReturnCode(); } - return code >= 200 && code < 600 ? Optional.of(code) : Optional.empty(); + // Only a refusal is a reply code here: 4xx (transient) and 5xx (permanent). Angus's sendpartial + // path throws SMTPSendFailedException carrying the last reply, which can be a success (250); + // read as a reply code it reached the classifier as a permanent refusal of a message the relay + // had partly accepted. Anything else takes the conservative "may have been delivered" path + // below (review 2026-11-03 RR-04-11). + return code >= 400 && code < 600 ? Optional.of(code) : Optional.empty(); } /** diff --git a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/SnsCertificateFetchTest.java b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/SnsCertificateFetchTest.java new file mode 100644 index 00000000..50ace0f3 --- /dev/null +++ b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/autoconfigure/SnsCertificateFetchTest.java @@ -0,0 +1,104 @@ +package dev.caskeleton.adapter.outbound.notification.platform.autoconfigure; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import com.sun.net.httpserver.HttpServer; +import dev.caskeleton.adapter.outbound.notification.platform.callback.CallbackVerificationUnavailableException; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.URI; +import java.net.http.HttpClient; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * What a failed SNS signing-certificate fetch says about the callback (review 2026-11-03 RR-07-03). + * + *

    The URL has passed the SNS host check before it is fetched, so the host is SNS's. A real SNS + * message names a certificate that exists; when SNS itself answers 4xx for it, the envelope named a + * certificate SNS never issued — a forged callback, to be rejected and audited. Only a failure that + * says nothing about the certificate — a 5xx, a 408 or 429 (the host is slow or throttling), or a + * network failure — is "cannot verify now" (503, the provider retries). + */ +class SnsCertificateFetchTest { + + private final HttpClient client = HttpClient.newHttpClient(); + + @ParameterizedTest + @ValueSource(ints = {403, 404, 410}) + @DisplayName("인증서 URL이 4xx로 답하면 없는 인증서를 가리킨 위조 콜백이므로 검증 실패로 거절된다") + void aCertificateTheHostDoesNotHaveFailsVerification(int status) throws IOException { + // given + HttpServer server = answering(status); + try { + URI url = URI.create("http://127.0.0.1:" + server.getAddress().getPort() + "/cert.pem"); + + // when + Throwable failure = + catchThrowable(() -> NotificationCallbackProviderConfig.fetchCertificate(client, url)); + + // then + assertThat(failure) + .isInstanceOf(IllegalArgumentException.class) + .isNotInstanceOf(CallbackVerificationUnavailableException.class); + } finally { + server.stop(0); + } + } + + @ParameterizedTest + @ValueSource(ints = {408, 429, 500, 503}) + @DisplayName("인증서 URL이 5xx나 시간 초과·요청 제한(408·429)으로 답하면 아무것도 모르므로 지금 검증할 수 없다고 답한다") + void aFailingCertificateHostLeavesTheCallbackUnverified(int status) throws IOException { + // given + HttpServer server = answering(status); + try { + URI url = URI.create("http://127.0.0.1:" + server.getAddress().getPort() + "/cert.pem"); + + // when + Throwable failure = + catchThrowable(() -> NotificationCallbackProviderConfig.fetchCertificate(client, url)); + + // then + assertThat(failure).isInstanceOf(CallbackVerificationUnavailableException.class); + } finally { + server.stop(0); + } + } + + @Test + @DisplayName("인증서 호스트에 닿지 못하면 지금 검증할 수 없다고 답한다") + void anUnreachableCertificateHostLeavesTheCallbackUnverified() throws IOException { + // given + int closedPort; + try (ServerSocket probe = new ServerSocket(0)) { + closedPort = probe.getLocalPort(); + } + URI url = URI.create("http://127.0.0.1:" + closedPort + "/cert.pem"); + + // when + Throwable failure = + catchThrowable(() -> NotificationCallbackProviderConfig.fetchCertificate(client, url)); + + // then + assertThat(failure).isInstanceOf(CallbackVerificationUnavailableException.class); + } + + private static HttpServer answering(int status) throws IOException { + HttpServer server = + HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext( + "/", + exchange -> { + exchange.sendResponseHeaders(status, -1); + exchange.close(); + }); + server.start(); + return server; + } +} diff --git a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/provider/ses/SesCallbackAdapterTest.java b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/provider/ses/SesCallbackAdapterTest.java index d425bf65..292f2b58 100644 --- a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/provider/ses/SesCallbackAdapterTest.java +++ b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/provider/ses/SesCallbackAdapterTest.java @@ -8,6 +8,7 @@ import dev.caskeleton.adapter.outbound.notification.platform.callback.CallbackVe import dev.caskeleton.adapter.outbound.notification.platform.callback.CallbackVerificationUnavailableException; import dev.caskeleton.adapter.outbound.notification.platform.callback.ProviderCallbackVerificationRequest; import dev.caskeleton.adapter.outbound.notification.platform.template.NotificationJsonMapper; +import dev.caskeleton.application.notification.platform.api.NormalizedProviderEvent; import dev.caskeleton.application.notification.platform.api.ProviderId; import dev.caskeleton.application.notification.platform.api.ProviderProfileId; import java.nio.charset.StandardCharsets; @@ -19,6 +20,7 @@ import java.time.Duration; import java.time.Instant; import java.util.Base64; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -135,6 +137,28 @@ class SesCallbackAdapterTest { assertThat(unavailable.reasonCode()).isEqualTo("SNS_CERTIFICATE_UNAVAILABLE")); } + @Test + @DisplayName( + "SES 이벤트의 provider event id는 서명된 SNS MessageId라 fingerprint 키를 회전해도 같은 콜백이 같은 이벤트로 남는다") + void theSignedSnsMessageIdIsTheProviderEventId() { + // given + CallbackVerificationResult verified = + adapter.verify(request(signed(ALLOWED_TOPIC, RECEIVED_AT))); + + // when + List events = + adapter.normalize(verified.verifiedCallback().orElseThrow()); + + // then + // The ledger is unique on (profile, provider_event_id) whenever an id is present. Without one + // the only identity was a fingerprint keyed by the active HMAC key, so a redelivery after a + // key rotation inside the one-hour replay window became a second row (RR-07-07). + assertThat(events) + .singleElement() + .extracting(NormalizedProviderEvent::providerEventId) + .isEqualTo(Optional.of("sns-message-1")); + } + /** The adapter with the window a deployment gets when it sets nothing. */ private static SesCallbackAdapter withDefaultWindow() { return new SesCallbackAdapter( diff --git a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/provider/smtp/JavaMailSenderSmtpDispatchTest.java b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/provider/smtp/JavaMailSenderSmtpDispatchTest.java index 4fd987ad..28009d6b 100644 --- a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/provider/smtp/JavaMailSenderSmtpDispatchTest.java +++ b/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/provider/smtp/JavaMailSenderSmtpDispatchTest.java @@ -16,9 +16,11 @@ import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Optional; import java.util.Properties; +import org.eclipse.angus.mail.smtp.SMTPSendFailedException; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.mail.MailAuthenticationException; +import org.springframework.mail.MailSendException; import org.springframework.mail.javamail.JavaMailSenderImpl; /** @@ -136,6 +138,38 @@ class JavaMailSenderSmtpDispatchTest { .containsExactly(Optional.empty(), false); } + @Test + @DisplayName("성공 코드(250)를 실은 부분 전송 실패는 거절 reply code가 아니라 전달됐을 수 있는 실패로 보고된다") + void aPartialSendCarryingASuccessCodeIsNotReportedAsARefusal() { + // given + // Angus's sendpartial path reports a partly delivered message as SMTPSendFailedException + // carrying the last reply, which is a success (250), not a refusal. + JavaMailSenderImpl partlyDelivered = + new JavaMailSenderImpl() { + @Override + public void send(MimeMessage... messages) { + throw new MailSendException( + "partly sent", + new SMTPSendFailedException( + "DATA", + 250, + "250 2.0.0 queued", + null, + new jakarta.mail.Address[0], + new jakarta.mail.Address[0], + new jakarta.mail.Address[0])); + } + }; + JavaMailSenderSmtpDispatch dispatch = new JavaMailSenderSmtpDispatch(partlyDelivered); + + // when + SmtpDispatchException failure = sendExpectingFailure(dispatch); + + // then + assertThat(List.of(failure.replyCode(), failure.dataCommitted())) + .containsExactly(Optional.empty(), true); + } + private static SmtpDispatchException sendExpectingFailure(JavaMailSenderSmtpDispatch dispatch) { return catchThrowableOfType(SmtpDispatchException.class, () -> dispatch.send(message())); } diff --git a/src/adapter/outbound/objectstorage/build.gradle b/src/adapter/outbound/objectstorage/build.gradle index ff65bbb7..ee772b62 100644 --- a/src/adapter/outbound/objectstorage/build.gradle +++ b/src/adapter/outbound/objectstorage/build.gradle @@ -155,9 +155,9 @@ strictTestLanes { } } -// The MinIO lanes start a container, so they are the main-branch stage of this leaf, not its -// pull-request `check` (review BUILD-계획-B3); object-storage-qualification.yml runs them on pull -// requests that touch object storage, and integrationCheck runs them with every integration lane. -tasks.named('mainCheck') { - dependsOn 'objectStorageMinioContractTest', 'objectStorageMinioFaultTest' -} +// The MinIO lanes start a container, so they are not this leaf's pull-request `check` (review +// BUILD-계획-B3). They are integration lanes, and the root integrationCheck owns every container lane +// on main and nightly; object-storage-qualification.yml runs them on pull requests that touch object +// storage. This leaf's `mainCheck` used to run them too, so each main push ran them twice in two +// parallel jobs (review 2026-11-03 RR-06-09, original 08-04 (d)). + diff --git a/src/adapter/outbound/persistence-fileserver/CLAUDE.md b/src/adapter/outbound/persistence-fileserver/CLAUDE.md index d4ae3bf1..21a15ad1 100644 --- a/src/adapter/outbound/persistence-fileserver/CLAUDE.md +++ b/src/adapter/outbound/persistence-fileserver/CLAUDE.md @@ -46,12 +46,18 @@ in `platform`), so the two capability leaves read the same way (review 04-16). write; Spring Data repositories are mocks). - `FileserverPersistenceActivationTest` — the scan covers this package only, behind `app.fileserver-platform.enabled`, and runs before the JPA root auto-configuration. - - `repository.CleanupClaimSqlTest` — the claim keeps its one-statement shape (the Claims rule). + - `repository.CleanupClaimSqlTest` — the claim's text keeps its one-statement shape (the Claims + rule); the locking itself is checked on PostgreSQL by + `PostgreSqlFileserverReclamationIntegrationTest`. - `entity.AssignedIdEntityTest` — newness from `Persistable`, on Hibernate over H2. A new store write adds its case to the guard test; a new auto-configuration adds its line to the activation test. -- Port contracts live in `application-fileserver` testFixtures (`*PortContract`). The in-memory - fakes and the PostgreSQL tests both extend them. +- Port contracts live in `application-fileserver` testFixtures (`*PortContract`, and + `FileQuotaLedgerContract` for the three quota ports, which share one table). The in-memory fakes + and the PostgreSQL tests both extend them; every port this leaf implements has one + (`docs/testing/TESTING_STRATEGY.md` 9.3). The PostgreSQL side of the seven added in review + 2026-11-03 RR-01-04 is tagged `jpa-contract` and runs in persistence-jpa's + `jpaPlatformContractTest` lane on `FileserverPostgreSqlStack`, whose clock a case moves. - The real-PostgreSQL tests are still in `persistence-jpa`'s `postgresqlIntegrationTest` source set, which owns the container and Flyway wiring, and the JPA readiness card (`config/jpa/readiness-cards.yaml`, checked by `JpaReadinessRegistryTest`) names them as that diff --git a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaFileQuota.java b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaFileQuota.java index 939f972b..4c935dc0 100644 --- a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaFileQuota.java +++ b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/JpaFileQuota.java @@ -18,8 +18,11 @@ import java.util.UUID; /** * JPA-backed {@link FileQuotaPort}. * - *

    Reservation, extension, commit, and release are conditional statements, so a reservation that - * already expired or was released can never be extended or committed. + *

    Reservation, extension, commit, and release are conditional statements: a reservation that + * expired or was released can never be extended, and a released or already committed one can never + * be committed. An expired reservation that was not released still commits — bytes that were + * written are durable usage whether or not the reservation outlived its TTL. {@code + * FileQuotaLedgerContract} holds this adapter and the in-memory ledger to the same cases. */ public class JpaFileQuota implements FileQuotaPort { diff --git a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/CleanupClaimSql.java b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/CleanupClaimSql.java index 40cddd7d..e9e198bf 100644 --- a/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/CleanupClaimSql.java +++ b/src/adapter/outbound/persistence-fileserver/src/main/java/dev/caskeleton/adapter/outbound/persistence/fileserver/repository/CleanupClaimSql.java @@ -15,8 +15,8 @@ package dev.caskeleton.adapter.outbound.persistence.fileserver.repository; * every claimed item is fenced on its own. * *

    Package-private: the repository is its one reader, and the leaf's public surface is a ceiling. - * {@link #positional(String)} rewrites the named parameters to JDBC placeholders for a test in this - * package that runs the identical text on a bare {@code Connection}. + * The PostgreSQL lane runs it through the repository ({@code + * PostgreSqlFileserverReclamationIntegrationTest}); nothing runs the text on its own. */ final class CleanupClaimSql { @@ -43,14 +43,4 @@ final class CleanupClaimSql { + " c.file_id, c.upload_id, c.content_key"; private CleanupClaimSql() {} - - /** - * Rewrites the named parameters to JDBC placeholders, left to right. - * - * @param namedSql the statement above - * @return the same statement with {@code ?} in place of every {@code :name} - */ - static String positional(String namedSql) { - return namedSql.replaceAll("(? :expected::jsonb"); - - // then - assertThat(positional).isEqualTo("payload::jsonb @> ?::jsonb"); - } } diff --git a/src/adapter/outbound/persistence-jpa/CLAUDE.md b/src/adapter/outbound/persistence-jpa/CLAUDE.md index 5b563043..a8c65e89 100644 --- a/src/adapter/outbound/persistence-jpa/CLAUDE.md +++ b/src/adapter/outbound/persistence-jpa/CLAUDE.md @@ -235,7 +235,11 @@ made Spring call `getTransactionIsolation()` at every begin, which pgjdbc answer Every mode is time-bounded (review 04-10). Spring hands the remaining time to each statement, so a slow query or lock wait is cancelled by the server (`57014` → `DB_QUERY_CANCELED`), and a deadline -that passed before the next statement (`TransactionTimedOutException`) gets the same code. +that passed before the next statement gets the same code: Spring's `TransactionTimedOutException` +on the JDBC path, and on the JPA path (`JpaTransactionManager`, production) Hibernate's own +`TransactionException`, which has no SQLState and is classified because `SpringTransaction` knows +its deadline passed (`translateAfterDeadline`, review RR-03-03). A business exception thrown after +the deadline is passed on unchanged. Pre-built templates are immutable after construction so concurrent callers cannot observe each other's reconfiguration. `inRootWrite` reuses the pre-built write template, @@ -302,7 +306,9 @@ the two paths disagree on any state. **No SQLState, classified by type:** a lost `@Version` race (`OptimisticLockingFailureException`, JPA `OptimisticLockException`) → `DB_OPTIMISTIC_CONFLICT` (409, review 04-05); an expired Spring -transaction deadline (`TransactionTimedOutException`) → `DB_QUERY_CANCELED`. +transaction deadline (`TransactionTimedOutException`) → `DB_QUERY_CANCELED`; any other persistence +failure (JPA, Spring data-access or Spring transaction exception) of a transaction +`SpringTransaction` began, once its deadline has passed → `DB_QUERY_CANCELED`. **Commit phase (`translateCommitFailure`, ADR-JPA-003):** once the action has returned and `SpringTransaction` is committing a write transaction it began, a failure with `08*`, `40003`, a diff --git a/src/adapter/outbound/persistence-jpa/README.md b/src/adapter/outbound/persistence-jpa/README.md index 5c31cfbd..a2fc8b65 100644 --- a/src/adapter/outbound/persistence-jpa/README.md +++ b/src/adapter/outbound/persistence-jpa/README.md @@ -296,14 +296,26 @@ A가 FAILED로 바꾸면 다시 발행되고(중복 전달), A의 늦은 PUBLISH `WHERE event_id = ? AND status = 'IN_FLIGHT' AND lease_owner = ? AND lease_token = ?`. owner는 비우고 token은 그대로 둔다(token은 증가만 한다). 1 row면 `APPLIED`, 0 row면 `STALE`. `findById` + setter 경로와 `@Version`은 쓰지 않는다. +- **실패 원인 코드(`V22`)**: `markFailed`·`markDeferred`·`markDead`는 같은 조건부 UPDATE에서 + `last_failure_code`를 쓴다(publisher가 붙인 코드, 없으면 결과·예외 타입 이름). entity 필드는 읽기 + 전용이라 append INSERT와 claim flush는 이 열을 건드리지 않는다. `longRetries(n, now)`는 미룬 claim이 + n번 이상인 FAILED 행 수와 가장 오래된 행의 나이를 센다(`outbox.retry.long.*` 게이지). 상태 `PARKED`는 + DDL이 없다 — claim이 PENDING·FAILED·IN_FLIGHT만 고르고, FIFO 게이트는 PUBLISHED 아닌 모든 행을 막는 앞 + 행으로 본다. claim이 시도 횟수가 어긋난 행(`deferred_attempt_count > attempt_count`)을 만나면 DEAD로 + 옮기고 WARN `error.reason=ATTEMPT_COUNTERS_INCONSISTENT`로 원인을 말한다(재리뷰 2026-11-03 RR-05-01·03). - **미룬 실패와 claim 해제(`V20`)**: `markDeferred(lease, next)`는 `markFailed`와 같은 전이에 `deferred_attempt_count + 1`을 더한다(일시적 실패는 시도 예산 밖). `releaseLease(lease, at)`는 손대지 않은 claim을 `PENDING`, `next_attempt_at = at`, `attempt_count - 1`로 되돌린다(종료 중인 relay가 쓴다). 둘 다 같은 lease 조건부 UPDATE다. - **aggregate 순서 번호(`V21`)**: `JpaOutboxStore`는 `OutboxAggregateSequencePort`도 구현한다. - `outbox_aggregate_sequence` counter를 `UPDATE ... + 1`(row lock)로 올리고, 없으면 그 aggregate의 기존 - outbox 최대 `aggregate_sequence` + 1로 만든다. V21은 `outbox_event`의 순서 키 유일 인덱스도 만든다 — - 기존 중복 row가 있으면 migration이 실패하므로 먼저 정리해야 한다. + `outbox_aggregate_sequence` counter를 `UPDATE`(row lock)로 올린다. 올린 값은 그 aggregate의 가장 큰 + outbox 번호보다 작아지지 않는다(rolling 배포 중 옛 인스턴스가 쓴 벽시계 번호가 먼저 온다). counter가 + 없으면 `INSERT ... ON CONFLICT DO NOTHING`으로 만든다. 시작값은 남은 outbox 최대 번호와 V21이 + 기록한 floor(`outbox_aggregate_sequence_floor`, V21 적용 시각의 epoch 마이크로초) 중 큰 값 + 1이다. + 첫 쓰기 두 개가 겹치면 두 번째는 첫 번째 커밋을 기다렸다가 아무것도 넣지 않고 증가만 한다(리뷰 + RR-03-01). V21은 남은 row가 있는 aggregate의 counter를 그 최대 번호로 미리 만들고, `outbox_event`의 + 순서 키 유일 인덱스도 만든다 — 기존 중복 row가 있으면 migration이 실패하므로 먼저 정리해야 한다 + (`docs/runbooks/outbox-dead-letter.md` §8). - **STALE은 예외가 아니다**: row의 결과는 지금 lease를 가진 relay가 쓴다. use case는 결과를 `OutboxRelayResult.Outcome.STALE`로 남기고 `OutboxRelayFailureReportPort.reportStaleCompletion` 으로 알린 뒤 다음 이벤트로 넘어간다. FAILED/DEAD report는 보내지 않는다(아무것도 저장되지 diff --git a/src/adapter/outbound/persistence-jpa/build.gradle b/src/adapter/outbound/persistence-jpa/build.gradle index be148436..9a7e7a7c 100644 --- a/src/adapter/outbound/persistence-jpa/build.gradle +++ b/src/adapter/outbound/persistence-jpa/build.gradle @@ -187,6 +187,7 @@ def readinessLanes = [ postgresqlOutboxRowLifecycleIntegrationTest : ['PostgreSqlOutboxRowLifecycleIntegrationTest'], postgresqlOutboxAggregateSequenceIntegrationTest : ['PostgreSqlOutboxAggregateSequenceIntegrationTest'], postgresqlOutboxOrderKeyRunbookIntegrationTest : ['PostgreSqlOutboxOrderKeyRunbookIntegrationTest'], + postgresqlOutboxDispositionRunbookIntegrationTest : ['PostgreSqlOutboxDispositionRunbookIntegrationTest'], postgresqlFileserverMigrationIntegrationTest : ['PostgreSqlFileserverMigrationIntegrationTest'], postgresqlFileserverMetadataIntegrationTest : ['PostgreSqlFileserverMetadataStoreIntegrationTest'], postgresqlFileserverReclamationIntegrationTest : ['PostgreSqlFileserverReclamationIntegrationTest'], @@ -243,13 +244,20 @@ strictTestLanes { } } -// The V21 pre-deploy check and cleanup are SQL an operator runs from the outbox runbook (review -// 2026-11-01, README 15 decision). The lane reads them out of the runbook, so the document is one of -// its inputs: editing either block reruns the lane against real PostgreSQL. -tasks.named('postgresqlOutboxOrderKeyRunbookIntegrationTest', Test) { - def outboxRunbook = new File(rootProject.projectDir.parentFile, 'docs/runbooks/outbox-dead-letter.md') - inputs.file(outboxRunbook).withPathSensitivity(PathSensitivity.RELATIVE).withPropertyName('outboxRunbook') - systemProperty 'outbox.runbook', outboxRunbook.absolutePath +// The outbox runbooks give an operator SQL to run: the V21 pre-deploy check and cleanup (review +// 2026-11-01, README 15 decision), the manual dispositions of a DEAD row (re-review 2026-11-03 +// RR-05-01), and finding, parking, resuming and skipping a row that is retried without end +// (RR-05-03). Each lane reads its blocks out of the runbooks, so the documents are its inputs: +// editing a block reruns the lane against real PostgreSQL. +['postgresqlOutboxOrderKeyRunbookIntegrationTest', 'postgresqlOutboxDispositionRunbookIntegrationTest'].each { laneName -> + tasks.named(laneName, Test) { + def outboxRunbook = new File(rootProject.projectDir.parentFile, 'docs/runbooks/outbox-dead-letter.md') + inputs.file(outboxRunbook).withPathSensitivity(PathSensitivity.RELATIVE).withPropertyName('outboxRunbook') + systemProperty 'outbox.runbook', outboxRunbook.absolutePath + def publishFailedRunbook = new File(rootProject.projectDir.parentFile, 'docs/runbooks/outbox-publish-failed.md') + inputs.file(publishFailedRunbook).withPathSensitivity(PathSensitivity.RELATIVE).withPropertyName('publishFailedRunbook') + systemProperty 'outbox.publish-failed.runbook', publishFailedRunbook.absolutePath + } } tasks.named('postgresqlSecurityBaselineIntegrationTest') { diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/api/error/FailureCategory.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/api/error/FailureCategory.java index 5e7c55de..5df08727 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/api/error/FailureCategory.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/api/error/FailureCategory.java @@ -46,8 +46,15 @@ public enum FailureCategory { QUERY_TIMEOUT, /** - * {@code 25P03} — the server ended a session that sat idle inside an open transaction. Nothing - * was committed, which is why its code is retryable where {@link #QUERY_TIMEOUT}'s is not. + * Not a transaction timeout: {@code 25P03}, the server ended a session that sat idle inside an + * open transaction ({@code idle_in_transaction_session_timeout}). Nothing was committed, which is + * why its code ({@code DB_IDLE_IN_TX_TIMEOUT}) is retryable. + * + *

    A transaction that ran past its own deadline ({@code SpringTransaction.Timeouts}) is {@link + * #QUERY_TIMEOUT} ({@code DB_QUERY_CANCELED}), not this. The name says otherwise and is kept only + * because the enum is public API ({@code docs/architecture/jpa-api-surface.txt}); it becomes + * {@code IDLE_IN_TRANSACTION_TIMEOUT}, with {@link TransactionTimeoutException}, at the next + * release that breaks this API anyway (review RR-03-10). */ TRANSACTION_TIMEOUT, diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/api/error/TransactionTimeoutException.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/api/error/TransactionTimeoutException.java index 6af43fff..fd1b971d 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/api/error/TransactionTimeoutException.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/api/error/TransactionTimeoutException.java @@ -1,10 +1,13 @@ package dev.caskeleton.adapter.outbound.persistence.api.error; /** - * A transaction exceeded its configured transaction timeout (design §15.4). + * The server ended a session that sat idle inside an open transaction ({@code 25P03}, {@link + * FailureCategory#TRANSACTION_TIMEOUT}). Not a transaction that ran past its own deadline: that is + * a cancelled statement ({@link FailureCategory#QUERY_TIMEOUT}). * - *

    The transaction is rolled back; whether the use case may be re-run is a business decision, so - * the platform does not retry it automatically. + *

    The transaction is rolled back and nothing was committed, so its code is retryable; whether + * the use case is re-run is still the caller's decision, and the platform does not retry it + * automatically. The name is kept until the next API-breaking release (review RR-03-10). */ public final class TransactionTimeoutException extends JpaPersistenceException { diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/failure/PersistenceExceptionTranslator.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/failure/PersistenceExceptionTranslator.java index ceeabdc9..71820781 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/failure/PersistenceExceptionTranslator.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/failure/PersistenceExceptionTranslator.java @@ -8,6 +8,7 @@ import dev.caskeleton.shared.error.ApiErrorCode; import dev.caskeleton.shared.error.OperationalError; import dev.caskeleton.shared.error.PersistenceFailureException; import jakarta.persistence.OptimisticLockException; +import jakarta.persistence.PersistenceException; import java.io.IOException; import java.sql.SQLException; import java.sql.SQLNonTransientConnectionException; @@ -23,6 +24,7 @@ import java.util.Optional; import java.util.Set; import org.springframework.dao.DataAccessException; import org.springframework.dao.OptimisticLockingFailureException; +import org.springframework.transaction.TransactionException; import org.springframework.transaction.TransactionTimedOutException; /** @@ -140,6 +142,42 @@ public final class PersistenceExceptionTranslator { return Optional.empty(); } + /** + * Classify a failure raised once the transaction's deadline had passed, when neither a SQLState + * nor a type above identifies it. + * + *

    Past the deadline Hibernate refuses the next statement itself, before the driver is asked: + * an {@code org.hibernate.TransactionException} with no SQLState that is not Spring's {@link + * TransactionTimedOutException} — in the action as it is, at commit wrapped in a {@code + * JpaSystemException} (review RR-03-03). That is the production path ({@code + * JpaTransactionManager}); the JDBC path gets {@code 57014} from the server instead. Both are + * {@code DB_QUERY_CANCELED}. The caller decides that the deadline has passed; this method only + * makes sure the failure came from the persistence stack — a JPA, Spring data-access or Spring + * transaction exception anywhere in the cause chain — so a business exception thrown late is + * passed on as it is, and one already classified is not classified again. + */ + public Optional translateAfterDeadline(Throwable ex) { + Objects.requireNonNull(ex, "ex"); + if (ex instanceof PersistenceFailureException) { + return Optional.empty(); + } + for (Throwable t = ex; t != null; t = t.getCause()) { + if (t instanceof PersistenceException + || t instanceof DataAccessException + || t instanceof TransactionException) { + return Optional.of( + new PersistenceFailureException( + OperationalError.DB_QUERY_CANCELED, + "persistence failure after the transaction deadline passed", + ex)); + } + if (t.getCause() == t) { + break; + } + } + return Optional.empty(); + } + /** * Classify a failure raised while committing a write transaction this call began. * diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStore.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStore.java index 40b83209..fba06bfb 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStore.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStore.java @@ -14,6 +14,7 @@ import dev.caskeleton.application.outbox.OutboxAppendPort; import dev.caskeleton.application.outbox.OutboxCompletion; import dev.caskeleton.application.outbox.OutboxEventStatus; import dev.caskeleton.application.outbox.OutboxLease; +import dev.caskeleton.application.outbox.OutboxLongRetries; import dev.caskeleton.application.outbox.OutboxStorePort; import dev.caskeleton.shared.digest.Sha256; import java.nio.charset.StandardCharsets; @@ -29,6 +30,7 @@ import java.util.Objects; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.spi.LoggingEventBuilder; /** * JPA-backed {@link OutboxAppendPort} + {@link OutboxStorePort}. {@link #append} and the claim/mark @@ -103,12 +105,21 @@ public class JpaOutboxStore * *

    The increment is an {@code UPDATE}, which holds the counter row's lock until the caller * commits: a second writer of the same aggregate waits, then increments the committed value, so - * the numbers follow commit order. An aggregate's first number is inserted instead, seeded after - * the highest {@code aggregate_sequence} its outbox rows already carry — rows numbered before the - * counter existed (the sample used epoch microseconds) must still come first. Two writers - * creating the same counter at once collide on its primary key and one transaction fails: a - * rollback the caller retries, never two events with one number. The counter outlives the - * reaper's deletes, so a number is never handed out twice. + * the numbers follow commit order. It never goes below the aggregate's highest outbox row, so a + * row an instance still on the wall-clock scheme writes during a rolling deploy comes first. The + * counter outlives the reaper's deletes, so a number is never handed out twice. + * + *

    An aggregate's first number is inserted instead, after the larger of its highest remaining + * row and the floor V21 recorded — the moment it ran, above every clock number already sent for + * rows the reaper has deleted or that were written under an identity that has since changed + * (review RR-03-02). Two writers creating the same counter at once both succeed: the insert is + * {@code ON CONFLICT DO NOTHING}, so the second waits for the first to commit, writes nothing, + * and increments the committed counter instead (review RR-03-01). + * + *

    Statements rather than one {@code INSERT … ON CONFLICT DO UPDATE … RETURNING} like {@code + * JpaLiveEventReplay}'s position allocator: the sample appends events on the H2 developer run + * ({@code bootRunH2}), and H2 accepts {@code ON CONFLICT DO NOTHING} but not {@code DO UPDATE}. + * The common case — the counter exists — is still one {@code UPDATE} and one read. */ @Override public long nextSequence(AggregateIdentity aggregate) { @@ -118,8 +129,11 @@ public class JpaOutboxStore String tenant = aggregate.tenantScope(); String type = aggregate.aggregateType(); String id = aggregate.aggregateId(); - if (repository.incrementAggregateSequence(tenant, type, id) == 0) { - repository.createAggregateSequence(tenant, type, id); + if (repository.incrementAggregateSequence(tenant, type, id) == 0 + && repository.createAggregateSequence(tenant, type, id) == 0) { + // Another writer created the counter after our UPDATE found none; its commit ended our wait + // in the insert, so the counter is there to increment now. + repository.incrementAggregateSequence(tenant, type, id); } return repository.aggregateSequence(tenant, type, id); } @@ -132,6 +146,11 @@ public class JpaOutboxStore * instead rolled the whole claim back, and the next claim selected the same row first again, so * one bad row stopped the relay for good. DEAD is fail-closed: the row is never published, and it * waits for the DEAD-row runbook like any other. + * + *

    The WARN names the cause. A row whose deferred claims outnumber its claims is not damaged: + * it is what a manual re-drive that reset {@code attempt_count} and left {@code + * deferred_attempt_count} produces (re-review 2026-11-03 RR-05-01), and "cannot read" sent the + * operator looking for corruption instead of re-running the corrected runbook SQL. */ @Override public List claimBatch( @@ -145,13 +164,26 @@ public class JpaOutboxStore entity.setNextAttemptAt(now.plus(inFlightTimeout)); entity.setLeaseOwner(owner); entity.setLeaseToken(entity.getLeaseToken() + 1); + if (entity.getDeferredAttemptCount() > entity.getAttemptCount()) { + deadLetterOnClaim(entity) + .addKeyValue("error.reason", "ATTEMPT_COUNTERS_INCONSISTENT") + .log( + "outbox claim moved a row to DEAD: deferred_attempt_count ({}) exceeds" + + " attempt_count ({}); a manual re-drive that did not reset" + + " deferred_attempt_count leaves this behind — re-drive it with the runbook" + + " SQL", + entity.getDeferredAttemptCount(), + entity.getAttemptCount()); + continue; + } ClaimedOutboxEvent event; try { event = toClaimedOutboxEvent(entity); } catch (RuntimeException unreadable) { - entity.setStatus(OutboxEventStatus.DEAD.name()); - entity.setLeaseOwner(null); - log.warn("outbox claim moved a row it cannot read to DEAD", unreadable); + deadLetterOnClaim(entity) + .addKeyValue("error.reason", "UNREADABLE_ROW") + .setCause(unreadable) + .log("outbox claim moved a row it cannot read as an event to DEAD"); continue; } leases.add(new OutboxLease(event, owner, entity.getLeaseToken())); @@ -159,6 +191,15 @@ public class JpaOutboxStore return List.copyOf(leases); } + /** Moves a claimed row it will not hand out to DEAD and starts the WARN that says why. */ + private static LoggingEventBuilder deadLetterOnClaim(OutboxEventEntity entity) { + entity.setStatus(OutboxEventStatus.DEAD.name()); + entity.setLeaseOwner(null); + return log.atWarn() + .addKeyValue("event_id", entity.getEventId()) + .addKeyValue("runbook_link", "runbook://outbox/dead-letter"); + } + /** * One conditional UPDATE, no load-and-set: it applies only while the lease still holds the row * (see README "outbox"). Zero rows is {@link OutboxCompletion#STALE}, not an error — the relay @@ -174,18 +215,21 @@ public class JpaOutboxStore /** Marks the leased row FAILED with its next attempt time; see {@link #markPublished}. */ @Override - public OutboxCompletion markFailed(OutboxLease lease, Instant nextAttemptAt) { + public OutboxCompletion markFailed(OutboxLease lease, Instant nextAttemptAt, String failureCode) { requireTransaction("markFailed"); return completion( - repository.failLease(lease.eventId(), lease.owner(), lease.token(), nextAttemptAt)); + repository.failLease( + lease.eventId(), lease.owner(), lease.token(), nextAttemptAt, failureCode)); } /** Marks the leased row FAILED as a deferred claim; see {@link #markPublished}. */ @Override - public OutboxCompletion markDeferred(OutboxLease lease, Instant nextAttemptAt) { + public OutboxCompletion markDeferred( + OutboxLease lease, Instant nextAttemptAt, String failureCode) { requireTransaction("markDeferred"); return completion( - repository.deferLease(lease.eventId(), lease.owner(), lease.token(), nextAttemptAt)); + repository.deferLease( + lease.eventId(), lease.owner(), lease.token(), nextAttemptAt, failureCode)); } /** Returns the leased row to PENDING without spending an attempt; see {@link #markPublished}. */ @@ -198,11 +242,10 @@ public class JpaOutboxStore /** Marks the leased row DEAD; see {@link #markPublished}. */ @Override - public OutboxCompletion markDead(OutboxLease lease) { + public OutboxCompletion markDead(OutboxLease lease, String failureCode) { requireTransaction("markDead"); return completion( - repository.completeLease( - lease.eventId(), lease.owner(), lease.token(), OutboxEventStatus.DEAD.name())); + repository.deadLease(lease.eventId(), lease.owner(), lease.token(), failureCode)); } private static OutboxCompletion completion(int updatedRows) { @@ -237,6 +280,21 @@ public class JpaOutboxStore return result; } + @Override + public OutboxLongRetries longRetries(int minDeferredClaims, Instant now) { + if (minDeferredClaims < 1) { + throw new IllegalArgumentException( + "minDeferredClaims must be >= 1, was " + minDeferredClaims); + } + Object[] row = repository.countLongRetries(minDeferredClaims).getFirst(); + long count = (Long) row[0]; + if (count == 0) { + return OutboxLongRetries.NONE; + } + Instant oldest = (Instant) row[1]; + return new OutboxLongRetries(count, Math.max(0, Duration.between(oldest, now).toSeconds())); + } + /** * Restores the appended event. A row without its envelope — written by the pre-canonical * generation, or damaged — is unreadable, and so is one whose stored hash is not the hash of its diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxEventJpaRepository.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxEventJpaRepository.java index 87b0c782..9f4d6d0c 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxEventJpaRepository.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/OutboxEventJpaRepository.java @@ -62,18 +62,22 @@ public interface OutboxEventJpaRepository extends Repository 'PUBLISHED' group by e.eventType") List findOldestUnpublishedOccurredAtByEventType(); + + /** + * {@code [count (Long), oldestOccurredAt (Instant)]} of the FAILED rows with at least {@code + * minDeferredClaims} deferred claims, for the long-retry gauges. One row, whose instant is null + * when the count is 0. + */ + @Query( + "select count(e), min(e.occurredAt) from OutboxEventEntity e " + + "where e.status = 'FAILED' and e.deferredAttemptCount >= :minDeferredClaims") + List countLongRetries(@Param("minDeferredClaims") int minDeferredClaims); } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/entity/OutboxAggregateSequenceFloorEntity.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/entity/OutboxAggregateSequenceFloorEntity.java new file mode 100644 index 00000000..fb79e5d7 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/entity/OutboxAggregateSequenceFloorEntity.java @@ -0,0 +1,32 @@ +package dev.caskeleton.adapter.outbound.persistence.outbox.jpa.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; + +/** + * The one-row floor a new aggregate counter starts after ({@code outbox_aggregate_sequence_floor}, + * V21): the moment V21 ran, in epoch microseconds, above every number the replaced wall-clock + * scheme wrote. Read by {@code JpaOutboxStore#nextSequence} with a native statement; mapped, like + * {@link OutboxAggregateSequenceEntity}, so the table exists where Hibernate builds the schema (H2, + * where no migration writes the row and the floor is 0) and is validated where Flyway does. + * Package-private: no code reads it as an entity. + */ +@Entity +@Table(name = "outbox_aggregate_sequence_floor") +class OutboxAggregateSequenceFloorEntity { + + @Id + @Column(name = "singleton", nullable = false) + private boolean singleton; + + @Column(name = "floor_sequence", nullable = false) + private long floorSequence; + + protected OutboxAggregateSequenceFloorEntity() {} + + long floorSequence() { + return floorSequence; + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/entity/OutboxEventEntity.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/entity/OutboxEventEntity.java index 72a6a537..4ed084a8 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/entity/OutboxEventEntity.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/entity/OutboxEventEntity.java @@ -58,6 +58,18 @@ public class OutboxEventEntity implements Persistable { private int deferredAttemptCount; /** Dual-purpose per status (PENDING / IN_FLIGHT / FAILED) — see README "outbox". */ + /** + * The code of the row's last failed completion (V22): the publisher's own name for the failure, + * or the outcome or exception type when it gave none. + * + *

    Read-only to the entity: only the relay's failed completions write it, each in its one + * fenced UPDATE ({@code OutboxEventJpaRepository#failLease} and its siblings). The append's + * INSERT and the claim's flush leave it out, so they neither clear a code nor need the column on + * a database that has not run V22 yet. + */ + @Column(name = "last_failure_code", length = 120, insertable = false, updatable = false) + private String lastFailureCode; + @Column(name = "next_attempt_at", nullable = false) private Instant nextAttemptAt; @@ -379,4 +391,8 @@ public class OutboxEventEntity implements Persistable { public void setCausationId(String causationId) { this.causationId = causationId; } + + public String getLastFailureCode() { + return lastFailureCode; + } } diff --git a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransaction.java b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransaction.java index 911ff56c..aad85fd9 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransaction.java +++ b/src/adapter/outbound/persistence-jpa/src/main/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransaction.java @@ -123,7 +123,9 @@ public final class SpringTransaction implements TransactionPort { * dry and every other request fails at {@code connection-timeout}. With one, Spring hands the * remaining time to every statement (Hibernate and {@code JdbcTemplate} both set the JDBC query * timeout), so the server cancels the statement ({@code 57014}, {@code DB_QUERY_CANCELED}) and - * the connection comes back. + * the connection comes back. A statement Hibernate is asked to run after the deadline has passed + * is refused by Hibernate itself, with no SQLState; this class reports that as {@code + * DB_QUERY_CANCELED} too (review RR-03-03). * *

    The default is 30 seconds for both: the budget {@code inNew} already has, and the Hikari * {@code leak-detection-threshold} at which the pool already reports a held connection as a leak. @@ -194,6 +196,9 @@ public final class SpringTransaction implements TransactionPort { private T execute(ModeTemplate mode, Supplier action) { long startedAt = System.nanoTime(); + // Whether this call began the transaction, and so owns its deadline. A call that joined an + // outer one leaves the deadline to the call that began it. + AtomicBoolean beganTransaction = new AtomicBoolean(); // Set only once the action has returned in a write transaction this call began: from then on // the only thing TransactionTemplate.execute still does is commit it. AtomicBoolean committingOwnWrite = new AtomicBoolean(); @@ -201,20 +206,32 @@ public final class SpringTransaction implements TransactionPort { return mode.template() .execute( status -> { + beganTransaction.set(status.isNewTransaction()); T result = action.get(); committingOwnWrite.set(status.isNewTransaction() && !mode.template().isReadOnly()); return result; }); } catch (RuntimeException failure) { + Duration elapsed = Duration.ofNanos(System.nanoTime() - startedAt); Optional translated = committingOwnWrite.get() - ? exceptionTranslator.translateCommitFailure( - failure, mode.commitOperation(), Duration.ofNanos(System.nanoTime() - startedAt)) + ? exceptionTranslator.translateCommitFailure(failure, mode.commitOperation(), elapsed) : exceptionTranslator.translate(failure); + if (translated.isEmpty() && beganTransaction.get() && pastDeadline(mode, elapsed)) { + // Hibernate's own deadline refusal carries no SQLState (review RR-03-03). + translated = exceptionTranslator.translateAfterDeadline(failure); + } throw translated.map(RuntimeException.class::cast).orElse(failure); } } + /** Whether {@code elapsed} reached the timeout the mode's transactions run under. */ + private static boolean pastDeadline(ModeTemplate mode, Duration elapsed) { + int timeoutSeconds = mode.template().getTimeout(); + return timeoutSeconds != TransactionDefinition.TIMEOUT_DEFAULT + && elapsed.compareTo(Duration.ofSeconds(timeoutSeconds)) >= 0; + } + /** * The vendor-neutral default translator. * diff --git a/src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V21__outbox_aggregate_sequence.sql b/src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V21__outbox_aggregate_sequence.sql index f22aa20f..c19a2ce2 100644 --- a/src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V21__outbox_aggregate_sequence.sql +++ b/src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V21__outbox_aggregate_sequence.sql @@ -7,8 +7,32 @@ -- -- outbox_aggregate_sequence holds one counter per aggregate. Incrementing it locks the row until -- the writer commits, so the numbers follow commit order; it outlives the reaper's deletes, so a --- number is never reused. JpaOutboxAggregateSequences seeds a new counter from the aggregate's --- highest existing aggregate_sequence, so rows numbered by the old clock scheme still come first. +-- number is never reused. JpaOutboxStore#nextSequence reads and writes it. +-- +-- Consumers have already seen clock numbers (about 1.7e15) for aggregates whose rows this +-- database no longer holds: the reaper deleted them, or they were written under an identity that +-- has since changed (the sample's tenant scope). A counter that started after the rows still +-- present would start at 1 for those aggregates, behind numbers already sent (review RR-03-02, +-- RR-05-02). So this migration records one floor, the moment it runs in epoch microseconds, which +-- is above every clock number written before it; a counter created later starts after the floor +-- or after the aggregate's highest remaining row, whichever is larger. The aggregates that still +-- have rows get their counter here, at their highest number. A clock number an instance still on +-- the old version writes during the rollout is above the floor; nextSequence also never goes below +-- the highest row it can see, so such a row still comes first. +-- +-- This file was edited after it was first written, before any deployment applied it (the branch +-- that added it was not merged): splitting the floor into a V22 would have left V21 alone handing +-- out numbers from 1 for one release, and its comment pointed at a class that does not exist. +-- applied-migrations.sha256 carries the new checksum. A development database that applied the +-- earlier text fails Flyway validation; recreate it. + +-- Bound the wait for the table lock the unique index below takes (SHARE on outbox_event, held until +-- this migration commits). Without a bound, the migration queues behind any open outbox write and +-- every outbox write after it queues behind the migration. Five seconds, then this migration fails, +-- rolls back whole and the database stays at V20: deploy again, or build the index first +-- (runbook §8, CONCURRENTLY). +SET LOCAL lock_timeout = '5s'; + CREATE TABLE IF NOT EXISTS outbox_aggregate_sequence ( tenant_scope VARCHAR(96) NOT NULL, aggregate_type VARCHAR(64) NOT NULL, @@ -18,10 +42,55 @@ CREATE TABLE IF NOT EXISTS outbox_aggregate_sequence ( CONSTRAINT ck_outbox_aggregate_sequence_positive CHECK (last_sequence >= 1) ); +-- One row, written once: the lowest number a counter created from now on may start after. +CREATE TABLE IF NOT EXISTS outbox_aggregate_sequence_floor ( + singleton BOOLEAN NOT NULL DEFAULT TRUE, + floor_sequence BIGINT NOT NULL, + CONSTRAINT pk_outbox_aggregate_sequence_floor PRIMARY KEY (singleton), + CONSTRAINT ck_outbox_aggregate_sequence_floor_singleton CHECK (singleton), + CONSTRAINT ck_outbox_aggregate_sequence_floor_non_negative CHECK (floor_sequence >= 0) +); + +INSERT INTO outbox_aggregate_sequence_floor (singleton, floor_sequence) +VALUES (TRUE, (extract(epoch FROM clock_timestamp()) * 1000000)::BIGINT) +ON CONFLICT (singleton) DO NOTHING; + +INSERT INTO outbox_aggregate_sequence (tenant_scope, aggregate_type, aggregate_id, last_sequence) +SELECT tenant_scope, aggregate_type, aggregate_id, MAX(aggregate_sequence) + FROM outbox_event + WHERE tenant_scope IS NOT NULL + AND aggregate_type IS NOT NULL + AND aggregate_sequence IS NOT NULL + GROUP BY tenant_scope, aggregate_type, aggregate_id +ON CONFLICT (tenant_scope, aggregate_type, aggregate_id) DO NOTHING; + -- One number, one event: a second event appended under an order its aggregate already used is -- refused at append time, in the writer's transaction, rather than released by the gate in no -- defined order. Rows without canonical order columns (pre-canonical, all NULL) are not compared. -- This fails if duplicates already exist; resolve them (they are exactly the rows the gate could --- not order) before migrating. +-- not order) before migrating (runbook §8). +-- +-- Built here, inside this migration's transaction, not CONCURRENTLY: a duplicate then rolls the +-- whole of V21 back and leaves the database at V20 with nothing to clean up, which is what the +-- runbook's retry relies on. A failed CONCURRENTLY build leaves an INVALID index that IF NOT EXISTS +-- would then skip, so the uniqueness would silently not hold; and CONCURRENTLY cannot run in the +-- transaction that writes the counters and the floor. The reaper bounds outbox_event to the +-- retention window, so the build is normally short. An outbox too large to lock for that long is +-- indexed first with the runbook's CONCURRENTLY statement (same name, same columns); the check +-- below refuses an index such a build left invalid, and IF NOT EXISTS keeps the valid one. +DO $$ +BEGIN + IF EXISTS (SELECT 1 + FROM pg_index + WHERE indexrelid = to_regclass('ux_outbox_event_aggregate_order') + AND NOT (indisvalid AND indisunique)) THEN + RAISE EXCEPTION 'ux_outbox_event_aggregate_order exists but is not a valid unique index' + USING HINT = 'A CREATE INDEX CONCURRENTLY failed or is still running. ' + || 'DROP INDEX CONCURRENTLY ux_outbox_event_aggregate_order, then rebuild it ' + || 'or let V21 build it (runbook outbox-dead-letter §8).'; + END IF; +END +$$; + CREATE UNIQUE INDEX IF NOT EXISTS ux_outbox_event_aggregate_order ON outbox_event (tenant_scope, aggregate_type, aggregate_id, aggregate_sequence, event_index); diff --git a/src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V22__outbox_event_last_failure_code.sql b/src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V22__outbox_event_last_failure_code.sql new file mode 100644 index 00000000..dbe4ed59 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/main/resources/db/migration/postgresql/V22__outbox_event_last_failure_code.sql @@ -0,0 +1,21 @@ +-- Why a row keeps failing, kept on the row. +-- +-- A publish refused for now (a closed transport, a credential refused) or left unanswered is +-- retried without spending the attempt budget (V20), so it has no end: the relay retries it five +-- minutes apart for as long as nobody looks. The outcome the relay records cannot say why — a +-- shutdown, a closed transport and a refused credential are one outcome — and the relay's log line +-- said it in one sentence for all four. The publisher's failure code can (a messaging platform code +-- such as KAFKA_AUTHENTICATIONEXCEPTION); every failed completion writes it here, so the runbook's +-- query for rows that have been retrying long shows what each one is waiting on (re-review +-- 2026-11-03 RR-05-03). +-- +-- A code, never a message: a driver message can carry an endpoint, a statement or a credential. +-- Nullable with no default, so adding it rewrites no row; rows written before it stay NULL until +-- their next failure. +-- +-- The new status PARKED needs no DDL: status has no CHECK, the claim selects PENDING, FAILED and +-- IN_FLIGHT only (so the partial index ix_outbox_event_eligible leaves PARKED out), and the order +-- gate and its index ix_outbox_event_unpublished_aggregate_order treat every status but PUBLISHED as +-- blocking — which is what holding a row in its aggregate's place means. +ALTER TABLE outbox_event + ADD COLUMN last_failure_code VARCHAR(120); diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/WorkerCrashRecoveryContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/WorkerCrashRecoveryContractTest.java index 0cddeb3c..a3af1b93 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/WorkerCrashRecoveryContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/notification/WorkerCrashRecoveryContractTest.java @@ -23,7 +23,8 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; /** - * What a worker's death leaves behind, at each phase of a dispatch. + * What a worker's death leaves behind, at each phase of a dispatch — the rows, not what recovery + * does with them. * *

    This is the {@code recoverable} claim in {@code docs/notification/evidence-manifest.json}, and * until now nothing backed it. Lease recovery and event replay had unit coverage against fakes; the @@ -36,9 +37,13 @@ import org.junit.jupiter.api.Test; * worker is from the database's point of view — a lease nobody will renew and a phase nobody will * advance. * - *

    The matrix walks the phases a dispatch passes through. For each, the question is the same and - * has only two acceptable answers: the delivery becomes claimable again, or it becomes a question - * for the provider. Never both, and never neither. + *

    The matrix walks the phases a dispatch passes through and pins, for each, the state recovery + * starts from: a lease that has passed (the claim takes the row again) and, from the attempt insert + * on, an unfinished attempt row carrying the evidence of that phase (what reconciliation asks the + * provider about). These rows are written by this test, so the tests check the shape of the input + * and the predicates recovery reads, not recovery itself. {@code + * PostgreSqlNotificationBackgroundPassIntegrationTest} runs recovery and reconciliation over rows + * the dispatcher's own factory wrote (review 05-01, RR-03-07). */ @Tag("jpa-contract") class WorkerCrashRecoveryContractTest { @@ -73,8 +78,8 @@ class WorkerCrashRecoveryContractTest { } @Test - @DisplayName("a crash after claiming and before any attempt leaves the delivery claimable") - void aCrashBeforeAnyAttemptLeavesItClaimable() throws SQLException { + @DisplayName("claim 뒤 attempt 전에 죽은 delivery는 attempt 행이 없고 lease가 지나 있다") + void aDeliveryDeadBeforeAnyAttemptHasNoAttemptRowAndAPassedLease() throws SQLException { UUID delivery = deliveryHeldByADeadWorker(); assertThat(isClaimableAgain(delivery)) @@ -88,12 +93,12 @@ class WorkerCrashRecoveryContractTest { /** * The attempt row is written before the call and keeps the factory's "unknown" evidence for the * whole call (review 05-01), so a crash anywhere between the insert and the provider's answer - * leaves the same row. Recovery cannot tell "never called" from "called and died", so the row - * goes to reconciliation rather than back to the queue. + * leaves the same row. This test writes that row itself; that recovery hands it to reconciliation + * is {@code PostgreSqlNotificationBackgroundPassIntegrationTest}'s. */ @Test - @DisplayName("attempt 행을 쓴 뒤 호출이 끝나기 전 죽으면 시작 여부 불명인 미완료 attempt 가 남아 reconcile 대상이 된다") - void aCrashAfterTheAttemptRowIsWrittenLeavesAnUnfinishedAttemptOfUnknownStart() + @DisplayName("attempt 행을 쓰고 호출 전에 죽은 delivery는 시작 여부 불명인 미완료 attempt와 지난 lease를 남긴다") + void aDeliveryDeadAfterTheAttemptInsertLeavesAnUnfinishedAttemptOfUnknownStart() throws SQLException { // given UUID delivery = deliveryHeldByADeadWorker(); @@ -103,17 +108,18 @@ class WorkerCrashRecoveryContractTest { // then assertThat(evidenceOf(attempt)) - .as("the row the factory writes records that whether the call started is not known") + .as("the row stands for the factory's: whether the call started is not known") .containsExactly("false", "UNKNOWN", "false", "UNKNOWN"); assertThat(isUnfinished(attempt)) - .as("an unfinished attempt is what recovery hands to reconciliation") + .as("completed_at is empty, the predicate recovery selects unfinished attempts by") .isTrue(); assertThat(isClaimableAgain(delivery)).isTrue(); } @Test - @DisplayName("a crash after the request started is a question for the provider, not a requeue") - void aCrashAfterTheRequestStartedIsAQuestion() throws SQLException { + @DisplayName("요청 시작 뒤 죽은 delivery는 시작 증거가 있는 미완료 attempt와 지난 lease를 남긴다") + void aDeliveryDeadAfterTheRequestStartedLeavesAnUnfinishedAttemptWithStartEvidence() + throws SQLException { UUID delivery = deliveryHeldByADeadWorker(); UUID attempt = attemptOn(delivery, true, "PROVEN", false, "UNKNOWN"); @@ -125,13 +131,14 @@ class WorkerCrashRecoveryContractTest { "the lease still expires — the row must not be stranded — but the attempt is unfinished") .isTrue(); assertThat(isUnfinished(attempt)) - .as("an unfinished attempt is what recovery hands to reconciliation") + .as("completed_at is empty, the predicate recovery selects unfinished attempts by") .isTrue(); } @Test - @DisplayName("a crash after the body was committed is never silently retried") - void aCrashAfterTheBodyWasCommittedIsNeverSilentlyRetried() throws SQLException { + @DisplayName("본문 전송 뒤 죽은 delivery는 전송 증거가 있는 미완료 attempt를 남긴다") + void aDeliveryDeadAfterTheBodyWasSentLeavesAnUnfinishedAttemptWithSendEvidence() + throws SQLException { UUID delivery = deliveryHeldByADeadWorker(); UUID attempt = attemptOn(delivery, true, "PROVEN", true, "PROVEN"); @@ -142,8 +149,8 @@ class WorkerCrashRecoveryContractTest { } @Test - @DisplayName("a completed attempt is not recovered at all") - void aCompletedAttemptIsNotRecovered() throws SQLException { + @DisplayName("완료 시각이 적힌 attempt는 미완료로 보이지 않는다") + void aCompletedAttemptIsNotUnfinished() throws SQLException { UUID delivery = deliveryHeldByADeadWorker(); UUID attempt = attemptOn(delivery, true, "PROVEN", true, "PROVEN"); complete(attempt); @@ -154,8 +161,9 @@ class WorkerCrashRecoveryContractTest { } @Test - @DisplayName("across every phase, exactly one delivery row survives with one attempt each") - void acrossEveryPhaseNothingIsLostOrDuplicated() throws SQLException { + @DisplayName("네 단계에서 죽은 워커들은 delivery 4개와 attempt 3개를 남기고 모든 lease가 지나 있다") + void deadWorkersAcrossEveryPhaseLeaveFourDeliveriesThreeAttemptsAndPassedLeases() + throws SQLException { List deliveries = new ArrayList<>(); // The four crash phases, run together, so the recovery sweep sees a mixed table rather than one // hand-picked row. diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSubmissionOutboxIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSubmissionOutboxIntegrationTest.java index fa0a4210..55db6eb9 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSubmissionOutboxIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/operation/DurableOperationSubmissionOutboxIntegrationTest.java @@ -59,7 +59,14 @@ import org.springframework.transaction.PlatformTransactionManager; class DurableOperationSubmissionOutboxIntegrationTest { private static final Instant NOW = Instant.parse("2026-08-25T09:00:00Z"); - private static final DurableOperationId ID = new DurableOperationId("op-1"); + + /** + * A canonical lower-case UUIDv7, as the HTTP adapter mints: the operation id is then the event id + * and the aggregate id itself (an id of any other spelling becomes a UUIDv7 derived from it). + */ + private static final String OPERATION_UUID = "01994e11-4d88-7000-8000-0000000000a1"; + + private static final DurableOperationId ID = new DurableOperationId(OPERATION_UUID); private static JpaPlatformContractSupport support; private static AnnotationConfigApplicationContext context; @@ -103,7 +110,7 @@ class DurableOperationSubmissionOutboxIntegrationTest { // then assertThat(queryStrings("SELECT contract_id || '|' || aggregate_id FROM outbox_event")) - .containsExactly("durable-operation.submitted|op-1"); + .containsExactly("durable-operation.submitted|" + OPERATION_UUID); } @Test diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/FileserverPostgreSqlStack.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/FileserverPostgreSqlStack.java new file mode 100644 index 00000000..a6e77b62 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/FileserverPostgreSqlStack.java @@ -0,0 +1,159 @@ +package dev.caskeleton.adapter.outbound.persistence.readiness; + +import dev.caskeleton.adapter.outbound.persistence.fileserver.JpaFileMetadata; +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.api.StorageNamespace; +import dev.caskeleton.application.fileserver.api.metadata.FileOwner; +import dev.caskeleton.application.fileserver.api.metadata.FileRecordDraft; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.UUID; +import java.util.function.Supplier; +import org.flywaydb.core.Flyway; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.support.TransactionTemplate; + +/** + * One real PostgreSQL with the Fileserver schema and stores, for the port contract tests that hold + * the JPA stores to the cases their in-memory fakes pass (review 2026-11-03 RR-01-04). + * + *

    The stores read a clock this stack owns and moves, so a contract case about expiry passes time + * the way it does against a fake instead of sleeping. One server per test class; {@link #empty()} + * makes each case independent of the one before it. + */ +final class FileserverPostgreSqlStack implements AutoCloseable { + + private final PostgreSqlReadinessSupport postgres; + private final MovableClock clock; + private final FileserverJpaTestContext context; + private final TransactionTemplate transactions; + private final JdbcTemplate jdbc; + + private FileserverPostgreSqlStack(PostgreSqlReadinessSupport postgres) { + this.postgres = postgres; + migrate("classpath:db/migration/postgresql", "flyway_schema_history", false, null); + migrate( + "classpath:db/migration/jpa/core", + "flyway_jpa_core_history", + true, + "explicit-jpa-core-adoption"); + migrate( + "classpath:db/migration/jpa/fileserver", + "flyway_jpa_fileserver_history", + true, + "explicit-jpa-fileserver-adoption"); + this.clock = new MovableClock(Instant.now().truncatedTo(ChronoUnit.MICROS)); + this.context = new FileserverJpaTestContext(postgres.dataSource(), clock); + this.transactions = context.transactions(); + this.jdbc = new JdbcTemplate(postgres.dataSource()); + } + + /** Starts a server, applies the three schema streams and builds the Fileserver stores. */ + static FileserverPostgreSqlStack start() { + PostgreSqlReadinessSupport.assertDockerAvailable(); + return new FileserverPostgreSqlStack(PostgreSqlReadinessSupport.start(8, 2_000)); + } + + T bean(Class type) { + return context.bean(type); + } + + /** Runs {@code work} in one committed transaction, as the application's transaction port does. */ + T inTransaction(Supplier work) { + return transactions.execute(status -> work.get()); + } + + /** The instant the stores read as now. */ + Instant now() { + return clock.instant(); + } + + /** Moves the stores' clock forward. */ + void elapse(Duration duration) { + clock.advance(duration); + } + + /** Records a file in {@code namespace}, so rows that reference a file may name it. */ + FileId recordFileIn(StorageNamespace namespace) { + FileId fileId = FileId.of(UUID.randomUUID()); + inTransaction( + () -> + bean(JpaFileMetadata.class) + .insert( + new FileRecordDraft( + fileId, + namespace, + "report.bin", + Optional.of("application/octet-stream"), + OptionalLong.of(1_000), + new FileOwner("user-1", Optional.empty())))); + return fileId; + } + + /** Deletes every Fileserver row, children first. */ + void empty() { + jdbc.execute( + "TRUNCATE fs_recovery_item, fs_cleanup_item, fs_quota_reservation," + + " fs_verification_result, fs_upload_session, fs_file CASCADE"); + } + + @Override + public void close() { + context.close(); + postgres.close(); + } + + private void migrate( + String location, String historyTable, boolean independent, String baselineDescription) { + var configuration = + Flyway.configure() + .dataSource(postgres.dataSource()) + .locations(location) + .table(historyTable) + .baselineOnMigrate(false) + .outOfOrder(false); + if (independent) { + configuration.baselineVersion("0").baselineDescription(baselineDescription); + } + Flyway flyway = configuration.load(); + if (independent) { + flyway.baseline(); + } + flyway.migrate(); + } + + /** A clock a test moves forward; microsecond precision, as PostgreSQL stores timestamps. */ + private static final class MovableClock extends Clock { + + private Instant now; + + private MovableClock(Instant start) { + this.now = start; + } + + private synchronized void advance(Duration duration) { + now = now.plus(duration); + } + + @Override + public synchronized Instant instant() { + return now; + } + + @Override + public ZoneId getZone() { + return ZoneOffset.UTC; + } + + @Override + public Clock withZone(ZoneId zone) { + throw new UnsupportedOperationException("the stack's clock is UTC"); + } + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/OutboxRunbookSql.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/OutboxRunbookSql.java new file mode 100644 index 00000000..395ffcc9 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/OutboxRunbookSql.java @@ -0,0 +1,80 @@ +package dev.caskeleton.adapter.outbound.persistence.readiness; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +/** + * The SQL an outbox runbook gives an operator, read out of the runbook itself. + * + *

    A runbook block is marked {@code } and followed by a {@code sql} + * fence. The lanes that run these blocks against real PostgreSQL read them from the document rather + * than keep a copy, so editing a block in the runbook is what the lane tests: a copy would pass + * while the document an operator follows is wrong. + */ +final class OutboxRunbookSql { + + private final String runbook; + + private OutboxRunbookSql(String runbook) { + this.runbook = runbook; + } + + /** + * Reads the runbook the lane was given in the system property {@code property}. + * + * @param property the system property holding the runbook's absolute path + * @return the runbook's SQL blocks + * @throws IOException when the file cannot be read + */ + static OutboxRunbookSql fromSystemProperty(String property) throws IOException { + String path = System.getProperty(property); + if (path == null || path.isBlank()) { + throw new IllegalStateException( + "system property " + property + " must name the runbook this lane runs"); + } + return new OutboxRunbookSql(Files.readString(Path.of(path), StandardCharsets.UTF_8)); + } + + /** + * The SQL block that follows {@code }. + * + * @param id the block's marker id + * @return the SQL text between the fences + */ + String block(String id) { + String marker = ""; + int at = runbook.indexOf(marker); + if (at < 0) { + throw new IllegalStateException("the outbox runbook has no SQL block marked " + id); + } + int open = runbook.indexOf("```sql\n", at); + int close = runbook.indexOf("\n```", open + 1); + if (open < 0 || close < 0) { + throw new IllegalStateException("the block marked " + id + " is not a closed sql fence"); + } + return runbook.substring(open + "```sql\n".length(), close); + } + + /** + * The block with its {@code }s filled in, as an operator would before running it. + * + * @param id the block's marker id + * @param values each placeholder as written in the runbook, without the angle brackets, and its + * value + * @return the runnable SQL + */ + String block(String id, Map values) { + String sql = block(id); + for (Map.Entry value : values.entrySet()) { + String placeholder = "<" + value.getKey() + ">"; + if (!sql.contains(placeholder)) { + throw new IllegalStateException("the block marked " + id + " has no " + placeholder); + } + sql = sql.replace(placeholder, value.getValue()); + } + return sql; + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlContentReferenceLedgerContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlContentReferenceLedgerContractTest.java new file mode 100644 index 00000000..8e6d3e14 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlContentReferenceLedgerContractTest.java @@ -0,0 +1,54 @@ +package dev.caskeleton.adapter.outbound.persistence.readiness; + +import dev.caskeleton.adapter.outbound.persistence.fileserver.JpaContentReferenceLedger; +import dev.caskeleton.adapter.outbound.persistence.fileserver.JpaFileMetadata; +import dev.caskeleton.application.fileserver.admin.ContentReferenceLedgerPort; +import dev.caskeleton.application.fileserver.api.metadata.FileMetadataPort; +import dev.caskeleton.application.fileserver.testkit.ContentReferenceLedgerPortContract; +import java.util.function.Supplier; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; + +/** + * {@link JpaContentReferenceLedger} held to the contract the in-memory metadata store passes, on + * real PostgreSQL (review 2026-11-03 RR-01-04). + */ +@Tag("jpa-contract") +class PostgreSqlContentReferenceLedgerContractTest extends ContentReferenceLedgerPortContract { + + private static FileserverPostgreSqlStack stack; + + @BeforeAll + static void start() { + stack = FileserverPostgreSqlStack.start(); + } + + @AfterAll + static void stop() { + if (stack != null) { + stack.close(); + } + } + + @BeforeEach + void empty() { + stack.empty(); + } + + @Override + protected ContentReferenceLedgerPort ledger() { + return stack.bean(JpaContentReferenceLedger.class); + } + + @Override + protected FileMetadataPort metadata() { + return stack.bean(JpaFileMetadata.class); + } + + @Override + protected T inTransaction(Supplier work) { + return stack.inTransaction(work); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlFileQuotaLedgerContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlFileQuotaLedgerContractTest.java new file mode 100644 index 00000000..e66aebd2 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlFileQuotaLedgerContractTest.java @@ -0,0 +1,92 @@ +package dev.caskeleton.adapter.outbound.persistence.readiness; + +import dev.caskeleton.adapter.outbound.persistence.fileserver.JpaFileQuota; +import dev.caskeleton.adapter.outbound.persistence.fileserver.JpaQuotaCommit; +import dev.caskeleton.adapter.outbound.persistence.fileserver.JpaQuotaReclaim; +import dev.caskeleton.adapter.outbound.persistence.fileserver.JpaUploadSession; +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.api.StorageNamespace; +import dev.caskeleton.application.fileserver.api.UploadId; +import dev.caskeleton.application.fileserver.api.metadata.FileQuotaPort; +import dev.caskeleton.application.fileserver.api.metadata.UploadSession; +import dev.caskeleton.application.fileserver.api.metadata.UploadSessionDraft; +import dev.caskeleton.application.fileserver.api.transfer.UploadMode; +import dev.caskeleton.application.fileserver.cleanup.QuotaReclaimPort; +import dev.caskeleton.application.fileserver.testkit.FileQuotaLedgerContract; +import dev.caskeleton.application.fileserver.upload.QuotaCommitPort; +import java.time.Duration; +import java.util.OptionalLong; +import java.util.UUID; +import java.util.function.Supplier; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; + +/** + * {@link JpaFileQuota}, {@link JpaQuotaCommit} and {@link JpaQuotaReclaim} held to the ledger + * contract the in-memory ledger passes, on real PostgreSQL (review 2026-11-03 RR-01-04). + */ +@Tag("jpa-contract") +class PostgreSqlFileQuotaLedgerContractTest extends FileQuotaLedgerContract { + + private static FileserverPostgreSqlStack stack; + + @BeforeAll + static void start() { + stack = FileserverPostgreSqlStack.start(); + } + + @AfterAll + static void stop() { + if (stack != null) { + stack.close(); + } + } + + @BeforeEach + void empty() { + stack.empty(); + } + + @Override + protected FileQuotaPort quota() { + return stack.bean(JpaFileQuota.class); + } + + @Override + protected QuotaCommitPort commits() { + return stack.bean(JpaQuotaCommit.class); + } + + @Override + protected QuotaReclaimPort reclaims() { + return stack.bean(JpaQuotaReclaim.class); + } + + @Override + protected UploadSession uploadIn(StorageNamespace namespace) { + FileId fileId = stack.recordFileIn(namespace); + return stack.inTransaction( + () -> + stack + .bean(JpaUploadSession.class) + .create( + new UploadSessionDraft( + UploadId.of(UUID.randomUUID()), + fileId, + UploadMode.SINGLE_SHOT, + OptionalLong.of(1_000), + stack.now().plus(Duration.ofHours(1))))); + } + + @Override + protected void elapse(Duration duration) { + stack.elapse(duration); + } + + @Override + protected T inTransaction(Supplier work) { + return stack.inTransaction(work); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java index 58809891..bf5117a8 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java @@ -50,7 +50,7 @@ class PostgreSqlMigrationIntegrationTest { assertThat(appliedVersions(postgres, "flyway_schema_history")) .containsExactly( "1", "3", "4", "5", "6", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", - "19", "20", "21"); + "19", "20", "21", "22"); assertThat( singleValue( postgres, diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxAggregateSequenceIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxAggregateSequenceIntegrationTest.java index e40cca12..f598452f 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxAggregateSequenceIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxAggregateSequenceIntegrationTest.java @@ -19,11 +19,18 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.jdbc.core.JdbcTemplate; /** * The aggregate order counter and the order uniqueness rule against real PostgreSQL (V21, review - * 06-07): the shared {@link OutboxAggregateSequenceContract}, the seed from rows numbered before - * the counter existed, and a second append under a used order refused at append time. + * 06-07): the shared {@link OutboxAggregateSequenceContract}, where a counter starts when the + * aggregate already has history, and a second append under a used order refused at append time. + * + *

    "History" is what the old wall-clock scheme left (epoch microseconds, about 1.7e15): rows that + * are still in the outbox, rows the reaper deleted, rows under an identity that has since changed, + * and rows an instance still on the old version writes during a rolling deploy. A consumer that + * reads the number as "which change came last" must never see a later change with a smaller number + * (review RR-03-02, RR-05-02). */ class PostgreSqlOutboxAggregateSequenceIntegrationTest extends OutboxAggregateSequenceContract { @@ -32,6 +39,7 @@ class PostgreSqlOutboxAggregateSequenceIntegrationTest extends OutboxAggregateSe private static PostgreSqlReadinessSupport postgres; private static AnnotationConfigApplicationContext context; private static OutboxAggregateSequencePort sequences; + private static JdbcTemplate jdbc; @BeforeAll static void setup() { @@ -40,6 +48,7 @@ class PostgreSqlOutboxAggregateSequenceIntegrationTest extends OutboxAggregateSe OutboxIntegrationSupport.migrate(dataSource); context = OutboxIntegrationSupport.buildContext(dataSource, e -> {}, Clock.systemUTC()); sequences = context.getBean(JpaOutboxStore.class); + jdbc = new JdbcTemplate(dataSource); } @AfterAll @@ -63,25 +72,71 @@ class PostgreSqlOutboxAggregateSequenceIntegrationTest extends OutboxAggregateSe } @Test - @DisplayName("카운터가 없던 aggregate의 첫 번호는 이미 있는 outbox 행의 가장 큰 번호 다음이다") + @DisplayName("카운터가 없던 aggregate의 첫 번호는 floor보다 큰 남은 행의 가장 큰 번호 다음이다") void aNewCounterStartsAfterTheHighestOrderTheAggregatesRowsAlreadyCarry() { - // given: a row numbered by the old wall-clock scheme + // given: a row an instance still on the clock scheme wrote five seconds after V21 ran String aggregateId = "clock-numbered-" + UUID.randomUUID(); - long clockNumber = 1_761_955_200_000_000L; + long clockNumber = floor() + 5_000_000L; append(aggregateId, clockNumber); // when - long next = - transactions() - .inWrite( - () -> - sequences.nextSequence( - new AggregateIdentity("default", "testkit", aggregateId))); + long next = next(aggregateId); // then assertThat(next).isEqualTo(clockNumber + 1); } + @Test + @DisplayName("옛 행이 reaper로 지워진 aggregate의 첫 번호는 지워진 번호가 아니라 floor 다음이다") + void anAggregateWhoseRowsWereReapedStartsAboveTheFloor() { + // given: a week-old clock-numbered row, published and deleted by the reaper since + String aggregateId = "reaped-" + UUID.randomUUID(); + long clockNumber = 1_761_955_200_000_000L; + append(aggregateId, clockNumber); + jdbc.update("DELETE FROM outbox_event WHERE aggregate_id = ?", aggregateId); + + // when + long next = next(aggregateId); + + // then + assertThat(next).as("a number the consumer has already seen").isGreaterThan(clockNumber); + assertThat(next).isEqualTo(floor() + 1); + } + + @Test + @DisplayName("tenant scope가 바뀐 aggregate의 첫 번호는 옛 scope 행의 번호보다 큰 floor 다음이다") + void anAggregateWhoseIdentityChangedStartsAboveTheFloor() { + // given: the sample's rows were written under scope "portfolio" before it moved to one scope + String aggregateId = "rescoped-" + UUID.randomUUID(); + long clockNumber = 1_761_955_200_000_000L; + append(aggregateId, clockNumber); + jdbc.update( + "UPDATE outbox_event SET tenant_scope = 'portfolio' WHERE aggregate_id = ?", aggregateId); + + // when + long next = next(aggregateId); + + // then + assertThat(next).as("a number the consumer has already seen").isGreaterThan(clockNumber); + assertThat(next).isEqualTo(floor() + 1); + } + + @Test + @DisplayName("카운터가 생긴 뒤 옛 버전 인스턴스가 더 큰 번호로 쓴 행이 있으면 다음 번호는 그 행 다음이다") + void aRowAnOldInstanceWroteAfterTheCounterExistedStillComesFirst() { + // given: a new instance numbered the aggregate, then an old one wrote its clock number + String aggregateId = "rolling-" + UUID.randomUUID(); + long counted = next(aggregateId); + long oldInstancesNumber = counted + 10_000_000L; + append(aggregateId, oldInstancesNumber); + + // when + long next = next(aggregateId); + + // then + assertThat(next).isEqualTo(oldInstancesNumber + 1); + } + @Test @DisplayName("같은 aggregate에 같은 순서 번호로 두 번 append하면 두 번째는 실패한다") void aSecondEventUnderAnOrderItsAggregateAlreadyUsedIsRefused() { @@ -96,6 +151,19 @@ class PostgreSqlOutboxAggregateSequenceIntegrationTest extends OutboxAggregateSe assertThat(thrown).isNotNull(); } + private static long next(String aggregateId) { + return context + .getBean(SpringTransaction.class) + .inWrite( + () -> sequences.nextSequence(new AggregateIdentity("default", "testkit", aggregateId))); + } + + /** The number V21 recorded once: the moment it ran, in epoch microseconds. */ + private static long floor() { + return jdbc.queryForObject( + "SELECT floor_sequence FROM outbox_aggregate_sequence_floor", Long.class); + } + private static void append(String aggregateId, long sequence) { JpaOutboxStore store = context.getBean(JpaOutboxStore.class); context diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxConcurrentRelayIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxConcurrentRelayIntegrationTest.java index ae3b56f2..54f253ca 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxConcurrentRelayIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxConcurrentRelayIntegrationTest.java @@ -179,6 +179,13 @@ class PostgreSqlOutboxConcurrentRelayIntegrationTest extends OutboxLeaseFencingC "select status from outbox_event where event_id = ?", String.class, eventId)); } + @Override + protected String lastFailureCodeOf(String eventId) { + return new JdbcTemplate(sharedDataSource) + .queryForObject( + "select last_failure_code from outbox_event where event_id = ?", String.class, eventId); + } + /** Runs the relay in a loop until claimedCount == 0 (store drained). */ private int drainRelay(PublishPendingOutboxEventsUseCase relay) { int total = 0; diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxDispositionRunbookIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxDispositionRunbookIntegrationTest.java new file mode 100644 index 00000000..027408b4 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxDispositionRunbookIntegrationTest.java @@ -0,0 +1,279 @@ +package dev.caskeleton.adapter.outbound.persistence.readiness; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; + +import com.zaxxer.hikari.HikariDataSource; +import dev.caskeleton.adapter.outbound.persistence.outbox.jpa.JpaOutboxStore; +import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; +import dev.caskeleton.application.outbox.OutboxLease; +import dev.caskeleton.application.outbox.testkit.OutboxEvents; +import java.io.IOException; +import java.sql.SQLException; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.jdbc.core.JdbcTemplate; + +/** + * The outbox runbooks' manual dispositions, run as written against real PostgreSQL. + * + *

    An operator resolves a DEAD row by re-driving or skipping it ({@code + * docs/runbooks/outbox-dead-letter.md} §4), and a row that keeps failing for ever by finding it, + * parking it, and resuming or skipping it ({@code docs/runbooks/outbox-publish-failed.md}, "끝나지 않는 + * 재시도"). The SQL for each is read out of the runbook, not copied, and its result is checked the + * only way that matters: whether the relay's next claim takes the row. A re-drive that leaves a + * counter behind is accepted by the database and then refused by the claim, which moves the row + * back to DEAD — the failure the first re-drive test pins (re-review 2026-11-03 RR-05-01). + */ +class PostgreSqlOutboxDispositionRunbookIntegrationTest { + + private static final Instant OCCURRED = Instant.parse("2026-11-01T00:00:00Z"); + private static final Duration IN_FLIGHT_TIMEOUT = Duration.ofMinutes(5); + + private static PostgreSqlReadinessSupport postgres; + private static OutboxRunbookSql deadLetterRunbook; + private static OutboxRunbookSql publishFailedRunbook; + + @BeforeAll + static void setup() throws IOException { + deadLetterRunbook = OutboxRunbookSql.fromSystemProperty("outbox.runbook"); + publishFailedRunbook = OutboxRunbookSql.fromSystemProperty("outbox.publish-failed.runbook"); + postgres = PostgreSqlReadinessSupport.start(); + } + + @AfterAll + static void cleanup() { + if (postgres != null) { + postgres.close(); + } + } + + @Test + @DisplayName("미뤄진 시도가 쌓인 DEAD 행을 runbook 재발행 SQL로 되돌리면 다음 claim이 그 행을 가져간다") + void theRedriveMakesADeadRowWithDeferredClaimsClaimableAgain() throws SQLException { + try (Outbox outbox = Outbox.create()) { + // given: dead-lettered after two claims an outage deferred and three that spent the budget + String dead = outbox.append("order-" + UUID.randomUUID(), 1, OCCURRED); + outbox.setCounters(dead, "DEAD", 5, 2); + String redrive = deadLetterRunbook.block("dead-redrive-one", Map.of("id", dead)); + + // when + outbox.jdbc().execute(redrive); + + // then + assertThat(outbox.claimedEventIds()).containsExactly(dead); + } + } + + @Test + @DisplayName("runbook 구간 재발행 SQL은 그 구간의 DEAD 행만 claim 가능하게 되돌린다") + void theWindowRedriveReturnsOnlyTheDeadRowsOfTheWindow() throws SQLException { + try (Outbox outbox = Outbox.create()) { + // given + String early = outbox.append("order-" + UUID.randomUUID(), 1, OCCURRED); + String late = outbox.append("order-" + UUID.randomUUID(), 1, OCCURRED.plusSeconds(60)); + String outside = outbox.append("order-" + UUID.randomUUID(), 1, OCCURRED.plusSeconds(3600)); + for (String eventId : List.of(early, late, outside)) { + outbox.setCounters(eventId, "DEAD", 3, 2); + } + String redrive = + deadLetterRunbook.block( + "dead-redrive-window", + Map.of( + "장애 시작", OCCURRED.minusSeconds(1).toString(), + "장애 끝", OCCURRED.plusSeconds(61).toString())); + + // when + outbox.jdbc().execute(redrive); + + // then + assertThat(outbox.claimedEventIds()).containsExactlyInAnyOrder(early, late); + } + } + + @Test + @DisplayName("runbook skip SQL은 DEAD 행을 발행된 것으로 두어 그 aggregate의 다음 행을 풀어 준다") + void theSkipReleasesTheNextRowOfTheDeadRowsAggregate() throws SQLException { + try (Outbox outbox = Outbox.create()) { + // given + String aggregate = "order-" + UUID.randomUUID(); + String dead = outbox.append(aggregate, 1, OCCURRED); + String next = outbox.append(aggregate, 2, OCCURRED.plusSeconds(1)); + outbox.setCounters(dead, "DEAD", 1, 0); + String skip = deadLetterRunbook.block("dead-skip-one", Map.of("id", dead)); + + // when + outbox.jdbc().execute(skip); + + // then + assertThat(List.of(outbox.claimedEventIds(), outbox.status(dead))) + .containsExactly(List.of(next), "PUBLISHED"); + } + } + + @Test + @DisplayName("오래 재시도 중인 행 조회는 미룬 claim이 기준 수 이상인 FAILED 행과 그 원인 코드를 보여 준다") + void theLongRetryQueryListsFailedRowsDeferredLongEnoughWithTheirFailureCode() + throws SQLException { + try (Outbox outbox = Outbox.create()) { + // given: one row deferred long enough, one that has only just started failing + String stuck = outbox.append("order-" + UUID.randomUUID(), 1, OCCURRED); + String fresh = outbox.append("order-" + UUID.randomUUID(), 1, OCCURRED); + outbox.setCounters(stuck, "FAILED", 12, 12); + outbox.setCounters(fresh, "FAILED", 1, 1); + outbox.setFailureCode(stuck, "KAFKA_INVALIDTIMESTAMPEXCEPTION"); + + // when + List> found = + outbox.jdbc().queryForList(publishFailedRunbook.block("long-retries")); + + // then + assertThat(found) + .extracting(row -> row.get("event_id"), row -> row.get("last_failure_code")) + .containsExactly(tuple(stuck, "KAFKA_INVALIDTIMESTAMPEXCEPTION")); + } + } + + @Test + @DisplayName("runbook park SQL로 보류한 행은 claim되지 않고 그 aggregate의 다음 행도 막는다") + void aParkedRowIsNotClaimedAndStillHoldsItsAggregatesNextRow() throws SQLException { + try (Outbox outbox = Outbox.create()) { + // given + String aggregate = "order-" + UUID.randomUUID(); + String failing = outbox.append(aggregate, 1, OCCURRED); + outbox.append(aggregate, 2, OCCURRED.plusSeconds(1)); + outbox.setCounters(failing, "FAILED", 12, 12); + String park = publishFailedRunbook.block("park-one", Map.of("id", failing)); + + // when + outbox.jdbc().execute(park); + + // then + assertThat(List.of(outbox.claimedEventIds(), outbox.status(failing))) + .containsExactly(List.of(), "PARKED"); + } + } + + @Test + @DisplayName("runbook resume SQL로 되돌린 보류 행은 다음 claim이 가져간다") + void aResumedRowIsClaimedAgain() throws SQLException { + try (Outbox outbox = Outbox.create()) { + // given + String parked = outbox.append("order-" + UUID.randomUUID(), 1, OCCURRED); + outbox.setCounters(parked, "FAILED", 12, 12); + outbox.jdbc().execute(publishFailedRunbook.block("park-one", Map.of("id", parked))); + String resume = publishFailedRunbook.block("resume-one", Map.of("id", parked)); + + // when + outbox.jdbc().execute(resume); + + // then + assertThat(outbox.claimedEventIds()).containsExactly(parked); + } + } + + @Test + @DisplayName("runbook skip SQL로 버린 보류 행은 발행된 것으로 남고 그 aggregate의 다음 행을 풀어 준다") + void aSkippedParkedRowReleasesItsAggregatesNextRow() throws SQLException { + try (Outbox outbox = Outbox.create()) { + // given + String aggregate = "order-" + UUID.randomUUID(); + String parked = outbox.append(aggregate, 1, OCCURRED); + String next = outbox.append(aggregate, 2, OCCURRED.plusSeconds(1)); + outbox.setCounters(parked, "FAILED", 12, 12); + outbox.jdbc().execute(publishFailedRunbook.block("park-one", Map.of("id", parked))); + String skip = publishFailedRunbook.block("parked-skip-one", Map.of("id", parked)); + + // when + outbox.jdbc().execute(skip); + + // then + assertThat(List.of(outbox.claimedEventIds(), outbox.status(parked))) + .containsExactly(List.of(next), "PUBLISHED"); + } + } + + /** A fresh database at the latest migration with the JPA outbox store over it. */ + private static final class Outbox implements AutoCloseable { + + private final HikariDataSource dataSource; + private final AnnotationConfigApplicationContext context; + private final JdbcTemplate jdbc; + + private Outbox(HikariDataSource dataSource) { + this.dataSource = dataSource; + OutboxIntegrationSupport.migrate(dataSource); + this.context = OutboxIntegrationSupport.buildContext(dataSource, e -> {}, Clock.systemUTC()); + this.jdbc = new JdbcTemplate(dataSource); + } + + static Outbox create() throws SQLException { + String name = + "runbook_" + UUID.randomUUID().toString().replace("-", "").toLowerCase(Locale.ROOT); + return new Outbox(postgres.createDatabase(name)); + } + + JdbcTemplate jdbc() { + return jdbc; + } + + String append(String aggregateId, long sequence, Instant occurredAt) { + String eventId = UUID.randomUUID().toString(); + JpaOutboxStore store = context.getBean(JpaOutboxStore.class); + context + .getBean(SpringTransaction.class) + .inWrite( + () -> store.append(OutboxEvents.event(eventId, aggregateId, sequence, occurredAt))); + return eventId; + } + + /** The counters a relay leaves on a row; written directly because only their values matter. */ + void setCounters(String eventId, String status, int attempts, int deferred) { + jdbc.update( + "UPDATE outbox_event SET status = ?, attempt_count = ?, deferred_attempt_count = ?" + + " WHERE event_id = ?", + status, + attempts, + deferred, + eventId); + } + + void setFailureCode(String eventId, String failureCode) { + jdbc.update( + "UPDATE outbox_event SET last_failure_code = ? WHERE event_id = ?", failureCode, eventId); + } + + String status(String eventId) { + return jdbc.queryForObject( + "SELECT status FROM outbox_event WHERE event_id = ?", String.class, eventId); + } + + /** What the relay's next claim takes, a minute from now so every re-driven row is due. */ + List claimedEventIds() { + JpaOutboxStore store = context.getBean(JpaOutboxStore.class); + Instant now = Instant.now().plusSeconds(60); + return context + .getBean(SpringTransaction.class) + .inWrite(() -> store.claimBatch("runbook-relay", 50, now, IN_FLIGHT_TIMEOUT)) + .stream() + .map(OutboxLease::eventId) + .toList(); + } + + @Override + public void close() { + context.close(); + dataSource.close(); + } + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxOrderKeyRunbookIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxOrderKeyRunbookIntegrationTest.java index 98ab6ecf..09a8381c 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxOrderKeyRunbookIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOutboxOrderKeyRunbookIntegrationTest.java @@ -9,16 +9,20 @@ import dev.caskeleton.adapter.outbound.persistence.outbox.jpa.JpaOutboxStore; import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; import dev.caskeleton.application.outbox.testkit.OutboxEvents; import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; +import java.sql.Connection; import java.sql.SQLException; +import java.sql.Statement; import java.time.Clock; import java.time.Instant; +import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import org.flywaydb.core.Flyway; import org.flywaydb.core.api.FlywayException; import org.junit.jupiter.api.AfterAll; @@ -43,15 +47,15 @@ class PostgreSqlOutboxOrderKeyRunbookIntegrationTest { private static final Instant OCCURRED = Instant.parse("2026-11-01T00:00:00Z"); private static PostgreSqlReadinessSupport postgres; + private static OutboxRunbookSql runbook; private static String duplicateCheck; private static String removePublishedDuplicates; @BeforeAll static void setup() throws IOException { - String runbook = - Files.readString(Path.of(System.getProperty("outbox.runbook")), StandardCharsets.UTF_8); - duplicateCheck = runbookSql(runbook, "v21-duplicate-order-keys"); - removePublishedDuplicates = runbookSql(runbook, "v21-remove-published-duplicates"); + runbook = OutboxRunbookSql.fromSystemProperty("outbox.runbook"); + duplicateCheck = runbook.block("v21-duplicate-order-keys"); + removePublishedDuplicates = runbook.block("v21-remove-published-duplicates"); postgres = PostgreSqlReadinessSupport.start(); } @@ -176,19 +180,143 @@ class PostgreSqlOutboxOrderKeyRunbookIntegrationTest { } } - /** The SQL block that follows {@code } in the runbook. */ - private static String runbookSql(String runbook, String id) { - String marker = ""; - int at = runbook.indexOf(marker); - if (at < 0) { - throw new IllegalStateException("the outbox runbook has no SQL block marked " + id); + @Test + @DisplayName("V21은 남은 행이 있는 aggregate의 카운터를 그 aggregate의 가장 큰 번호로 미리 만든다") + void v21SeedsTheCounterOfEveryAggregateWithRowsAtItsHighestNumber() throws SQLException { + try (BeforeV21 database = BeforeV21.create()) { + // given + String twice = "twice-" + UUID.randomUUID(); + database.append(twice, 1_761_955_200_000_003L); + database.append(twice, 1_761_955_200_000_009L); + String once = "once-" + UUID.randomUUID(); + database.append(once, 1_761_955_200_000_004L); + + // when + database.migrateToLatest(); + + // then + assertThat(List.of(database.counterOf(twice), database.counterOf(once))) + .containsExactly(1_761_955_200_000_009L, 1_761_955_200_000_004L); } - int open = runbook.indexOf("```sql\n", at); - int close = runbook.indexOf("\n```", open + 1); - if (open < 0 || close < 0) { - throw new IllegalStateException("the block marked " + id + " is not a closed sql fence"); + } + + @Test + @DisplayName("V21은 적용한 시각을 마이크로초 floor로 한 번 기록한다") + void v21RecordsTheMomentItRanAsTheFloorInMicroseconds() throws SQLException { + try (BeforeV21 database = BeforeV21.create()) { + // given + long before = microsecondsNow() - 1_000_000L; + + // when + database.migrateToLatest(); + + // then + long after = microsecondsNow() + 1_000_000L; + assertThat( + database + .jdbc() + .queryForList( + "SELECT floor_sequence FROM outbox_aggregate_sequence_floor", Long.class)) + .singleElement() + .satisfies(floor -> assertThat(floor).isBetween(before, after)); } - return runbook.substring(open + "```sql\n".length(), close); + } + + @Test + @DisplayName("outbox 쓰기 트랜잭션이 잠금을 잡고 있으면 V21은 기다리다 실패하고 DB는 V20 그대로 남는다") + void v21GivesUpOnALockAWriterHoldsAndLeavesTheDatabaseAtV20() throws Exception { + try (BeforeV21 database = BeforeV21.create(); + Connection writer = database.connection()) { + // given: a writer transaction holding the lock every outbox insert takes, released after 15 s + writer.setAutoCommit(false); + try (Statement statement = writer.createStatement()) { + statement.execute("LOCK TABLE outbox_event IN ROW EXCLUSIVE MODE"); + } + ScheduledExecutorService release = Executors.newSingleThreadScheduledExecutor(); + ScheduledFuture released = + release.schedule( + () -> { + writer.rollback(); + return null; + }, + 15, + TimeUnit.SECONDS); + try { + // when + Throwable thrown = catchThrowable(database::migrateToLatest); + + // then + assertThat( + List.of( + thrown instanceof FlywayException, + database.flyway(null).info().current().getVersion().getVersion())) + .containsExactly(true, "20"); + } finally { + released.cancel(true); + release.shutdownNow(); + } + } + } + + @Test + @DisplayName("실패한 CONCURRENTLY 빌드가 남긴 무효 인덱스가 있으면 V21은 그것을 건너뛰지 않고 실패한다") + void anInvalidIndexLeftByAFailedConcurrentBuildFailsV21() throws SQLException { + try (BeforeV21 database = BeforeV21.create()) { + // given: a concurrent build that met a duplicate leaves an invalid index behind + String shared = "shared-" + UUID.randomUUID(); + String kept = database.append(shared, 1); + database.append(shared, 1); + Throwable failedBuild = + catchThrowable( + () -> + database + .jdbc() + .execute( + "CREATE UNIQUE INDEX CONCURRENTLY ux_outbox_event_aggregate_order ON" + + " outbox_event (tenant_scope, aggregate_type, aggregate_id," + + " aggregate_sequence, event_index)")); + assertThat(failedBuild).as("the concurrent build met the duplicate").isNotNull(); + database + .jdbc() + .update( + "DELETE FROM outbox_event WHERE aggregate_id = ? AND event_id <> ?", shared, kept); + + // when + Throwable thrown = catchThrowable(database::migrateToLatest); + + // then + assertThat( + List.of( + thrown instanceof FlywayException, + database.flyway(null).info().current().getVersion().getVersion())) + .containsExactly(true, "20"); + } + } + + @Test + @DisplayName("runbook대로 순서 키 인덱스를 CONCURRENTLY로 먼저 만들면 V21은 그 인덱스를 쓰고 적용된다") + void v21AppliesOverTheIndexTheRunbookBuiltConcurrently() throws SQLException { + try (BeforeV21 database = BeforeV21.create()) { + // given + database.append("ordered-" + UUID.randomUUID(), 1); + database.jdbc().execute(runbook.block("v21-prebuild-order-index")); + + // when + database.migrateToLatest(); + + // then + assertThat( + List.of( + Arrays.stream(database.flyway(null).info().applied()) + .anyMatch(migration -> "21".equals(migration.getVersion().getVersion())), + database.validIndexExists("ux_outbox_event_aggregate_order"))) + .containsExactly(true, true); + } + } + + private static long microsecondsNow() { + Instant now = Instant.now(); + return now.getEpochSecond() * 1_000_000L + now.getNano() / 1_000L; } /** @@ -259,6 +387,26 @@ class PostgreSqlOutboxOrderKeyRunbookIntegrationTest { jdbc.queryForObject("SELECT to_regclass(?) IS NOT NULL", Boolean.class, table)); } + Connection connection() throws SQLException { + return dataSource.getConnection(); + } + + long counterOf(String aggregateId) { + return jdbc.queryForObject( + "SELECT last_sequence FROM outbox_aggregate_sequence WHERE aggregate_id = ?", + Long.class, + aggregateId); + } + + boolean validIndexExists(String index) { + return Boolean.TRUE.equals( + jdbc.queryForObject( + "SELECT EXISTS (SELECT 1 FROM pg_index WHERE indexrelid = to_regclass(?)" + + " AND indisvalid)", + Boolean.class, + index)); + } + boolean indexExists(String index) { return Boolean.TRUE.equals( jdbc.queryForObject( diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlRecoveryQueueContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlRecoveryQueueContractTest.java new file mode 100644 index 00000000..9ba78957 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlRecoveryQueueContractTest.java @@ -0,0 +1,60 @@ +package dev.caskeleton.adapter.outbound.persistence.readiness; + +import dev.caskeleton.adapter.outbound.persistence.fileserver.JpaRecoveryQueue; +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.api.StorageNamespace; +import dev.caskeleton.application.fileserver.recovery.RecoveryQueuePort; +import dev.caskeleton.application.fileserver.testkit.RecoveryQueuePortContract; +import java.time.Duration; +import java.util.function.Supplier; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; + +/** + * {@link JpaRecoveryQueue} held to the contract the in-memory queue passes, on real PostgreSQL + * (review 2026-11-03 RR-01-04). + */ +@Tag("jpa-contract") +class PostgreSqlRecoveryQueueContractTest extends RecoveryQueuePortContract { + + private static FileserverPostgreSqlStack stack; + + @BeforeAll + static void start() { + stack = FileserverPostgreSqlStack.start(); + } + + @AfterAll + static void stop() { + if (stack != null) { + stack.close(); + } + } + + @BeforeEach + void empty() { + stack.empty(); + } + + @Override + protected RecoveryQueuePort queue() { + return stack.bean(JpaRecoveryQueue.class); + } + + @Override + protected FileId recordedFile() { + return stack.recordFileIn(StorageNamespace.of("tenant-a")); + } + + @Override + protected void elapse(Duration duration) { + stack.elapse(duration); + } + + @Override + protected T inTransaction(Supplier work) { + return stack.inTransaction(work); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlStagingUploadLocatorContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlStagingUploadLocatorContractTest.java new file mode 100644 index 00000000..105eb397 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlStagingUploadLocatorContractTest.java @@ -0,0 +1,73 @@ +package dev.caskeleton.adapter.outbound.persistence.readiness; + +import dev.caskeleton.adapter.outbound.persistence.fileserver.JpaStagingUploadLocator; +import dev.caskeleton.adapter.outbound.persistence.fileserver.JpaUploadSession; +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.api.StorageNamespace; +import dev.caskeleton.application.fileserver.api.metadata.UploadSessionPort; +import dev.caskeleton.application.fileserver.recovery.StagingUploadLocatorPort; +import dev.caskeleton.application.fileserver.testkit.StagingUploadLocatorPortContract; +import java.time.Duration; +import java.time.Instant; +import java.util.function.Supplier; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; + +/** + * {@link JpaStagingUploadLocator} held to the contract the in-memory session store passes, on real + * PostgreSQL (review 2026-11-03 RR-01-04). + */ +@Tag("jpa-contract") +class PostgreSqlStagingUploadLocatorContractTest extends StagingUploadLocatorPortContract { + + private static FileserverPostgreSqlStack stack; + + @BeforeAll + static void start() { + stack = FileserverPostgreSqlStack.start(); + } + + @AfterAll + static void stop() { + if (stack != null) { + stack.close(); + } + } + + @BeforeEach + void empty() { + stack.empty(); + } + + @Override + protected StagingUploadLocatorPort locator() { + return stack.bean(JpaStagingUploadLocator.class); + } + + @Override + protected UploadSessionPort sessions() { + return stack.bean(JpaUploadSession.class); + } + + @Override + protected FileId recordedFile() { + return stack.recordFileIn(StorageNamespace.of("tenant-a")); + } + + @Override + protected Instant now() { + return stack.now(); + } + + @Override + protected void elapse(Duration duration) { + stack.elapse(duration); + } + + @Override + protected T inTransaction(Supplier work) { + return stack.inTransaction(work); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlTransactionIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlTransactionIntegrationTest.java index 2ee91d75..96d928a8 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlTransactionIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlTransactionIntegrationTest.java @@ -293,6 +293,49 @@ class PostgreSqlTransactionIntegrationTest { assertThat(elapsed).isLessThan(Duration.ofSeconds(5)); } + /** + * The production transaction manager is {@code JpaTransactionManager}, and past the deadline + * Hibernate refuses the next statement itself, before the driver is asked: no {@code 57014}, no + * SQLState at all (review RR-03-03). The test above covers the JDBC path only. + */ + @Test + @Tag("jpa-failure") + @Timeout(20) + @DisplayName("JPA 트랜잭션이 제한 시간을 넘긴 뒤 다음 문장을 Hibernate가 거부해도 DB_QUERY_CANCELED로 끝난다") + void aJpaWriteTransactionPastItsBoundIsAQueryCancellation() { + // given + Configuration configuration = new Configuration(); + configuration.getProperties().put("hibernate.connection.datasource", postgres.dataSource()); + configuration.setProperty("hibernate.hbm2ddl.auto", "create"); + configuration.addAnnotatedClass(VersionedRow.class); + try (SessionFactory sessionFactory = configuration.buildSessionFactory()) { + EntityManager entityManager = + SharedEntityManagerCreator.createSharedEntityManager(sessionFactory); + SpringTransaction port = + new SpringTransaction( + new JpaTransactionManager(sessionFactory), + translator(), + new SpringTransaction.Timeouts(Duration.ofSeconds(1), Duration.ofSeconds(1))); + + // when + Throwable thrown = + catchThrowable( + () -> + port.inWrite( + () -> { + sleepPastOneSecond(); + entityManager.persist(new VersionedRow(7L, "late")); + entityManager.flush(); + return null; + })); + + // then + assertThat(thrown).isInstanceOf(PersistenceFailureException.class); + assertThat(((PersistenceFailureException) thrown).errorCode()) + .isEqualTo(OperationalError.DB_QUERY_CANCELED); + } + } + // ---- review 09-05: READ COMMITTED comes from the pool, with no per-transaction round trip ---- @Test @@ -323,6 +366,15 @@ class PostgreSqlTransactionIntegrationTest { } } + private static void sleepPastOneSecond() { + try { + Thread.sleep(1_200); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(interrupted); + } + } + /** A data source whose connections count {@code getTransactionIsolation()} calls. */ private static DataSource countingIsolationQueries(DataSource target, AtomicInteger calls) { return (DataSource) diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlUploadSessionContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlUploadSessionContractTest.java new file mode 100644 index 00000000..0bf47b5d --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlUploadSessionContractTest.java @@ -0,0 +1,66 @@ +package dev.caskeleton.adapter.outbound.persistence.readiness; + +import dev.caskeleton.adapter.outbound.persistence.fileserver.JpaUploadSession; +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.api.StorageNamespace; +import dev.caskeleton.application.fileserver.api.metadata.UploadSessionPort; +import dev.caskeleton.application.fileserver.testkit.UploadSessionPortContract; +import java.time.Duration; +import java.time.Instant; +import java.util.function.Supplier; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; + +/** + * {@link JpaUploadSession} held to the writer-lease contract the in-memory store passes, on real + * PostgreSQL (review 2026-11-03 RR-01-04). + */ +@Tag("jpa-contract") +class PostgreSqlUploadSessionContractTest extends UploadSessionPortContract { + + private static FileserverPostgreSqlStack stack; + + @BeforeAll + static void start() { + stack = FileserverPostgreSqlStack.start(); + } + + @AfterAll + static void stop() { + if (stack != null) { + stack.close(); + } + } + + @BeforeEach + void empty() { + stack.empty(); + } + + @Override + protected UploadSessionPort sessions() { + return stack.bean(JpaUploadSession.class); + } + + @Override + protected FileId recordedFile() { + return stack.recordFileIn(StorageNamespace.of("tenant-a")); + } + + @Override + protected Instant now() { + return stack.now(); + } + + @Override + protected void elapse(Duration duration) { + stack.elapse(duration); + } + + @Override + protected T inTransaction(Supplier work) { + return stack.inTransaction(work); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStoreTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStoreTest.java index 3905930f..6b359c1e 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStoreTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/outbox/jpa/JpaOutboxStoreTest.java @@ -13,6 +13,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import dev.caskeleton.adapter.outbound.persistence.outbox.jpa.entity.OutboxEventEntity; import dev.caskeleton.application.messaging.contract.ContractId; import dev.caskeleton.application.messaging.contract.LogicalDestinationId; @@ -29,13 +32,16 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.util.HexFormat; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; +import org.slf4j.LoggerFactory; import org.springframework.transaction.support.TransactionSynchronizationManager; /** @@ -181,10 +187,10 @@ class JpaOutboxStoreTest { assertThatThrownBy(() -> adapter.markPublished(lease("evt-guard"))) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("markPublished"); - assertThatThrownBy(() -> adapter.markFailed(lease("evt-guard"), NOW.plusSeconds(1))) + assertThatThrownBy(() -> adapter.markFailed(lease("evt-guard"), NOW.plusSeconds(1), "CODE")) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("markFailed"); - assertThatThrownBy(() -> adapter.markDead(lease("evt-guard"))) + assertThatThrownBy(() -> adapter.markDead(lease("evt-guard"), "CODE")) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("markDead"); } @@ -281,6 +287,56 @@ class JpaOutboxStoreTest { assertThat(List.of(claimed, entity.getStatus())).containsExactly(List.of(), "DEAD"); } + /** + * A row whose deferred claims outnumber its claims was not damaged: a manual re-drive reset + * {@code attempt_count} and left {@code deferred_attempt_count} (re-review 2026-11-03 RR-05-01). + * The WARN says that, so the operator re-runs the corrected runbook SQL instead of hunting for + * corruption. + */ + @Test + @DisplayName("claim이 시도 횟수가 어긋난 행을 DEAD로 옮기면 WARN이 어긋난 두 값과 runbook을 말한다") + void claimBatchNamesInconsistentAttemptCountersWhenItMovesSuchARowToDead() { + // given + OutboxEventEntity redriven = pendingEntity("evt-redriven", "worklog-42"); + redriven.setDeferredAttemptCount(2); + when(claimRepo.claimEligible(eq(NOW), eq(1))).thenReturn(List.of(redriven)); + Logger storeLog = (Logger) LoggerFactory.getLogger(JpaOutboxStore.class); + ListAppender captured = new ListAppender<>(); + captured.start(); + storeLog.addAppender(captured); + + // when + try { + adapter.claimBatch(OWNER, 1, NOW, IN_FLIGHT_TIMEOUT); + } finally { + storeLog.detachAppender(captured); + } + + // then + assertThat(captured.list) + .singleElement() + .satisfies( + warn -> + assertThat(List.of(warn.getFormattedMessage(), keyValues(warn))) + .containsExactly( + "outbox claim moved a row to DEAD: deferred_attempt_count (2) exceeds" + + " attempt_count (1); a manual re-drive that did not reset" + + " deferred_attempt_count leaves this behind — re-drive it with the" + + " runbook SQL", + Map.of( + "event_id", "evt-redriven", + "error.reason", "ATTEMPT_COUNTERS_INCONSISTENT", + "runbook_link", "runbook://outbox/dead-letter"))); + } + + private static Map keyValues(ILoggingEvent event) { + Map values = new LinkedHashMap<>(); + if (event.getKeyValuePairs() != null) { + event.getKeyValuePairs().forEach(pair -> values.put(pair.key, pair.value)); + } + return values; + } + @Test void claimBatchReturnsEmptyWhenRepoReturnsNothing() { when(claimRepo.claimEligible(any(), anyInt())).thenReturn(List.of()); @@ -319,16 +375,18 @@ class JpaOutboxStoreTest { @Test void markFailedCompletesOnlyUnderTheLeaseAndSchedulesTheRetry() { Instant retryAt = NOW.plus(Duration.ofSeconds(30)); - when(repo.failLease("evt-004", OWNER, 1, retryAt)).thenReturn(1); + when(repo.failLease("evt-004", OWNER, 1, retryAt, "IllegalStateException")).thenReturn(1); - assertThat(adapter.markFailed(lease("evt-004"), retryAt)).isEqualTo(OutboxCompletion.APPLIED); + assertThat(adapter.markFailed(lease("evt-004"), retryAt, "IllegalStateException")) + .isEqualTo(OutboxCompletion.APPLIED); } @Test void markDeadCompletesOnlyUnderTheLease() { - when(repo.completeLease("evt-005", OWNER, 1, "DEAD")).thenReturn(1); + when(repo.deadLease("evt-005", OWNER, 1, "REJECTED_BEFORE_SEND")).thenReturn(1); - assertThat(adapter.markDead(lease("evt-005"))).isEqualTo(OutboxCompletion.APPLIED); + assertThat(adapter.markDead(lease("evt-005"), "REJECTED_BEFORE_SEND")) + .isEqualTo(OutboxCompletion.APPLIED); } /** diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/FlywayProdSafetyValidatorTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/FlywayProdSafetyValidatorTest.java index 39ead9d6..38cd71e2 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/FlywayProdSafetyValidatorTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/FlywayProdSafetyValidatorTest.java @@ -111,7 +111,7 @@ class FlywayProdSafetyValidatorTest { @Configuration static class ValidatorConfig { @Bean - FlywayProdSafetyValidator flywayProdSafetyValidator(Environment environment) { + static FlywayProdSafetyValidator flywayProdSafetyValidator(Environment environment) { return new FlywayProdSafetyValidator(); } } diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/HikariPoolConstraintValidatorTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/HikariPoolConstraintValidatorTest.java index 906cc346..1864e69c 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/HikariPoolConstraintValidatorTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/HikariPoolConstraintValidatorTest.java @@ -300,7 +300,7 @@ class HikariPoolConstraintValidatorTest { @Configuration static class ValidatorConfig { @Bean - HikariPoolConstraintValidator hikariPoolConstraintValidator(Environment environment) { + static HikariPoolConstraintValidator hikariPoolConstraintValidator(Environment environment) { return new HikariPoolConstraintValidator(); } } diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/OpenInViewSafetyValidatorTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/OpenInViewSafetyValidatorTest.java index fcf871f6..45c48aa4 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/OpenInViewSafetyValidatorTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/OpenInViewSafetyValidatorTest.java @@ -72,7 +72,7 @@ class OpenInViewSafetyValidatorTest { @Configuration static class ValidatorConfig { @Bean - OpenInViewSafetyValidator openInViewSafetyValidator(Environment environment) { + static OpenInViewSafetyValidator openInViewSafetyValidator(Environment environment) { return new OpenInViewSafetyValidator(); } } diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/PersistenceVendorProdSafetyValidatorTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/PersistenceVendorProdSafetyValidatorTest.java index 46b078b5..14a60e32 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/PersistenceVendorProdSafetyValidatorTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/PersistenceVendorProdSafetyValidatorTest.java @@ -101,7 +101,7 @@ class PersistenceVendorProdSafetyValidatorTest { static class ValidatorConfig { @Bean - PersistenceVendorProdSafetyValidator validator(Environment environment) { + static PersistenceVendorProdSafetyValidator validator(Environment environment) { return new PersistenceVendorProdSafetyValidator(); } } diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/PostgreSqlTransportSecurityValidatorTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/PostgreSqlTransportSecurityValidatorTest.java index e012f579..876fe01e 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/PostgreSqlTransportSecurityValidatorTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/PostgreSqlTransportSecurityValidatorTest.java @@ -102,7 +102,7 @@ class PostgreSqlTransportSecurityValidatorTest { static class ValidatorConfig { @Bean - PostgreSqlTransportSecurityValidator postgreSqlTransportSecurityValidator( + static PostgreSqlTransportSecurityValidator postgreSqlTransportSecurityValidator( Environment environment) { return new PostgreSqlTransportSecurityValidator(); } diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/RequiredEnvironmentValidatorTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/RequiredEnvironmentValidatorTest.java index 2552d2c2..8c6842e4 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/RequiredEnvironmentValidatorTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/startup/RequiredEnvironmentValidatorTest.java @@ -81,7 +81,7 @@ class RequiredEnvironmentValidatorTest { @Configuration static class ValidatorConfig { @Bean - RequiredEnvironmentValidator requiredEnvironmentValidator(Environment environment) { + static RequiredEnvironmentValidator requiredEnvironmentValidator(Environment environment) { return new RequiredEnvironmentValidator(); } } diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/JpaArchitectureRulesTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/JpaArchitectureRulesTest.java index 89a771c0..d9405ea5 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/JpaArchitectureRulesTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/JpaArchitectureRulesTest.java @@ -10,6 +10,8 @@ import dev.caskeleton.adapter.outbound.persistence.testkit.arch.entity.FinalEnti import dev.caskeleton.adapter.outbound.persistence.testkit.arch.entity.InverseCollectionParentEntity; import dev.caskeleton.adapter.outbound.persistence.testkit.arch.entity.InverseManyToManyEntity; import dev.caskeleton.adapter.outbound.persistence.testkit.arch.entity.InverseOneToOneEntity; +import dev.caskeleton.adapter.outbound.persistence.testkit.arch.entity.JoinColumnChildEntity; +import dev.caskeleton.adapter.outbound.persistence.testkit.arch.entity.JoinColumnParentEntity; import dev.caskeleton.adapter.outbound.persistence.testkit.arch.entity.NoDefaultConstructorEntity; import dev.caskeleton.adapter.outbound.persistence.testkit.arch.entity.OrderEntity; import dev.caskeleton.adapter.outbound.persistence.testkit.arch.entity.OwningChildEntity; @@ -19,6 +21,7 @@ import dev.caskeleton.adapter.outbound.persistence.testkit.arch.entity.Unidirect import dev.caskeleton.adapter.outbound.persistence.testkit.arch.web.BadOrderController; import dev.caskeleton.adapter.outbound.persistence.testkit.arch.web.GoodOrderController; import dev.caskeleton.adapter.outbound.persistence.testkit.arch.web.ListReturningController; +import dev.caskeleton.adapter.outbound.persistence.testkit.outsider.OrderEntityReader; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -181,6 +184,27 @@ class JpaArchitectureRulesTest { .hasMessageNotContaining("OwningChildEntity.parent"); } + @Test + @DisplayName("mappedBy 없이 두 엔티티가 서로를 연관으로 가리키면 양쪽 모두 거부된다") + void aPairThatPointsBothWaysWithoutMappedByIsRefusedOnBothSides() { + // given + var classes = + new ClassFileImporter() + .importClasses(JoinColumnParentEntity.class, JoinColumnChildEntity.class); + + // when + Throwable failure = + catchThrowable( + () -> JpaArchitectureRules.associationsAreMappedFromOneSideOnly().check(classes)); + + // then + assertThat(failure) + .isInstanceOf(AssertionError.class) + .hasMessageContaining("JoinColumnParentEntity.children") + .hasMessageContaining("JoinColumnChildEntity.parent") + .hasMessageContaining("one side only"); + } + @Test @DisplayName("단방향 연관관계는 통과한다") void unidirectionalAssociationsPass() { @@ -195,4 +219,44 @@ class JpaArchitectureRulesTest { assertThatCode(() -> JpaArchitectureRules.associationsAreMappedFromOneSideOnly().check(classes)) .doesNotThrowAnyException(); } + + @Test + @DisplayName("leaf 밖의 클래스가 그 leaf의 엔티티를 쓰면 거부된다") + void aClassOutsideTheLeafThatUsesItsEntityIsRefused() { + // given + var classes = new ClassFileImporter().importClasses(OrderEntity.class, OrderEntityReader.class); + + // when + Throwable failure = + catchThrowable( + () -> + JpaArchitectureRules.entitiesAreUsedOnlyInsideTheirLeaf( + "dev.caskeleton.adapter.outbound.persistence.testkit.arch") + .check(classes)); + + // then + assertThat(failure) + .isInstanceOf(AssertionError.class) + .hasMessageContaining("OrderEntityReader") + .hasMessageContaining("OrderEntity"); + } + + @Test + @DisplayName("같은 leaf 안의 클래스가 그 leaf의 엔티티를 쓰는 것은 통과한다") + void aClassInsideTheLeafMayUseItsEntity() { + // given + var classes = + new ClassFileImporter().importClasses(OrderEntity.class, BadOrderController.class); + + // when + Throwable failure = + catchThrowable( + () -> + JpaArchitectureRules.entitiesAreUsedOnlyInsideTheirLeaf( + "dev.caskeleton.adapter.outbound.persistence.testkit.arch") + .check(classes)); + + // then + assertThat(failure).isNull(); + } } diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/entity/JoinColumnChildEntity.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/entity/JoinColumnChildEntity.java new file mode 100644 index 00000000..f8700aa8 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/entity/JoinColumnChildEntity.java @@ -0,0 +1,36 @@ +package dev.caskeleton.adapter.outbound.persistence.testkit.arch.entity; + +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; + +/** + * Refused by the association rule: the read-only half of {@link JoinColumnParentEntity}'s pair. + * + *

    {@code insertable = false, updatable = false} makes this field a second, read-only copy of the + * link the parent's collection writes. Within one persistence context it does not follow a change + * made through the collection. + */ +@Entity +public class JoinColumnChildEntity { + + @Id private Long id; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "parent_id", insertable = false, updatable = false) + private JoinColumnParentEntity parent; + + protected JoinColumnChildEntity() {} + + /** The identifier. */ + public Long id() { + return id; + } + + /** The read-only reference back to the parent. */ + public JoinColumnParentEntity parent() { + return parent; + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/entity/JoinColumnParentEntity.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/entity/JoinColumnParentEntity.java new file mode 100644 index 00000000..87cbb31a --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/entity/JoinColumnParentEntity.java @@ -0,0 +1,38 @@ +package dev.caskeleton.adapter.outbound.persistence.testkit.arch.entity; + +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.OneToMany; +import java.util.ArrayList; +import java.util.List; + +/** + * Refused by the association rule: one half of a bidirectional pair that has no {@code mappedBy}. + * + *

    This collection and {@link JoinColumnChildEntity#parent} map the same {@code parent_id} + * column, the {@code @JoinColumn} form of a bidirectional association. Neither side says it is the + * inverse, so an attribute check finds nothing, yet the pair has the same two copies of one link + * that the rule exists to refuse (review RR-03-04). + */ +@Entity +public class JoinColumnParentEntity { + + @Id private Long id; + + @OneToMany + @JoinColumn(name = "parent_id") + private List children = new ArrayList<>(); + + protected JoinColumnParentEntity() {} + + /** The identifier. */ + public Long id() { + return id; + } + + /** The collection that writes the foreign key. */ + public List children() { + return List.copyOf(children); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/testkit/outsider/OrderEntityReader.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/testkit/outsider/OrderEntityReader.java new file mode 100644 index 00000000..36ed6a22 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/testkit/outsider/OrderEntityReader.java @@ -0,0 +1,18 @@ +package dev.caskeleton.adapter.outbound.persistence.testkit.outsider; + +import dev.caskeleton.adapter.outbound.persistence.testkit.arch.entity.OrderEntity; + +/** + * Refused by the entity-confinement rule when {@code testkit.arch} plays the leaf: a class outside + * that package that reads one of its entities. It lives in a package of its own because the rule + * selects by package, and a fixture inside the leaf would pass for the wrong reason. + */ +public final class OrderEntityReader { + + private OrderEntityReader() {} + + /** Reads the entity's identifier, which is all it takes to depend on it. */ + public static Long idOf(OrderEntity order) { + return order.id(); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransactionDeadlineTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransactionDeadlineTest.java new file mode 100644 index 00000000..66d31a50 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/transaction/SpringTransactionDeadlineTest.java @@ -0,0 +1,152 @@ +package dev.caskeleton.adapter.outbound.persistence.transaction; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.adapter.outbound.persistence.failure.PersistenceExceptionTranslator; +import dev.caskeleton.adapter.outbound.persistence.failure.StandardSqlStateErrorMapping; +import dev.caskeleton.shared.error.OperationalError; +import dev.caskeleton.shared.error.PersistenceFailureException; +import jakarta.persistence.Entity; +import jakarta.persistence.EntityManager; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import java.time.Duration; +import java.util.List; +import java.util.UUID; +import org.hibernate.SessionFactory; +import org.hibernate.cfg.Configuration; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.SharedEntityManagerCreator; + +/** + * A transaction deadline that expires on the JPA path, where the production transaction manager + * runs (review RR-03-03). + * + *

    Past the deadline Hibernate refuses the next statement itself, before any JDBC call, with an + * {@code org.hibernate.TransactionException} that carries no SQLState and is not Spring's {@code + * TransactionTimedOutException}. The JDBC path's {@code 57014} never happens, so the failure used + * to reach the web catch-all as a 500. The H2 database here only gives Hibernate somewhere to + * connect: the refusal is Hibernate's and does not depend on the vendor. + */ +class SpringTransactionDeadlineTest { + + private static final SpringTransaction.Timeouts ONE_SECOND = + new SpringTransaction.Timeouts(Duration.ofSeconds(1), Duration.ofSeconds(1)); + private static final Duration PAST_THE_DEADLINE = Duration.ofMillis(1_200); + + private SessionFactory sessionFactory; + private EntityManager entityManager; + private SpringTransaction port; + + @BeforeEach + void openHibernate() { + DriverManagerDataSource dataSource = + new DriverManagerDataSource( + "jdbc:h2:mem:deadline-" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1", "sa", ""); + Configuration configuration = new Configuration(); + configuration.getProperties().put("hibernate.connection.datasource", dataSource); + configuration.setProperty("hibernate.hbm2ddl.auto", "create"); + configuration.addAnnotatedClass(DeadlineRow.class); + sessionFactory = configuration.buildSessionFactory(); + entityManager = SharedEntityManagerCreator.createSharedEntityManager(sessionFactory); + port = + new SpringTransaction( + new JpaTransactionManager(sessionFactory), + new PersistenceExceptionTranslator(List.of(new StandardSqlStateErrorMapping())), + ONE_SECOND); + } + + @AfterEach + void closeHibernate() { + sessionFactory.close(); + } + + @Test + @DisplayName("마감이 지난 뒤 본문의 flush를 Hibernate가 거부하면 DB_QUERY_CANCELED로 알린다") + void aFlushHibernateRefusesPastTheDeadlineIsAQueryCancellation() { + // when + Throwable thrown = + catchThrowable( + () -> + port.inWrite( + () -> { + sleep(PAST_THE_DEADLINE); + entityManager.persist(new DeadlineRow(1L)); + entityManager.flush(); + return null; + })); + + // then + assertThat(thrown).isInstanceOf(PersistenceFailureException.class); + assertThat(((PersistenceFailureException) thrown).errorCode()) + .isEqualTo(OperationalError.DB_QUERY_CANCELED); + } + + @Test + @DisplayName("마감이 지난 뒤 커밋의 flush를 Hibernate가 거부해도 DB_QUERY_CANCELED로 알린다") + void aCommitFlushHibernateRefusesPastTheDeadlineIsAQueryCancellation() { + // when + Throwable thrown = + catchThrowable( + () -> + port.inWrite( + () -> { + sleep(PAST_THE_DEADLINE); + entityManager.persist(new DeadlineRow(2L)); + return null; + })); + + // then + assertThat(thrown).isInstanceOf(PersistenceFailureException.class); + assertThat(((PersistenceFailureException) thrown).errorCode()) + .isEqualTo(OperationalError.DB_QUERY_CANCELED); + } + + @Test + @DisplayName("마감이 지난 뒤라도 업무 코드가 던진 예외는 바꾸지 않고 그대로 전한다") + void aBusinessFailurePastTheDeadlineIsNotRelabelled() { + // given + IllegalStateException businessFailure = new IllegalStateException("the order is closed"); + + // when + Throwable thrown = + catchThrowable( + () -> + port.inWrite( + () -> { + sleep(PAST_THE_DEADLINE); + throw businessFailure; + })); + + // then + assertThat(thrown).isSameAs(businessFailure); + } + + private static void sleep(Duration duration) { + try { + Thread.sleep(duration); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(interrupted); + } + } + + /** The row the scenarios write; its table exists only to give the flush a statement. */ + @Entity + @Table(name = "deadline_row") + static class DeadlineRow { + @Id Long id; + + protected DeadlineRow() {} + + DeadlineRow(Long id) { + this.id = id; + } + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 index ca048746..bf86884e 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 +++ b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 @@ -40,7 +40,8 @@ b8f571e6cada72a8f2776f4462509ce4b535c964c75b1c3cfb34b9dc07446d59 postgresql/V17 44cacbde7d1381ca5290fb8ec28005e8127ff6b453ff84277ec8d1b3293d76c8 postgresql/V18__outbox_event_unpublished_aggregate_order.sql a0306e5b0fbcaf43984d256362dc4244aec57a0ce5031e4f5263be3762fe1caf postgresql/V19__outbox_event_canonical_only.sql c997fa20e10b68e0ca1b30cd7275018deaadc99bb0370b7b806332e9826e5217 postgresql/V20__outbox_event_deferred_attempt_count.sql -f830cda5a8cb72378c5af5d98810db61dc0dc36e416758c5da0de2ac34a68e27 postgresql/V21__outbox_aggregate_sequence.sql +21744ea1aab2b29fbb3417b9a5c4389522199413048cc7035aa649ffcdd852cf postgresql/V21__outbox_aggregate_sequence.sql +459942722c69d43b15fdab34849c6e34bbca35727e800eca18fbc5b3eebe5f4c postgresql/V22__outbox_event_last_failure_code.sql 8ab106731d4d04d3719da0692069947736dc490ca46aa5a3ea9bdf65a5d956dc postgresql/V1__idempotency_record.sql 6b62a01a4eeb792b81015b4810ecc6f3f021e0a17dea50c871493f7694d4f0b9 postgresql/V3__outbox_event.sql c57ab8a35a10079190ad8a563be58fae4147c9406c5bb752debb6998324ccdf6 postgresql/V4__int_lock.sql diff --git a/src/adapter/outbound/persistence-jpa/src/testFixtures/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/JpaArchitectureRules.java b/src/adapter/outbound/persistence-jpa/src/testFixtures/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/JpaArchitectureRules.java index 42c1f5ae..a35d8ef4 100644 --- a/src/adapter/outbound/persistence-jpa/src/testFixtures/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/JpaArchitectureRules.java +++ b/src/adapter/outbound/persistence-jpa/src/testFixtures/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/JpaArchitectureRules.java @@ -1,9 +1,12 @@ package dev.caskeleton.adapter.outbound.persistence.testkit.arch; +import static com.tngtech.archunit.core.domain.properties.CanBeAnnotated.Predicates.annotatedWith; import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.classes; import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.methods; import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.noClasses; +import com.tngtech.archunit.base.DescribedPredicate; +import com.tngtech.archunit.core.domain.JavaClass; import com.tngtech.archunit.lang.ArchRule; import jakarta.persistence.Embeddable; import jakarta.persistence.Entity; @@ -144,4 +147,36 @@ public final class JpaArchitectureRules { + " use a unidirectional @ManyToOne or an ID reference, and a repository query for" + " the collection"); } + + /** + * The entities of the leaf rooted at {@code leafPackage} are used by that leaf's classes only. + * + *

    A capability persistence leaf splits its entities and its stores into two packages (review + * 04-16), so the entity constructors and mutators the stores call are public, and Java can no + * longer say "this row never leaves the leaf". The ports carry records instead; this rule is what + * holds that line now that visibility does not (review RR-03-09). Nothing outside the leaf may + * name one of its {@code @Entity}, {@code @MappedSuperclass} or {@code @Embeddable} types. + * + * @param leafPackage the leaf's root package, e.g. {@code + * dev.caskeleton.adapter.outbound.persistence.notification} + */ + public static ArchRule entitiesAreUsedOnlyInsideTheirLeaf(String leafPackage) { + DescribedPredicate leafEntity = + JavaClass.Predicates.resideInAPackage(leafPackage + "..") + .and( + annotatedWith(Entity.class) + .or(annotatedWith(MappedSuperclass.class)) + .or(annotatedWith(Embeddable.class))) + .as("an entity of " + leafPackage); + return noClasses() + .that() + .resideOutsideOfPackage(leafPackage + "..") + .should() + .dependOnClassesThat(leafEntity) + .as("the entities of " + leafPackage + " are used inside that leaf only") + .because( + "the stores map them to the records the ports carry; an entity used elsewhere is" + + " mutable persistence state reached through its public mutators") + .allowEmptyShould(true); + } } diff --git a/src/adapter/outbound/persistence-jpa/src/testFixtures/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/SingleSidedAssociationCondition.java b/src/adapter/outbound/persistence-jpa/src/testFixtures/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/SingleSidedAssociationCondition.java index 0456c877..cbe87cde 100644 --- a/src/adapter/outbound/persistence-jpa/src/testFixtures/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/SingleSidedAssociationCondition.java +++ b/src/adapter/outbound/persistence-jpa/src/testFixtures/java/dev/caskeleton/adapter/outbound/persistence/testkit/arch/SingleSidedAssociationCondition.java @@ -2,69 +2,179 @@ package dev.caskeleton.adapter.outbound.persistence.testkit.arch; import com.tngtech.archunit.core.domain.JavaAnnotation; import com.tngtech.archunit.core.domain.JavaClass; +import com.tngtech.archunit.core.domain.JavaField; import com.tngtech.archunit.core.domain.JavaMember; +import com.tngtech.archunit.core.domain.JavaMethod; +import com.tngtech.archunit.core.domain.JavaParameterizedType; +import com.tngtech.archunit.core.domain.JavaType; import com.tngtech.archunit.lang.ArchCondition; import com.tngtech.archunit.lang.ConditionEvents; import com.tngtech.archunit.lang.SimpleConditionEvent; import jakarta.persistence.ManyToMany; +import jakarta.persistence.ManyToOne; import jakarta.persistence.OneToMany; import jakarta.persistence.OneToOne; +import java.lang.annotation.Annotation; import java.util.List; +import java.util.Optional; import java.util.stream.Stream; /** - * Reports every association declared with {@code mappedBy} — the inverse half of a bidirectional - * pair. + * Reports every association that is one half of a bidirectional pair. * - *

    {@code mappedBy} is the one attribute that says "this field is a second copy of a link the - * other side owns". JPA writes only the owning side, so the inverse copy is correct only while - * application code remembers to update both — and nothing fails when it forgets: the row is left - * unlinked, or the in-memory graph disagrees with the database until the next load. Checking the - * attribute rather than the annotation keeps a unidirectional {@code @OneToMany} with its own - * {@code @JoinColumn}, which has nothing to keep in step, out of the report. + *

    A pair is two entities whose associations point at each other. JPA writes the link from one + * place only, so the second copy is correct only while application code remembers to update both — + * and nothing fails when it forgets: the row is left unlinked, or the in-memory graph disagrees + * with the database until the next load. There are two ways to write the pair: * - *

    Fields and methods are both inspected, because property access puts the mapping on the getter. + *

      + *
    • With {@code mappedBy}: the inverse half says so. Only that half is reported, because the + * foreign key belongs to the owning {@code @ManyToOne} either way. + *
    • Without it: both halves map the same join column, one of them usually {@code insertable = + * false, updatable = false} (Hibernate's {@code @JoinColumn} form). Neither half says it is + * the copy, so both are reported (review RR-03-04). Reading the attribute alone let this form + * pass. + *
    + * + *

    A unidirectional {@code @OneToMany} with its own {@code @JoinColumn}, or a {@code @ManyToOne} + * nobody points back at, has nothing to keep in step and passes. A self-reference is a pair only + * when a second member of the same class points back the other way ({@code @ManyToOne parent} and + * {@code @OneToMany children}); two {@code @ManyToOne} links of a class to itself are two one-way + * links. Fields and methods are both inspected, because property access puts the mapping on the + * getter. */ public final class SingleSidedAssociationCondition extends ArchCondition { - private static final List> INVERSE_CAPABLE = - List.of(OneToMany.class, OneToOne.class, ManyToMany.class); + private static final List> ASSOCIATIONS = + List.of(OneToMany.class, ManyToOne.class, OneToOne.class, ManyToMany.class); + + private static final String ALTERNATIVE = + " Map each association from one side only: keep the owning @ManyToOne (LAZY) or hold the" + + " other aggregate's ID, and load the collection with a repository query" + + " (docs/jpa/entity-mapping-guide.md, Associations)"; public SingleSidedAssociationCondition() { - super("map each association from one side only (no mappedBy)"); + super("map each association from one side only (no bidirectional pair)"); } @Override public void check(JavaClass mappedType, ConditionEvents events) { - Stream.concat(mappedType.getFields().stream(), mappedType.getMethods().stream()) - .forEach(member -> reportInverseSide(mappedType, member, events)); + members(mappedType).forEach(member -> report(mappedType, member, events)); } - private static void reportInverseSide( - JavaClass mappedType, JavaMember member, ConditionEvents events) { - for (Class type : INVERSE_CAPABLE) { - if (!member.isAnnotatedWith(type)) { + private static void report(JavaClass mappedType, JavaMember member, ConditionEvents events) { + Optional association = Association.of(member); + if (association.isEmpty()) { + return; + } + Association own = association.get(); + if (!own.mappedBy().isEmpty()) { + events.add( + SimpleConditionEvent.violated( + mappedType, + name(mappedType, member) + + " is @" + + own.annotation() + + "(mappedBy = \"" + + own.mappedBy() + + "\"), the inverse side of a bidirectional association." + + ALTERNATIVE)); + return; + } + JavaClass target = own.target(); + boolean selfReference = target.getName().equals(mappedType.getName()); + for (JavaMember counterpart : members(target).toList()) { + if (counterpart.equals(member)) { continue; } - JavaAnnotation annotation = member.getAnnotationOfType(type.getName()); - String mappedBy = String.valueOf(annotation.get("mappedBy").orElse("")); - if (mappedBy.isEmpty()) { + Optional back = Association.of(counterpart); + if (back.isEmpty() + || !back.get().target().getName().equals(mappedType.getName()) + || !back.get().mappedBy().isEmpty()) { + // No pair, or a mappedBy pair whose inverse half is reported on its own. + continue; + } + if (selfReference && !back.get().annotation().equals(inverseOf(own.annotation()))) { + // Two links of one class to itself (a manager and a mentor) are two one-way links. continue; } events.add( SimpleConditionEvent.violated( mappedType, - mappedType.getSimpleName() - + "." - + member.getName() - + " is @" - + type.getSimpleName() - + "(mappedBy = \"" - + mappedBy - + "\"), the inverse side of a bidirectional association. Map each association" - + " from one side only: keep the owning @ManyToOne (LAZY) or hold the other" - + " aggregate's ID, and load the collection with a repository query" - + " (docs/jpa/entity-mapping-guide.md, Associations)")); + name(mappedType, member) + + " (@" + + own.annotation() + + ") and " + + name(target, counterpart) + + " (@" + + back.get().annotation() + + ") point at each other without mappedBy: two fields map one link, and" + + " the read-only one does not follow a change made through the other." + + ALTERNATIVE)); + return; + } + } + + /** The annotation the other half of a pair carries. */ + private static String inverseOf(String annotation) { + return switch (annotation) { + case "ManyToOne" -> "OneToMany"; + case "OneToMany" -> "ManyToOne"; + default -> annotation; + }; + } + + private static Stream members(JavaClass type) { + return Stream.concat(type.getFields().stream(), type.getMethods().stream()); + } + + private static String name(JavaClass owner, JavaMember member) { + return owner.getSimpleName() + "." + member.getName(); + } + + /** One association member: its annotation, its {@code mappedBy}, and the entity it points at. */ + private record Association(String annotation, String mappedBy, JavaClass target) { + + static Optional of(JavaMember member) { + for (Class type : ASSOCIATIONS) { + if (!member.isAnnotatedWith(type)) { + continue; + } + JavaAnnotation annotation = member.getAnnotationOfType(type.getName()); + String mappedBy = String.valueOf(annotation.get("mappedBy").orElse("")); + return target(member, annotation) + .map(target -> new Association(type.getSimpleName(), mappedBy, target)); + } + return Optional.empty(); + } + + /** {@code targetEntity} when set, else the declared type, else a collection's element type. */ + private static Optional target(JavaMember member, JavaAnnotation annotation) { + Optional declared = + annotation + .get("targetEntity") + .filter(JavaClass.class::isInstance) + .map(JavaClass.class::cast) + .filter(type -> !type.getName().equals("void")); + if (declared.isPresent()) { + return declared; + } + JavaType type = + switch (member) { + case JavaField field -> field.getType(); + case JavaMethod method -> method.getReturnType(); + default -> null; + }; + if (type == null) { + return Optional.empty(); + } + if (type instanceof JavaParameterizedType parameterized + && !parameterized.getActualTypeArguments().isEmpty()) { + List arguments = parameterized.getActualTypeArguments(); + // Map: the entity is the value. + return Optional.of(arguments.get(arguments.size() - 1).toErasure()); + } + return Optional.of(type.toErasure()); } } } diff --git a/src/adapter/outbound/persistence-notification/CLAUDE.md b/src/adapter/outbound/persistence-notification/CLAUDE.md index e7b25f5c..9d736619 100644 --- a/src/adapter/outbound/persistence-notification/CLAUDE.md +++ b/src/adapter/outbound/persistence-notification/CLAUDE.md @@ -13,8 +13,11 @@ outbox helpers), `NotificationRecordMapper` and the payload codec. - `configuration` — the auto-configuration facade that registers the stores as beans; the leaf root holds the scan configuration and the schema stream. - Entity accessors are public because the stores read them from another package; an entity never - leaves the leaf (the ports carry records). + Entity constructors, accessors and mutators are public because the stores use them from another + package; an entity never leaves the leaf (the ports carry records). Visibility no longer says so, + so a rule does: `JpaArchitectureRules.entitiesAreUsedOnlyInsideTheirLeaf`, run over the production + graph by `JpaProductionArchitectureTest` (app-bootstrap `architectureTest`) for this leaf and + `persistence-fileserver` (review RR-03-09). ## Where the tests are diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcReconciliationJobStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcReconciliationJobStore.java index 444b180a..03d1345e 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcReconciliationJobStore.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcReconciliationJobStore.java @@ -38,6 +38,12 @@ import org.springframework.jdbc.core.RowMapper; * — open across that call is exactly what the lease exists to avoid. {@link #reschedule} and {@link * #complete} match on the fence the job was claimed with, so a holder that outlived its lease * changes nothing once somebody else has claimed the job. + * + *

    A claim whose lease ran out without a reschedule or a completion counts as an attempt: the + * claim that takes the job over adds one to {@code attempts}. {@link #reschedule} clears {@code + * claimed_until}, so a lease still set at claim time is exactly that abandoned claim. Without it a + * worker that dies on the same job every time would reclaim it every lease, forever, below the + * attempt limit — the file server's reclaim counts the same way (review RR-03-08). */ public final class JdbcReconciliationJobStore implements ReconciliationJobStorePort { @@ -64,7 +70,8 @@ public final class JdbcReconciliationJobStore implements ReconciliationJobStoreP FOR UPDATE SKIP LOCKED) UPDATE notification_reconciliation_job AS job SET claimed_until = ?, - claim_fence = job.claim_fence + 1 + claim_fence = job.claim_fence + 1, + attempts = job.attempts + CASE WHEN job.claimed_until IS NULL THEN 0 ELSE 1 END FROM claimable WHERE job.id = claimable.id RETURNING job.id, job.attempt_id, job.provider_profile_id, job.next_check_at, job.attempts, diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientClaimSql.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientClaimSql.java index 6d11bbb5..79a60d15 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientClaimSql.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientClaimSql.java @@ -10,10 +10,8 @@ package dev.caskeleton.adapter.outbound.persistence.notification.platform.reposi * *

    The lease store contract ({@code RecipientLeaseStoreContract}, run on PostgreSQL by {@code * PostgreSqlRecipientLeaseStoreContractTest}) now drives this statement through the repository and - * the store, so no test retypes it or runs it beside the adapter (review 2026-11-01 04-14). - * - *

    The named parameters are Spring Data's. {@link #positional(String)} rewrites them to JDBC - * placeholders for a caller with a bare {@code Connection}. + * the store, so no test retypes it or runs it beside the adapter (review 2026-11-01 04-14). The + * named parameters are Spring Data's. */ public final class RecipientClaimSql { @@ -70,14 +68,4 @@ public final class RecipientClaimSql { + " LIMIT :batchSize)"; private RecipientClaimSql() {} - - /** - * Rewrites the named parameters to JDBC placeholders, left to right. - * - * @param namedSql one of the statements above - * @return the same statement with {@code ?} in place of every {@code :name} - */ - public static String positional(String namedSql) { - return namedSql.replaceAll("(? :expected::jsonb")) - .isEqualTo("payload::jsonb @> ?::jsonb"); - } -} diff --git a/src/adapter/outbound/redis-runtime/build.gradle b/src/adapter/outbound/redis-runtime/build.gradle index ab55a982..f6ac29d0 100644 --- a/src/adapter/outbound/redis-runtime/build.gradle +++ b/src/adapter/outbound/redis-runtime/build.gradle @@ -14,6 +14,11 @@ dependencies { implementation 'io.lettuce:lettuce-core' implementation 'io.projectreactor:reactor-core' implementation 'org.slf4j:slf4j-api' + // Per-lane wait, refusal and reconnect meters (review 2026-11-03 RR-04-02). RedisLaneMetrics in + // the config package is the only importer: the connection lifecycle reports through + // RedisLaneListener and knows nothing of Micrometer, and a context without a MeterRegistry bean + // composes no meter at all. + implementation 'io.micrometer:micrometer-core' // Test fixtures only: the in-memory gateway and the real-topology endpoint that both redis-sdk's // and cache-redis's tests drive. One fake of RedisCommandGateway, owned by the leaf that owns it. diff --git a/src/adapter/outbound/redis-runtime/gradle.lockfile b/src/adapter/outbound/redis-runtime/gradle.lockfile index a83112f9..67ed098f 100644 --- a/src/adapter/outbound/redis-runtime/gradle.lockfile +++ b/src/adapter/outbound/redis-runtime/gradle.lockfile @@ -42,6 +42,7 @@ io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnota io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor io.lettuce:lettuce-core:6.8.2.RELEASE=compileClasspath,runtimeClasspath,testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath io.micrometer:micrometer-commons:1.16.7=compileClasspath,runtimeClasspath,testCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath +io.micrometer:micrometer-core:1.16.7=compileClasspath,runtimeClasspath,testCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath io.micrometer:micrometer-observation:1.16.7=compileClasspath,runtimeClasspath,testCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath io.netty:netty-buffer:4.2.17.Final=compileClasspath,runtimeClasspath,testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath io.netty:netty-codec-base:4.2.17.Final=compileClasspath,runtimeClasspath,testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath @@ -90,6 +91,7 @@ org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle org.codehaus.plexus:plexus-utils:3.3.0=checkstyle org.dom4j:dom4j:2.2.0=spotbugs org.hamcrest:hamcrest:3.0=testCompileClasspath,testRuntimeClasspath +org.hdrhistogram:HdrHistogram:2.2.2=runtimeClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath org.javassist:javassist:3.28.0-GA=checkstyle org.jspecify:jspecify:1.0.1=annotationProcessor,checkstyle,compileClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testFixturesAnnotationProcessor,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath org.junit.jupiter:junit-jupiter-api:6.0.3=testCompileClasspath,testRuntimeClasspath @@ -101,6 +103,7 @@ org.junit.platform:junit-platform-engine:6.0.3=testRuntimeClasspath org.junit.platform:junit-platform-launcher:6.0.3=testRuntimeClasspath org.junit:junit-bom:6.0.3=testCompileClasspath,testRuntimeClasspath org.junit:junit-bom:6.1.0=spotbugs +org.latencyutils:LatencyUtils:2.0.3=runtimeClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath org.mockito:mockito-core:5.20.0=mockitoAgent,testCompileClasspath,testRuntimeClasspath org.mockito:mockito-junit-jupiter:5.20.0=testCompileClasspath,testRuntimeClasspath org.objenesis:objenesis:3.3=testRuntimeClasspath diff --git a/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisLaneMetrics.java b/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisLaneMetrics.java new file mode 100644 index 00000000..5d4b9476 --- /dev/null +++ b/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisLaneMetrics.java @@ -0,0 +1,161 @@ +package dev.caskeleton.adapter.outbound.cache.redis.runtime.config; + +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLane; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLaneListener; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisRuntimeOwner; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import java.time.Duration; +import java.util.Locale; +import java.util.Objects; + +/** + * Each Redis connection lane's wait, refusals and reconnects, as meters (review 2026-11-03 + * RR-04-02). + * + *

    The question these answer is the one the cache's own lane ({@code CACHE_SCRIPT}) was made for: + * is a lane running out, and is it the lane that may run out? A refusal counted for the whole + * runtime cannot say whether the cache or idempotency ran dry, and a reconnect nobody counts is the + * extra load a discarded connection adds to a Redis that is already slow. So every meter carries + * the lane's name, and every lane the owner pools — the runtime's own, the cache's, the typed SDK's + * — has its series from startup on. + * + * + * + * + * + * + * + * + * + * + * + *
    Meters
    NameTypeTagsAnswers
    {@value #ACQUIRE_WAIT}timerlane, outcome (admitted, refused)how long a borrow waited for a connection
    {@value #ACQUIRE_REFUSED}counterlane, reason (saturated, closing, + * interrupted, connect_failed)why a borrow got no connection
    {@value #CONNECTIONS_OPENED}counterlanenew connections: + * the lane's first ones, then every reconnect
    {@value #CONNECTIONS_DISCARDED}counterlane, reason (invalidated, + * broken)connections closed instead of pooled — what the reconnects replace
    {@value #CONNECTIONS_ACTIVE}gaugelaneconnections lent out + * now
    {@value #CONNECTIONS_PENDING}gaugelaneborrows waiting for + * a connection now
    + * + *

    The lane tag is the lane's name ({@code REGULAR}, {@code SCRIPT}, {@code CACHE_SCRIPT}, the + * SDK's {@code BLOCKING}, {@code TRANSACTION}, {@code ADMIN}): a closed set of enum constants, the + * same spelling the lane refusal's log line carries. The registry rows are in {@code + * docs/registries/metrics.yaml}. + */ +public final class RedisLaneMetrics implements RedisLaneListener { + + /** How long a borrow waited for a lane permit. */ + public static final String ACQUIRE_WAIT = "redis.lane.acquire.wait.seconds"; + + /** Borrows refused before any command existed. */ + public static final String ACQUIRE_REFUSED = "redis.lane.acquire.refused.total"; + + /** Connections opened: a lane's first ones, then every reconnect. */ + public static final String CONNECTIONS_OPENED = "redis.lane.connections.opened.total"; + + /** Connections closed instead of returned to the pool. */ + public static final String CONNECTIONS_DISCARDED = "redis.lane.connections.discarded.total"; + + /** Connections lent out right now. */ + public static final String CONNECTIONS_ACTIVE = "redis.lane.connections.active"; + + /** Borrows waiting for a connection right now. */ + public static final String CONNECTIONS_PENDING = "redis.lane.connections.pending"; + + private static final String LANE = "lane"; + private static final String REASON = "reason"; + + private final MeterRegistry registry; + + /** + * Creates the meters' owner; nothing is registered until {@link #bind(RedisRuntimeOwner)}. + * + * @param registry where the meters go + */ + public RedisLaneMetrics(MeterRegistry registry) { + this.registry = Objects.requireNonNull(registry, "registry must be non-null"); + } + + /** + * Registers every lane's gauges and zero-valued counters. + * + *

    The counters up front because a rate over a series that appears with its first refusal + * misses that refusal, and an alert on "refusals above zero" is exactly about the first one. + * + * @param owner the owner whose lanes are measured + */ + public void bind(RedisRuntimeOwner owner) { + Objects.requireNonNull(owner, "owner must be non-null"); + for (RedisLane lane : owner.lanes()) { + Gauge.builder(CONNECTIONS_ACTIVE, owner, measured -> measured.outstanding(lane)) + .description("Redis connections of this lane lent out now") + .tag(LANE, lane.name()) + .register(registry); + Gauge.builder(CONNECTIONS_PENDING, owner, measured -> measured.waiting(lane)) + .description("borrows waiting for a Redis connection of this lane now") + .tag(LANE, lane.name()) + .register(registry); + for (Refusal reason : Refusal.values()) { + refusals(lane, reason); + } + for (Discard reason : Discard.values()) { + discards(lane, reason); + } + openings(lane); + } + } + + @Override + public void waited(RedisLane lane, Duration waited, boolean admitted) { + Timer.builder(ACQUIRE_WAIT) + .description("how long a borrow waited for a Redis connection of this lane") + .tag(LANE, lane.name()) + .tag("outcome", admitted ? "admitted" : "refused") + // A histogram, so the p99 of the wait is a query rather than a guess from the mean. + .publishPercentileHistogram() + .minimumExpectedValue(Duration.ofMillis(1)) + .maximumExpectedValue(Duration.ofSeconds(10)) + .register(registry) + .record(waited); + } + + @Override + public void refused(RedisLane lane, Refusal reason) { + refusals(lane, reason).increment(); + } + + @Override + public void opened(RedisLane lane) { + openings(lane).increment(); + } + + @Override + public void discarded(RedisLane lane, Discard reason) { + discards(lane, reason).increment(); + } + + private Counter refusals(RedisLane lane, Refusal reason) { + return Counter.builder(ACQUIRE_REFUSED) + .description("borrows of this Redis lane refused before any command was sent") + .tag(LANE, lane.name()) + .tag(REASON, reason.name().toLowerCase(Locale.ROOT)) + .register(registry); + } + + private Counter openings(RedisLane lane) { + return Counter.builder(CONNECTIONS_OPENED) + .description("Redis connections opened for this lane, reconnects included") + .tag(LANE, lane.name()) + .register(registry); + } + + private Counter discards(RedisLane lane, Discard reason) { + return Counter.builder(CONNECTIONS_DISCARDED) + .description("Redis connections of this lane closed instead of pooled") + .tag(LANE, lane.name()) + .tag(REASON, reason.name().toLowerCase(Locale.ROOT)) + .register(registry); + } +} diff --git a/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisRuntimeAutoConfiguration.java b/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisRuntimeAutoConfiguration.java index 9dfabc89..f7535286 100644 --- a/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisRuntimeAutoConfiguration.java +++ b/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisRuntimeAutoConfiguration.java @@ -7,6 +7,7 @@ import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisRunti import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisRuntimeOwner; import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisTopologyClientFactory; import dev.caskeleton.adapter.outbound.cache.redis.runtime.key.RedisNamespace; +import io.micrometer.core.instrument.MeterRegistry; import java.nio.charset.StandardCharsets; import java.util.LinkedHashMap; import java.util.List; @@ -232,16 +233,23 @@ public class RedisRuntimeAutoConfiguration { * further lanes on the runtime contributes their ceilings as {@link RedisLaneContribution} beans; * the runtime pools and drains those exactly like its own, and knows nothing else about them. * + *

    With a {@link MeterRegistry} in the context every lane — the runtime's, the cache's, the + * SDK's — gets its wait, refusal and reconnect meters ({@link RedisLaneMetrics}, review + * 2026-11-03 RR-04-02). Without one the owner reports to nobody and nothing about metrics is + * created. + * * @param client the runtime client * @param settings the validated settings * @param contributedLanes lanes other modules compose on the runtime + * @param meterRegistry where the lane meters go, when the application has a registry * @return the lifecycle owner */ @Bean(destroyMethod = "close") public RedisRuntimeOwner redisRuntimeOwner( RedisRuntimeClient client, RedisRuntimeSettings settings, - ObjectProvider contributedLanes) { + ObjectProvider contributedLanes, + ObjectProvider meterRegistry) { Map limits = new LinkedHashMap<>(); limits.put(RedisConnectionKind.REGULAR, settings.getCapacity().getMaximumInFlightCommands()); limits.put(RedisConnectionKind.SCRIPT, settings.getCapacity().getMaximumScriptConnections()); @@ -255,11 +263,24 @@ public class RedisRuntimeAutoConfiguration { "Redis lane " + contribution.lane().name() + " was composed twice"); } }); - return new RedisRuntimeOwner( - client, - limits, - settings.getLifecycle().getAcquireTimeout(), - settings.getLifecycle().getDrainTimeout()); + MeterRegistry registry = meterRegistry.getIfUnique(); + if (registry == null) { + return new RedisRuntimeOwner( + client, + limits, + settings.getLifecycle().getAcquireTimeout(), + settings.getLifecycle().getDrainTimeout()); + } + RedisLaneMetrics metrics = new RedisLaneMetrics(registry); + RedisRuntimeOwner owner = + new RedisRuntimeOwner( + client, + limits, + settings.getLifecycle().getAcquireTimeout(), + settings.getLifecycle().getDrainTimeout(), + metrics); + metrics.bind(owner); + return owner; } /** diff --git a/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/connection/RedisLaneListener.java b/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/connection/RedisLaneListener.java new file mode 100644 index 00000000..12fa70a8 --- /dev/null +++ b/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/connection/RedisLaneListener.java @@ -0,0 +1,77 @@ +package dev.caskeleton.adapter.outbound.cache.redis.runtime.connection; + +import java.time.Duration; + +/** + * What happens at a lane's door: how long a borrow waited, why one was refused, and when a + * connection was opened or thrown away. + * + *

    The owner reports these; it does not count them. Turning them into meters is the composition's + * job ({@code RedisLaneMetrics}), so the connection lifecycle stays free of any metrics library and + * a context without one pays nothing. + * + *

    Called on the borrowing or returning thread, never while the owner holds its lock. An + * implementation must be cheap and must not block. One that throws is logged and ignored: a meter + * that cannot be recorded is never a reason to refuse a connection. + */ +public interface RedisLaneListener { + + /** Reports nothing. */ + RedisLaneListener NONE = new RedisLaneListener() {}; + + /** + * A borrow finished waiting for a lane permit. + * + * @param lane the lane + * @param waited how long the borrow waited; near zero when the lane had room + * @param admitted {@code true} when the borrow got a permit, {@code false} when it gave up + */ + default void waited(RedisLane lane, Duration waited, boolean admitted) {} + + /** + * A borrow was refused before any command existed. + * + * @param lane the lane + * @param reason why + */ + default void refused(RedisLane lane, Refusal reason) {} + + /** + * A new connection was opened for a lane: the lane's first ones, and every reconnect after a + * connection was thrown away. + * + * @param lane the lane + */ + default void opened(RedisLane lane) {} + + /** + * A connection was closed instead of going back to the pool. + * + * @param lane the lane + * @param reason why + */ + default void discarded(RedisLane lane, Discard reason) {} + + /** Why a borrow was refused. Every one of them is "nothing was sent". */ + enum Refusal { + /** The lane stayed at its ceiling for the whole wait. */ + SATURATED, + /** The runtime is draining or closed. */ + CLOSING, + /** The borrowing thread was interrupted while it waited. */ + INTERRUPTED, + /** A connection had to be opened and could not be. */ + CONNECT_FAILED + } + + /** Why a connection was closed instead of pooled. */ + enum Discard { + /** + * Its holder marked it unusable — a command that timed out or failed, a transaction whose + * cleanup did not land. The next borrow on the lane opens a new connection. + */ + INVALIDATED, + /** The driver reported it closed, while it sat in the pool or when it came back. */ + BROKEN + } +} diff --git a/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/connection/RedisRuntimeOwner.java b/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/connection/RedisRuntimeOwner.java index 45014405..5ad8134f 100644 --- a/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/connection/RedisRuntimeOwner.java +++ b/src/adapter/outbound/redis-runtime/src/main/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/connection/RedisRuntimeOwner.java @@ -11,9 +11,11 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,6 +57,7 @@ public final class RedisRuntimeOwner implements AutoCloseable { private final Map permits; private final Duration acquireTimeout; private final Duration drainTimeout; + private final RedisLaneListener listener; private final AtomicReference state = new AtomicReference<>(State.OPEN); private final Object monitor = new Object(); @@ -73,6 +76,27 @@ public final class RedisRuntimeOwner implements AutoCloseable { Map limits, Duration acquireTimeout, Duration drainTimeout) { + this(client, limits, acquireTimeout, drainTimeout, RedisLaneListener.NONE); + } + + /** + * Creates the owner, reporting what happens at each lane's door. + * + * @param client the topology client whose connections this owns + * @param limits per-lane maximum connections; see {@link #RedisRuntimeOwner(RedisRuntimeClient, + * Map, Duration, Duration)} + * @param acquireTimeout how long a borrow waits for a returned connection when its lane is full + * @param drainTimeout how long shutdown waits for outstanding leases + * @param listener told of every wait, refusal, opened and discarded connection, per lane (review + * 2026-11-03 RR-04-02); {@link RedisLaneListener#NONE} to report nothing + */ + public RedisRuntimeOwner( + RedisRuntimeClient client, + Map limits, + Duration acquireTimeout, + Duration drainTimeout, + RedisLaneListener listener) { + this.listener = Objects.requireNonNull(listener, "listener must be non-null"); this.client = Objects.requireNonNull(client, "client must be non-null"); this.acquireTimeout = Objects.requireNonNull(acquireTimeout, "acquire timeout must be non-null"); @@ -175,6 +199,7 @@ public final class RedisRuntimeOwner implements AutoCloseable { + " was never composed; the module that uses it contributes its limit"); } if (state.get() != State.OPEN) { + report(sink -> sink.refused(kind, RedisLaneListener.Refusal.CLOSING)); throw notAccepting(kind); } // The limit is the lane's connection count: a lease holds one connection for one round trip, @@ -184,13 +209,22 @@ public final class RedisRuntimeOwner implements AutoCloseable { // thread waiting here does not pin its carrier. Semaphore lane = permits.get(kind); boolean admitted; + long waitStarted = System.nanoTime(); try { admitted = lane.tryAcquire(wait.toNanos(), TimeUnit.NANOSECONDS); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); + Duration waited = Duration.ofNanos(System.nanoTime() - waitStarted); + report(sink -> sink.waited(kind, waited, false)); + report(sink -> sink.refused(kind, RedisLaneListener.Refusal.INTERRUPTED)); throw reject(kind, "interrupted while waiting for a connection on lane " + kind.name()); } + // Every borrow's wait is reported, the near-zero ones included: a lane whose admitted waits + // grow is a lane about to refuse, and only the distribution shows it before the refusals do. + Duration waited = Duration.ofNanos(System.nanoTime() - waitStarted); + report(sink -> sink.waited(kind, waited, admitted)); if (!admitted) { + report(sink -> sink.refused(kind, RedisLaneListener.Refusal.SATURATED)); throw reject( kind, "connection lane " @@ -200,12 +234,18 @@ public final class RedisRuntimeOwner implements AutoCloseable { + " and no connection was returned within " + wait); } + boolean closing; synchronized (monitor) { - if (state.get() != State.OPEN) { + closing = state.get() != State.OPEN; + if (closing) { lane.release(); - throw notAccepting(kind); + } else { + outstanding.put(kind, outstanding.get(kind) + 1); } - outstanding.put(kind, outstanding.get(kind) + 1); + } + if (closing) { + report(sink -> sink.refused(kind, RedisLaneListener.Refusal.CLOSING)); + throw notAccepting(kind); } RedisRuntimeClient.RedisLaneConnection connection = null; try { @@ -219,6 +259,7 @@ public final class RedisRuntimeOwner implements AutoCloseable { // would otherwise get a failure that looks like theirs. closeQuietly(connection); connection = null; + report(sink -> sink.discarded(kind, RedisLaneListener.Discard.BROKEN)); } if (connection == null) { connection = openLane(kind, routingKey); @@ -228,7 +269,7 @@ public final class RedisRuntimeOwner implements AutoCloseable { if (connection != null) { closeQuietly(connection); } - release(kind, null, false); + release(kind, null, false, false); throw failure; } } @@ -244,27 +285,43 @@ public final class RedisRuntimeOwner implements AutoCloseable { */ private RedisRuntimeClient.RedisLaneConnection openLane( RedisLane kind, Optional routingKey) { + RedisRuntimeClient.RedisLaneConnection opened; try { - return client.openLane(kind, routingKey); + opened = client.openLane(kind, routingKey); } catch (RedisLaneUnavailableException refused) { + report(sink -> sink.refused(kind, RedisLaneListener.Refusal.CONNECT_FAILED)); throw refused; } catch (RuntimeException failure) { + report(sink -> sink.refused(kind, RedisLaneListener.Refusal.CONNECT_FAILED)); throw new RedisLaneUnavailableException( kind.name(), "a connection for lane " + kind.name() + " could not be opened; nothing was sent", failure); } + // Counted per lane because every one after the lane's first few is a reconnect: a connection + // was thrown away (see discarded) and a slow Redis now also pays for a new handshake. + report(sink -> sink.opened(kind)); + return opened; } private void release( - RedisLane kind, RedisRuntimeClient.RedisLaneConnection connection, boolean reusable) { + RedisLane kind, + RedisRuntimeClient.RedisLaneConnection connection, + boolean pooled, + boolean invalidated) { boolean drained; + RedisLaneListener.Discard discarded = null; synchronized (monitor) { outstanding.put(kind, outstanding.get(kind) - 1); if (connection != null) { - if (reusable && state.get() == State.OPEN && connection.open()) { + boolean open = connection.open(); + if (pooled && !invalidated && state.get() == State.OPEN && open) { idle.get(kind).addLast(connection); connection = null; + } else if (invalidated) { + discarded = RedisLaneListener.Discard.INVALIDATED; + } else if (!open) { + discarded = RedisLaneListener.Discard.BROKEN; } } drained = state.get() == State.DRAINING && totalOutstanding() == 0; @@ -279,6 +336,23 @@ public final class RedisRuntimeOwner implements AutoCloseable { } // After the counter, so a waiter admitted by this permit never sees the lane over its limit. permits.get(kind).release(); + if (discarded != null) { + RedisLaneListener.Discard reason = discarded; + report(sink -> sink.discarded(kind, reason)); + } + } + + /** + * Tells the listener, outside the lock, and never lets it fail the borrow or the return. + * + * @param event what to tell it + */ + private void report(Consumer event) { + try { + event.accept(listener); + } catch (RuntimeException failure) { + LOG.debug("a Redis lane listener failed; the borrow is unaffected", failure); + } } private int totalOutstanding() { @@ -357,6 +431,31 @@ public final class RedisRuntimeOwner implements AutoCloseable { } } + /** + * Returns how many borrows are waiting for a connection of a lane right now. + * + *

    An estimate, as {@link Semaphore#getQueueLength()} is: meant for a gauge, not for control. + * + * @param lane the lane + * @return the waiting borrows + */ + public int waiting(RedisLane lane) { + Semaphore permit = permits.get(Objects.requireNonNull(lane, "lane must be non-null")); + if (permit == null) { + throw new IllegalArgumentException("connection lane " + lane.name() + " was never composed"); + } + return permit.getQueueLength(); + } + + /** + * Returns every lane this owner pools: the runtime's own and the ones other modules composed. + * + * @return the lanes + */ + public Set lanes() { + return limits.keySet(); + } + /** * Returns the bound deployment mode. * @@ -371,14 +470,15 @@ public final class RedisRuntimeOwner implements AutoCloseable { private final RedisLane kind; private final RedisRuntimeClient.RedisLaneConnection connection; - private boolean reusable; + private final boolean pooled; + private boolean invalidated; private boolean closed; private Lease( - RedisLane kind, RedisRuntimeClient.RedisLaneConnection connection, boolean reusable) { + RedisLane kind, RedisRuntimeClient.RedisLaneConnection connection, boolean pooled) { this.kind = kind; this.connection = connection; - this.reusable = reusable; + this.pooled = pooled; } @Override @@ -409,7 +509,7 @@ public final class RedisRuntimeOwner implements AutoCloseable { // A connection whose cleanup failed — a transaction whose DISCARD did not land, a window // that may still be open — must not go back into the pool. The next borrower would queue // their command into somebody else's MULTI. - reusable = false; + invalidated = true; } @Override @@ -418,7 +518,7 @@ public final class RedisRuntimeOwner implements AutoCloseable { return; } closed = true; - release(kind, connection, reusable); + release(kind, connection, pooled, invalidated); } } } diff --git a/src/adapter/outbound/redis-runtime/src/main/resources/config/module-defaults/redis-runtime.yml b/src/adapter/outbound/redis-runtime/src/main/resources/config/module-defaults/redis-runtime.yml new file mode 100644 index 00000000..2dc63100 --- /dev/null +++ b/src/adapter/outbound/redis-runtime/src/main/resources/config/module-defaults/redis-runtime.yml @@ -0,0 +1,13 @@ +# redis-runtime 의 redisOptional 은 Redis 에 닿지 못하면 DEGRADED 를 보고한다. Boot 기본 순서 +# (DOWN, OUT_OF_SERVICE, UP, UNKNOWN)에 없는 상태는 DOWN 보다 앞에 정렬돼, 다른 contributor 가 DOWN +# 이어도 전체 health 가 HTTP 200 DEGRADED 로 보였다(리뷰 05-23e). DEGRADED 를 DOWN 뒤, UP 앞에 둔다. +# +# 상태를 내는 이 leaf 가 순서도 싣는다. 예전에는 cache-redis 가 실었는데, redis-sdk 만 쓰는 조립은 +# redis-runtime 을 끌어오고 cache-redis 는 끌어오지 않아 순서 없이 DEGRADED 가 났다(리뷰 2026-11-03 +# RR-04-10). 이 키를 정하는 module-defaults 는 이 파일 하나뿐이다(두 파일이 같은 키를 실으면 기동이 +# 멈춘다). 서비스가 순서를 바꾸면 DEGRADED 를 넣어야 한다. +management: + endpoint: + health: + status: + order: DOWN,OUT_OF_SERVICE,DEGRADED,UP,UNKNOWN diff --git a/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisLaneMetricsTest.java b/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisLaneMetricsTest.java new file mode 100644 index 00000000..37f4bb85 --- /dev/null +++ b/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisLaneMetricsTest.java @@ -0,0 +1,388 @@ +package dev.caskeleton.adapter.outbound.cache.redis.runtime.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.adapter.outbound.cache.redis.runtime.RedisDeploymentMode; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.command.SemanticRedisCommands; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisAccount; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisConnectionKind; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisDriverConnection; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLane; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLaneListener; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLaneUnavailableException; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLease; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisRuntimeClient; +import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisRuntimeOwner; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The lanes' own meters: how long a borrow waited, why one was refused, and how often a connection + * was opened or thrown away (review 2026-11-03 RR-04-02). + * + *

    The cache got a lane of its own so that cache fills cannot take the connections idempotency, + * rate limiting and leases need. Whether that holds under load is a question only per-lane numbers + * answer: a refusal counted for the whole runtime cannot say which lane ran out, and a reconnect + * nobody counts is the extra load a timed-out lease adds to a Redis that is already slow. + */ +class RedisLaneMetricsTest { + + /** A lane another module composes on the runtime, the way the cache does. */ + private enum ContributedLane implements RedisLane { + EXCLUSIVE; + + @Override + public RedisAccount account() { + return RedisAccount.APPLICATION; + } + } + + private final SimpleMeterRegistry registry = new SimpleMeterRegistry(); + private final FakeClient client = new FakeClient(); + + private RedisRuntimeOwner owner(int contributedLimit, Duration acquireTimeout) { + return owner(contributedLimit, acquireTimeout, new RedisLaneMetrics(registry)); + } + + private RedisRuntimeOwner owner( + int contributedLimit, Duration acquireTimeout, RedisLaneMetrics metrics) { + RedisRuntimeOwner owner = + new RedisRuntimeOwner( + client, laneLimits(contributedLimit), acquireTimeout, Duration.ofSeconds(1), metrics); + metrics.bind(owner); + return owner; + } + + private static Map laneLimits(int contributedLimit) { + Map limits = new LinkedHashMap<>(); + limits.put(RedisConnectionKind.REGULAR, 4); + limits.put(RedisConnectionKind.SCRIPT, 4); + limits.put(ContributedLane.EXCLUSIVE, contributedLimit); + return limits; + } + + private double refused(RedisLane lane, String reason) { + Counter counter = + registry + .find(RedisLaneMetrics.ACQUIRE_REFUSED) + .tag("lane", lane.name()) + .tag("reason", reason) + .counter(); + return counter == null ? 0 : counter.count(); + } + + private Timer waited(RedisLane lane, String outcome) { + return registry + .find(RedisLaneMetrics.ACQUIRE_WAIT) + .tag("lane", lane.name()) + .tag("outcome", outcome) + .timer(); + } + + private double counted(String name, RedisLane lane) { + Counter counter = registry.find(name).tag("lane", lane.name()).counter(); + return counter == null ? 0 : counter.count(); + } + + private double discarded(RedisLane lane, String reason) { + Counter counter = + registry + .find(RedisLaneMetrics.CONNECTIONS_DISCARDED) + .tag("lane", lane.name()) + .tag("reason", reason) + .counter(); + return counter == null ? 0 : counter.count(); + } + + private double gauge(String name, RedisLane lane) { + Gauge gauge = registry.find(name).tag("lane", lane.name()).gauge(); + assertThat(gauge).as("%s{lane=%s} is registered", name, lane.name()).isNotNull(); + return gauge.value(); + } + + @Test + @DisplayName("상한에 찬 lane의 거절은 그 lane의 saturated 거절과 refused 대기로 세고 다른 lane 수치는 건드리지 않는다") + void aSaturatedLaneCountsItsOwnRefusalAndWait() { + // given + try (RedisRuntimeOwner owner = owner(1, Duration.ofMillis(30))) { + RedisLease held = owner.borrow(ContributedLane.EXCLUSIVE); + + // when + Throwable refused = catchThrowable(() -> owner.borrow(ContributedLane.EXCLUSIVE)); + + // then + assertThat(refused).isInstanceOf(RedisLaneUnavailableException.class); + assertThat(refused(ContributedLane.EXCLUSIVE, "saturated")).isEqualTo(1); + assertThat(refused(RedisConnectionKind.SCRIPT, "saturated")) + .as("a full cache lane is not a full SCRIPT lane") + .isZero(); + Timer wait = waited(ContributedLane.EXCLUSIVE, "refused"); + assertThat(wait).isNotNull(); + assertThat(wait.count()).isEqualTo(1); + assertThat(wait.totalTime(TimeUnit.MILLISECONDS)) + .as("the refused borrow waited out the acquire timeout") + .isGreaterThanOrEqualTo(25); + held.close(); + } + } + + @Test + @DisplayName("lane에서 연결을 받은 borrow는 admitted 대기로 센다") + void anAdmittedBorrowRecordsItsWait() { + // given + try (RedisRuntimeOwner owner = owner(2, Duration.ofMillis(50))) { + + // when + owner.borrow(ContributedLane.EXCLUSIVE).close(); + + // then + Timer wait = waited(ContributedLane.EXCLUSIVE, "admitted"); + assertThat(wait).isNotNull(); + assertThat(wait.count()).isEqualTo(1); + } + } + + @Test + @DisplayName("연결을 열지 못한 borrow는 그 lane의 connect_failed 거절로 센다") + void aConnectionThatCouldNotBeOpenedCountsAsConnectFailed() { + // given + client.failOpen = true; + try (RedisRuntimeOwner owner = owner(1, Duration.ofMillis(50))) { + + // when + Throwable refused = catchThrowable(() -> owner.borrow(ContributedLane.EXCLUSIVE)); + + // then + assertThat(refused).isInstanceOf(RedisLaneUnavailableException.class); + assertThat(refused(ContributedLane.EXCLUSIVE, "connect_failed")).isEqualTo(1); + assertThat(counted(RedisLaneMetrics.CONNECTIONS_OPENED, ContributedLane.EXCLUSIVE)).isZero(); + } + } + + @Test + @DisplayName("닫힌 runtime에 온 borrow는 그 lane의 closing 거절로 센다") + void aBorrowAfterShutdownCountsAsClosing() { + // given + RedisRuntimeOwner owner = owner(1, Duration.ofMillis(50)); + owner.close(); + + // when + Throwable refused = catchThrowable(() -> owner.borrow(RedisConnectionKind.REGULAR)); + + // then + assertThat(refused).isInstanceOf(RedisLaneUnavailableException.class); + assertThat(refused(RedisConnectionKind.REGULAR, "closing")).isEqualTo(1); + } + + @Test + @DisplayName("무효화된 연결은 invalidated로 버린 연결로 세고 다음 borrow가 다시 연 연결도 센다") + void anInvalidatedConnectionCountsAsDiscardedAndItsReplacementAsOpened() { + // given + try (RedisRuntimeOwner owner = owner(2, Duration.ofMillis(50))) { + RedisLease lease = owner.borrow(ContributedLane.EXCLUSIVE); + lease.invalidate(); + lease.close(); + + // when + owner.borrow(ContributedLane.EXCLUSIVE).close(); + + // then + assertThat(discarded(ContributedLane.EXCLUSIVE, "invalidated")).isEqualTo(1); + assertThat(counted(RedisLaneMetrics.CONNECTIONS_OPENED, ContributedLane.EXCLUSIVE)) + .as("the first connection and the reconnect after it was thrown away") + .isEqualTo(2); + } + } + + @Test + @DisplayName("idle 중에 끊긴 연결은 broken으로 버린 연결로 센다") + void aConnectionThatDiedWhileIdleCountsAsBroken() { + // given + try (RedisRuntimeOwner owner = owner(2, Duration.ofMillis(50))) { + owner.borrow(RedisConnectionKind.REGULAR).close(); + client.killAll(); + + // when + owner.borrow(RedisConnectionKind.REGULAR).close(); + + // then + assertThat(discarded(RedisConnectionKind.REGULAR, "broken")).isEqualTo(1); + assertThat(counted(RedisLaneMetrics.CONNECTIONS_OPENED, RedisConnectionKind.REGULAR)) + .isEqualTo(2); + } + } + + @Test + @DisplayName("lane마다 빌려 간 연결 수와 연결을 기다리는 borrow 수를 gauge로 낸다") + void eachLaneReportsActiveAndPendingBorrows() throws Exception { + // given + try (RedisRuntimeOwner owner = owner(1, Duration.ofSeconds(5))) { + RedisLease held = owner.borrow(ContributedLane.EXCLUSIVE); + Thread waiter = + Thread.ofVirtual().start(() -> owner.borrow(ContributedLane.EXCLUSIVE).close()); + long deadline = System.nanoTime() + Duration.ofSeconds(5).toNanos(); + while (owner.waiting(ContributedLane.EXCLUSIVE) == 0 && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + + // when + double active = gauge(RedisLaneMetrics.CONNECTIONS_ACTIVE, ContributedLane.EXCLUSIVE); + double pending = gauge(RedisLaneMetrics.CONNECTIONS_PENDING, ContributedLane.EXCLUSIVE); + + // then + assertThat(active).isEqualTo(1); + assertThat(pending).isEqualTo(1); + assertThat(gauge(RedisLaneMetrics.CONNECTIONS_ACTIVE, RedisConnectionKind.SCRIPT)).isZero(); + held.close(); + waiter.join(Duration.ofSeconds(5)); + } + } + + @Test + @DisplayName("거절이 한 번도 없던 lane도 0인 거절 series를 처음부터 낸다") + void everyLaneStartsWithZeroRefusalSeries() { + // given + try (RedisRuntimeOwner owner = owner(1, Duration.ofMillis(50))) { + + // when + Counter saturated = + registry + .find(RedisLaneMetrics.ACQUIRE_REFUSED) + .tag("lane", RedisConnectionKind.SCRIPT.name()) + .tag("reason", "saturated") + .counter(); + + // then + assertThat(saturated) + .as("an alert on a rate needs the series before the first refusal, not after it") + .isNotNull(); + assertThat(saturated.count()).isZero(); + } + } + + @Test + @DisplayName("지표 기록이 실패해도 borrow는 그대로 연결을 빌려 주고 돌려받는다") + void aFailingListenerNeverFailsABorrow() { + // given + RedisLaneListener failing = + new RedisLaneListener() { + @Override + public void waited(RedisLane lane, Duration waited, boolean admitted) { + throw new IllegalStateException("meter registry rejected the sample"); + } + + @Override + public void opened(RedisLane lane) { + throw new IllegalStateException("meter registry rejected the sample"); + } + + @Override + public void discarded(RedisLane lane, Discard reason) { + throw new IllegalStateException("meter registry rejected the sample"); + } + }; + try (RedisRuntimeOwner owner = + new RedisRuntimeOwner( + client, laneLimits(1), Duration.ofMillis(50), Duration.ofSeconds(1), failing)) { + + // when + Throwable failure = + catchThrowable( + () -> { + RedisLease lease = owner.borrow(ContributedLane.EXCLUSIVE); + lease.invalidate(); + lease.close(); + }); + + // then + assertThat(failure).isNull(); + assertThat(owner.outstanding(ContributedLane.EXCLUSIVE)).isZero(); + } + } + + @Test + @DisplayName("listener 없이 만든 owner는 아무 지표도 내지 않는다") + void anOwnerWithoutAListenerRecordsNothing() { + // given + try (RedisRuntimeOwner owner = + new RedisRuntimeOwner( + client, + laneLimits(1), + Duration.ofMillis(50), + Duration.ofSeconds(1), + RedisLaneListener.NONE)) { + + // when + owner.borrow(RedisConnectionKind.REGULAR).close(); + + // then + assertThat(registry.getMeters()).isEmpty(); + } + } + + /** A driver stand-in: opens connections that can be made to fail or die. */ + private static final class FakeClient implements RedisRuntimeClient { + + private final List live = new ArrayList<>(); + private boolean failOpen; + + void killAll() { + live.forEach(connection -> connection.alive = false); + } + + @Override + public RedisDeploymentMode mode() { + return RedisDeploymentMode.STANDALONE; + } + + @Override + public RedisLaneConnection openLane(RedisLane lane, Optional routingKey) { + if (failOpen) { + throw new IllegalStateException("connection refused"); + } + FakeConnection connection = new FakeConnection(); + live.add(connection); + return connection; + } + + @Override + public void close() {} + } + + private static final class FakeConnection implements RedisRuntimeClient.RedisLaneConnection { + + private boolean alive = true; + + @Override + public SemanticRedisCommands commands() { + return null; + } + + @Override + public RedisDriverConnection driver() { + return null; + } + + @Override + public boolean open() { + return alive; + } + + @Override + public void close() { + alive = false; + } + } +} diff --git a/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisRuntimeAutoConfigurationTest.java b/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisRuntimeAutoConfigurationTest.java index b7d2b57e..2284aed8 100644 --- a/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisRuntimeAutoConfigurationTest.java +++ b/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisRuntimeAutoConfigurationTest.java @@ -7,6 +7,8 @@ import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisAccou import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLane; import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisRuntimeClient; import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisRuntimeOwner; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.boot.autoconfigure.AutoConfigurations; @@ -404,6 +406,29 @@ class RedisRuntimeAutoConfigurationTest { }); } + @Test + @DisplayName("MeterRegistry가 있으면 runtime lane과 다른 모듈이 기여한 lane마다 연결 지표를 등록한다") + void aMeterRegistryGetsTheMetersOfEveryComposedLane() { + authenticated + .withPropertyValues("app.redis.enabled=true") + .withBean(MeterRegistry.class, SimpleMeterRegistry::new) + .withBean( + RedisLaneContribution.class, + () -> new RedisLaneContribution(ContributedLane.EXCLUSIVE, 3)) + .run( + context -> { + // when + MeterRegistry registry = context.getBean(MeterRegistry.class); + + // then + assertThat(context).hasNotFailed(); + assertThat( + registry.find(RedisLaneMetrics.CONNECTIONS_ACTIVE).gauges().stream() + .map(gauge -> gauge.getId().getTag("lane"))) + .containsExactlyInAnyOrder("REGULAR", "SCRIPT", "EXCLUSIVE"); + }); + } + @Test void aLaneComposedTwiceFailsTheContext() { authenticated diff --git a/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisRuntimeHealthStatusOrderTest.java b/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisRuntimeHealthStatusOrderTest.java new file mode 100644 index 00000000..cb9520f6 --- /dev/null +++ b/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisRuntimeHealthStatusOrderTest.java @@ -0,0 +1,51 @@ +package dev.caskeleton.adapter.outbound.cache.redis.runtime.config; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.boot.env.YamlPropertySourceLoader; +import org.springframework.boot.health.actuate.endpoint.SimpleStatusAggregator; +import org.springframework.boot.health.contributor.Status; +import org.springframework.core.env.PropertySource; +import org.springframework.core.io.ClassPathResource; + +/** + * The leaf that reports {@code DEGRADED} ships the order that ranks it (review 2026-11-03 + * RR-04-10). + * + *

    {@code redisOptional} is created by this leaf whenever {@code app.redis.enabled=true}. The + * order that puts {@code DEGRADED} after {@code DOWN} used to ship with cache-redis, so an assembly + * that used Redis only through the typed SDK (redis-sdk pulls this leaf in, not cache-redis) had a + * status Boot's default order does not know — and an unknown status sorts before {@code DOWN}, + * which reported a real outage as HTTP 200 {@code DEGRADED} (review 05-23e). + */ +class RedisRuntimeHealthStatusOrderTest { + + private static final String MODULE_DEFAULTS = "config/module-defaults/redis-runtime.yml"; + + @Test + @DisplayName("DEGRADED를 내는 redis-runtime이 DOWN 뒤·UP 앞 순서를 직접 싣는다") + void theLeafThatReportsDegradedShipsItsOrder() throws IOException { + // given + List> shipped = + new YamlPropertySourceLoader() + .load(MODULE_DEFAULTS, new ClassPathResource(MODULE_DEFAULTS)); + String order = + String.valueOf(shipped.get(0).getProperty("management.endpoint.health.status.order")); + SimpleStatusAggregator aggregator = + new SimpleStatusAggregator(Arrays.stream(order.split(",")).map(String::trim).toList()); + + // when + Status withAnOutage = + aggregator.getAggregateStatus(Status.UP, new Status("DEGRADED"), Status.DOWN); + + // then + assertThat(withAnOutage).isEqualTo(Status.DOWN); + assertThat(aggregator.getAggregateStatus(Status.UP, new Status("DEGRADED")).getCode()) + .isEqualTo("DEGRADED"); + } +} diff --git a/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisScriptLaneAdmissionTest.java b/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisScriptLaneAdmissionTest.java index bbed4f58..4addd172 100644 --- a/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisScriptLaneAdmissionTest.java +++ b/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisScriptLaneAdmissionTest.java @@ -10,6 +10,7 @@ import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLaneU import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLease; import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisRuntimeOwner; import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.StubRedisRuntimeClient; +import io.micrometer.core.instrument.MeterRegistry; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; @@ -89,7 +90,8 @@ class RedisScriptLaneAdmissionTest { .redisRuntimeOwner( StubRedisRuntimeClient.serving(null), settings, - new StaticListableBeanFactory().getBeanProvider(RedisLaneContribution.class)); + new StaticListableBeanFactory().getBeanProvider(RedisLaneContribution.class), + new StaticListableBeanFactory().getBeanProvider(MeterRegistry.class)); } /** One semantic script call: the lease is held until the test lets the script complete. */ diff --git a/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisScriptLaneCapacityTest.java b/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisScriptLaneCapacityTest.java index 6d8c40bf..fb69a31f 100644 --- a/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisScriptLaneCapacityTest.java +++ b/src/adapter/outbound/redis-runtime/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/runtime/config/RedisScriptLaneCapacityTest.java @@ -7,6 +7,7 @@ import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLaneU import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisLease; import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.RedisRuntimeOwner; import dev.caskeleton.adapter.outbound.cache.redis.runtime.connection.StubRedisRuntimeClient; +import io.micrometer.core.instrument.MeterRegistry; import java.time.Duration; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.support.StaticListableBeanFactory; @@ -25,7 +26,8 @@ class RedisScriptLaneCapacityTest { .redisRuntimeOwner( StubRedisRuntimeClient.serving(null), settings, - new StaticListableBeanFactory().getBeanProvider(RedisLaneContribution.class)); + new StaticListableBeanFactory().getBeanProvider(RedisLaneContribution.class), + new StaticListableBeanFactory().getBeanProvider(MeterRegistry.class)); RedisLease first = owner.borrow(RedisConnectionKind.SCRIPT); RedisLease second = owner.borrow(RedisConnectionKind.SCRIPT); diff --git a/src/adapter/outbound/redis-sdk/gradle.lockfile b/src/adapter/outbound/redis-sdk/gradle.lockfile index 8c5ab46a..a4b7fb69 100644 --- a/src/adapter/outbound/redis-sdk/gradle.lockfile +++ b/src/adapter/outbound/redis-sdk/gradle.lockfile @@ -42,6 +42,7 @@ io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnota io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor io.lettuce:lettuce-core:6.8.2.RELEASE=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath io.micrometer:micrometer-commons:1.16.7=runtimeClasspath,testCompileClasspath,testRuntimeClasspath +io.micrometer:micrometer-core:1.16.7=runtimeClasspath,testRuntimeClasspath io.micrometer:micrometer-observation:1.16.7=runtimeClasspath,testCompileClasspath,testRuntimeClasspath io.netty:netty-buffer:4.2.17.Final=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath io.netty:netty-codec-base:4.2.17.Final=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath @@ -90,6 +91,7 @@ org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle org.codehaus.plexus:plexus-utils:3.3.0=checkstyle org.dom4j:dom4j:2.2.0=spotbugs org.hamcrest:hamcrest:3.0=testCompileClasspath,testRuntimeClasspath +org.hdrhistogram:HdrHistogram:2.2.2=runtimeClasspath,testRuntimeClasspath org.javassist:javassist:3.28.0-GA=checkstyle org.jspecify:jspecify:1.0.1=annotationProcessor,checkstyle,compileClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath org.junit.jupiter:junit-jupiter-api:6.0.3=testCompileClasspath,testRuntimeClasspath @@ -101,6 +103,7 @@ org.junit.platform:junit-platform-engine:6.0.3=testRuntimeClasspath org.junit.platform:junit-platform-launcher:6.0.3=testRuntimeClasspath org.junit:junit-bom:6.0.3=testCompileClasspath,testRuntimeClasspath org.junit:junit-bom:6.1.0=spotbugs +org.latencyutils:LatencyUtils:2.0.3=runtimeClasspath,testRuntimeClasspath org.mockito:mockito-core:5.20.0=mockitoAgent,testCompileClasspath,testRuntimeClasspath org.mockito:mockito-junit-jupiter:5.20.0=testCompileClasspath,testRuntimeClasspath org.objenesis:objenesis:3.3=testRuntimeClasspath diff --git a/src/adapter/outbound/support/src/main/java/dev/caskeleton/adapter/outbound/support/startup/StartupRule.java b/src/adapter/outbound/support/src/main/java/dev/caskeleton/adapter/outbound/support/startup/StartupRule.java index b19930be..668ef1bd 100644 --- a/src/adapter/outbound/support/src/main/java/dev/caskeleton/adapter/outbound/support/startup/StartupRule.java +++ b/src/adapter/outbound/support/src/main/java/dev/caskeleton/adapter/outbound/support/startup/StartupRule.java @@ -21,6 +21,11 @@ import org.springframework.core.env.Environment; * dependencies, read only when the rule runs (the secret guard reads each module's {@code * RequiredProductionSecrets} this way). A check that needs a live connection — the datasource * version check — is not a rule; it stays a bean and is ordered before what it protects. + * + *

    The runner enforces this: when creating or running a rule brings any other singleton into + * existence — a configuration class behind a non-static {@code @Bean} method, a settings bean the + * method takes — the boot fails with that bean's name, because a bean created before the + * post-processors would keep unbound settings for the whole run (RR-06-01). */ public interface StartupRule { diff --git a/src/adapter/outbound/support/src/main/java/dev/caskeleton/adapter/outbound/support/startup/StartupRuleRunner.java b/src/adapter/outbound/support/src/main/java/dev/caskeleton/adapter/outbound/support/startup/StartupRuleRunner.java index 55934d0e..51ebba76 100644 --- a/src/adapter/outbound/support/src/main/java/dev/caskeleton/adapter/outbound/support/startup/StartupRuleRunner.java +++ b/src/adapter/outbound/support/src/main/java/dev/caskeleton/adapter/outbound/support/startup/StartupRuleRunner.java @@ -1,8 +1,11 @@ package dev.caskeleton.adapter.outbound.support.startup; import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; +import java.util.Set; import org.springframework.beans.factory.config.BeanFactoryPostProcessor; import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; import org.springframework.core.Ordered; @@ -33,6 +36,17 @@ import org.springframework.core.env.Environment; * static} {@code @Bean} method: it then pulls neither another bean nor its configuration class * forward. Rules are found without eager initialization, so no {@code FactoryBean} — an entity * manager factory, say — is created to learn its type. + * + *

    What a rule may not pull forward

    + * + *

    A bean created this early is created without the post-processors: a JavaBean + * {@code @ConfigurationProperties} class it takes is never bound and keeps its defaults for the + * whole run, and so does every other consumer of that singleton (RR-06-01). The {@code static} + * declaration used to be a sentence in Javadoc; now the runner checks it. Any singleton that comes + * into existence while a rule is created or run fails the boot with its name — unless it is itself + * a {@link StartupRule} or a {@link RequiredProductionSecrets}, the two kinds that are declared + * {@code static} and argument-free for exactly this reason (a non-static one creates its + * configuration class, which is reported). */ public final class StartupRuleRunner implements BeanFactoryPostProcessor, Ordered { @@ -49,8 +63,22 @@ public final class StartupRuleRunner implements BeanFactoryPostProcessor, Ordere @Override public void postProcessBeanFactory(ConfigurableListableBeanFactory beanFactory) { - for (StartupRule rule : rules(beanFactory)) { - enforce(rule, environment); + IdentityHashMap names = new IdentityHashMap<>(); + List rules = new ArrayList<>(); + for (String name : beanFactory.getBeanNamesForType(StartupRule.class, true, false)) { + Set before = singletons(beanFactory); + StartupRule rule = beanFactory.getBean(name, StartupRule.class); + refusePulledForward(beanFactory, name, "creating it", before); + rules.add(rule); + names.put(rule, name); + } + AnnotationAwareOrderComparator.sort(rules); + for (StartupRule rule : rules) { + String name = names.get(rule); + Set before = singletons(beanFactory); + List violations = rule.violations(environment); + refusePulledForward(beanFactory, name, "running it", before); + raise(rule, violations); } } @@ -60,13 +88,46 @@ public final class StartupRuleRunner implements BeanFactoryPostProcessor, Ordere return Ordered.HIGHEST_PRECEDENCE; } - private static List rules(ConfigurableListableBeanFactory beanFactory) { - List rules = new ArrayList<>(); - for (String name : beanFactory.getBeanNamesForType(StartupRule.class, true, false)) { - rules.add(beanFactory.getBean(name, StartupRule.class)); + private static Set singletons(ConfigurableListableBeanFactory beanFactory) { + return Set.of(beanFactory.getSingletonNames()); + } + + /** + * Fails when creating or running the rule {@code ruleName} brought a singleton into existence + * other than a rule or a {@link RequiredProductionSecrets}: that bean was built before the bean + * post-processors, so its settings are unbound and it is not proxied, for the rest of the run. + */ + private static void refusePulledForward( + ConfigurableListableBeanFactory beanFactory, + String ruleName, + String stage, + Set before) { + Set pulled = new LinkedHashSet<>(); + for (String name : beanFactory.getSingletonNames()) { + if (!before.contains(name) + && !beanFactory.isTypeMatch(name, StartupRule.class) + && !beanFactory.isTypeMatch(name, RequiredProductionSecrets.class)) { + pulled.add(name); + } } - AnnotationAwareOrderComparator.sort(rules); - return rules; + if (pulled.isEmpty()) { + return; + } + // A configuration error of the composition, so the configuration exit code (78): no + // environment fixes it, but neither does a retry, and it must not look like a crash. + throw StartupFailures.envValidation( + "startup rule '" + + ruleName + + "' created " + + pulled + + " while " + + stage + + ". Rules run before any bean post-processor exists, so a bean created then is never" + + " bound to its @ConfigurationProperties and never proxied, for the whole run. Declare" + + " the rule with a static @Bean method that takes only the Environment, the bean" + + " factory, or an ObjectProvider of RequiredProductionSecrets beans that are" + + " themselves static @Bean methods without parameters. A check that needs another" + + " bean is not a rule: keep it a bean, ordered before what it protects."); } /** @@ -77,8 +138,12 @@ public final class StartupRuleRunner implements BeanFactoryPostProcessor, Ordere * @throws StartupFailureException when the rule found a violation */ public static void enforce(StartupRule rule, Environment environment) { + raise(rule, rule.violations(environment)); + } + + private static void raise(StartupRule rule, List violations) { StartupValidationContext.forPhase(rule.phase()) - .rejectAll(rule.violations(environment)) + .rejectAll(violations) .failIfViolated(rule.summary(), rule.remedy()); } } diff --git a/src/adapter/outbound/support/src/test/java/dev/caskeleton/adapter/outbound/support/messaging/JsonIntegrationEventEncoderTest.java b/src/adapter/outbound/support/src/test/java/dev/caskeleton/adapter/outbound/support/messaging/JsonIntegrationEventEncoderTest.java index 7c193bb6..ca5d5b9e 100644 --- a/src/adapter/outbound/support/src/test/java/dev/caskeleton/adapter/outbound/support/messaging/JsonIntegrationEventEncoderTest.java +++ b/src/adapter/outbound/support/src/test/java/dev/caskeleton/adapter/outbound/support/messaging/JsonIntegrationEventEncoderTest.java @@ -50,7 +50,7 @@ class JsonIntegrationEventEncoderTest { // then assertThat(new String(event.envelopeBytes(), StandardCharsets.UTF_8)) .isEqualTo( - "{\"envelopeVersion\":1,\"eventId\":\"event-1\",\"contractId\":\"testkit.encoder-sample\"," + "{\"envelopeVersion\":1,\"eventId\":\"01994e11-4d88-7000-8000-0000000000e1\",\"contractId\":\"testkit.encoder-sample\"," + "\"payloadVersion\":1,\"logicalDestination\":\"sample-events\"," + "\"aggregate\":{\"type\":\"sample\",\"id\":\"sample-1\",\"sequence\":3,\"eventIndex\":0}," + "\"occurredAt\":\"2026-09-28T01:02:03Z\",\"correlationId\":\"correlation-1\"," @@ -267,7 +267,7 @@ class JsonIntegrationEventEncoderTest { private static IntegrationEventDraft draft( SamplePayload payload, String destination) { return new IntegrationEventDraft<>( - new EventId("event-1"), + new EventId("01994e11-4d88-7000-8000-0000000000e1"), new ContractId("testkit.encoder-sample"), 1, new LogicalDestinationId(destination), diff --git a/src/adapter/outbound/support/src/test/java/dev/caskeleton/adapter/outbound/support/startup/StartupRuleRunnerTest.java b/src/adapter/outbound/support/src/test/java/dev/caskeleton/adapter/outbound/support/startup/StartupRuleRunnerTest.java index c35b885f..398c108d 100644 --- a/src/adapter/outbound/support/src/test/java/dev/caskeleton/adapter/outbound/support/startup/StartupRuleRunnerTest.java +++ b/src/adapter/outbound/support/src/test/java/dev/caskeleton/adapter/outbound/support/startup/StartupRuleRunnerTest.java @@ -7,8 +7,13 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.core.env.Environment; class StartupRuleRunnerTest { @@ -103,6 +108,61 @@ class StartupRuleRunnerTest { assertThat(seen).containsExactly("resolved"); } + @Test + @DisplayName("설정 bean을 받는 non-static 규칙은 바인딩 안 된 설정으로 조용히 돌지 않고, 앞당긴 bean 이름을 대며 기동을 실패시킨다") + void aRuleThatPullsASettingsBeanForwardFailsTheBootInsteadOfReadingUnboundSettings() { + // given + ProbeRuleConfig.SEEN.clear(); + + // when + runner + .withPropertyValues("probe.name=bound") + .withUserConfiguration(ProbeRuleConfig.class) + .run( + context -> { + // then + assertThat(ProbeRuleConfig.SEEN).doesNotContainNull(); + assertThat(context.getStartupFailure()) + .isInstanceOf(StartupFailureException.class) + .hasMessageContaining("probeRule") + .hasMessageContaining("probe-dev.caskeleton") + .hasMessageContaining("static"); + }); + } + + @Test + @DisplayName("규칙이 실행 중에 꺼내는 RequiredProductionSecrets가 다른 bean을 앞당기면 기동이 실패한다") + void aProductionSecretsBeanThatPullsItsConfigurationForwardFailsTheBoot() { + // when + runner + .withUserConfiguration(SecretReadingRuleConfig.class, NonStaticSecretsConfig.class) + .run( + context -> { + // then + assertThat(context.getStartupFailure()) + .isInstanceOf(StartupFailureException.class) + .hasMessageContaining("secretReadingRule") + .hasMessageContaining("NonStaticSecretsConfig"); + }); + } + + @Test + @DisplayName("static이고 의존이 없는 RequiredProductionSecrets를 규칙이 실행 중에 읽는 것은 허용된다") + void aRuleMayReadStaticProductionSecretsWhenItRuns() { + // given + SecretReadingRuleConfig.SEEN.clear(); + + // when + runner + .withUserConfiguration(SecretReadingRuleConfig.class, StaticSecretsConfig.class) + .run( + context -> { + // then + assertThat(context).hasNotFailed(); + assertThat(SecretReadingRuleConfig.SEEN).containsExactly("APP_PROBE_SECRET"); + }); + } + private static StartupRule rule(StartupPhase phase, String summary, List violations) { return new StartupRule() { @Override @@ -121,4 +181,87 @@ class StartupRuleRunnerTest { } }; } + + /** The shape RR-06-01 found: a JavaBean settings class, and a rule that takes it. */ + @ConfigurationProperties("probe") + public static class ProbeSettings { + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + + @Configuration(proxyBeanMethods = false) + @EnableConfigurationProperties(ProbeSettings.class) + static class ProbeRuleConfig { + static final List SEEN = new ArrayList<>(); + + @Bean + StartupRule probeRule(ProbeSettings settings) { + return new StartupRule() { + @Override + public StartupPhase phase() { + return StartupPhase.ENV_VALIDATION; + } + + @Override + public String summary() { + return "probe"; + } + + @Override + public List violations(Environment environment) { + SEEN.add(settings.getName()); + return List.of(); + } + }; + } + } + + @Configuration(proxyBeanMethods = false) + static class SecretReadingRuleConfig { + static final List SEEN = new ArrayList<>(); + + @Bean + static StartupRule secretReadingRule(ObjectProvider secrets) { + return new StartupRule() { + @Override + public StartupPhase phase() { + return StartupPhase.ENV_VALIDATION; + } + + @Override + public String summary() { + return "secrets"; + } + + @Override + public List violations(Environment environment) { + secrets.orderedStream().forEach(each -> SEEN.addAll(each.requiredSecrets(environment))); + return List.of(); + } + }; + } + } + + @Configuration(proxyBeanMethods = false) + static class StaticSecretsConfig { + @Bean + static RequiredProductionSecrets probeSecrets() { + return environment -> List.of("APP_PROBE_SECRET"); + } + } + + @Configuration(proxyBeanMethods = false) + static class NonStaticSecretsConfig { + @Bean + RequiredProductionSecrets probeSecrets() { + return environment -> List.of("APP_PROBE_SECRET"); + } + } } diff --git a/src/app-bootstrap/CLAUDE.md b/src/app-bootstrap/CLAUDE.md index 98ba21c7..4d816bcb 100644 --- a/src/app-bootstrap/CLAUDE.md +++ b/src/app-bootstrap/CLAUDE.md @@ -87,7 +87,9 @@ configurations, Redis as one 376-line class. this leaf. `runtime/startup/` keeps only `StartupFailureExceptionReporter`. - **An environment-only guard is a `StartupRule` bean** (`support.startup`): it returns violations, and the one `StartupRuleRunner` raises them before any other singleton exists (declare it with a - parameterless `static` `@Bean` method). A guard that needs another bean stays a bean. A + `static` `@Bean` method that takes only the `Environment`, the bean factory or an `ObjectProvider` + of static `RequiredProductionSecrets`; the runner fails the boot naming any other bean a rule + creates). A guard that needs another bean stays a bean. A capability's production secrets are the capability's: its module contributes a `RequiredProductionSecrets` bean, and `SecretSourceValidator` checks the names without knowing which roles exist. diff --git a/src/app-bootstrap/README.md b/src/app-bootstrap/README.md index 56902f31..3f6fc3ab 100644 --- a/src/app-bootstrap/README.md +++ b/src/app-bootstrap/README.md @@ -129,7 +129,11 @@ README의 "startup" 절에 있다. ### SecretSourceValidator - **prod 에서 깨지면 부팅 자체를 거부하는 fail-fast 가드다.** 두 가지 prod 불변식을 context refresh 완료 전에 강제한다. (1) `__LOCAL_DEV_` 접두사를 가진 dev/local 가짜 자격증명 값이 prod 프로파일에 - 도달하는 것은 잘못 로드된 사고이므로 부팅을 중단시킨다. (2) 레지스트리에서 `secret` + + 도달하는 것은 잘못 로드된 사고이므로 부팅을 중단시킨다. `fixture-` 접두사 값도 같다. 이 저장소가 + lane용으로 적어 둔 공개 비밀번호(`infra/redis-sdk/acl`의 Redis ACL fixture, compose lane의 + `REDIS_LANE_*`)라서, lane 환경이나 lane ACL 파일을 베낀 배포가 그대로 운영에 뜨는 것을 막는다 + (review 2026-11-03 RR-07-04). 비밀이 아닌 설정값이라도 prod에서 `fixture-`로 시작하면 거부되니 이름을 + 바꾼다. 실패 메시지는 변수 이름만 적고 값은 적지 않는다. (2) 레지스트리에서 `secret` + `prod_default: null`로 분류된 모든 키는 prod 에서 비어 있지 않게 주입돼야 한다 — 빈 시크릿으로 부팅하는 것은 금지다. dev/local 프로파일은 두 검사를 건너뛴다(`__LOCAL_DEV_*`가 의도된 로컬 폴백이므로). diff --git a/src/app-bootstrap/build.gradle b/src/app-bootstrap/build.gradle index 8b312763..8d093f16 100644 --- a/src/app-bootstrap/build.gradle +++ b/src/app-bootstrap/build.gradle @@ -122,10 +122,11 @@ sourceSets { } // Every capability a service can select — the commented runtimeOnly lines below — listed once. -// Two consumers read it: the whole-repository ArchUnit import (spec §7: the architecture suite sees +// Three consumers read it — the environment contract's metadata classpath (verifyEnvKeys, below) +// and these two: the whole-repository ArchUnit import (spec §7: the architecture suite sees // every module of this build and the sample, the shipped runtime only the selected ones; the // optional-platforms build is held by its own Gradle direction gate instead) and the qualification artifact the -// Compose lanes boot (qualificationBootJar, below). Neither reaches `bootJar`, `test` or +// Compose lanes boot (qualificationBootJar, below). None of them reaches `bootJar`, `test` or // `systemTest`, because a capability on a classpath is switched on there by its mere presence. // A new capability leaf is added here and as a commented line below; nowhere else. def capabilityProjects = [ @@ -479,6 +480,12 @@ def qualificationClasspath = compositionRuntimeClasspath('qualificationRuntimeCl dependencies { capabilityProjects.each { qualificationRuntimeClasspath project(it) } } +// The environment contract reads every selectable capability's configuration metadata from this +// classpath, so a module that ships typed settings is checked because it can be selected, and the +// task names no leaf (review 2026-11-03 RR-06-10). +tasks.named('verifyEnvKeys') { + metadataClasspath.from(qualificationClasspath) +} def qualificationBootJar = tasks.register('qualificationBootJar', org.springframework.boot.gradle.tasks.bundling.BootJar) { group = 'build' description = 'Assembles the Compose qualification jar: the default composition plus every capability.' diff --git a/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ApplicationBoundaryArchitectureTest.java b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ApplicationBoundaryArchitectureTest.java index 7017eacc..d64f500f 100644 --- a/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ApplicationBoundaryArchitectureTest.java +++ b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ApplicationBoundaryArchitectureTest.java @@ -1,5 +1,7 @@ package dev.caskeleton.bootstrap.architecture; +import static com.tngtech.archunit.core.domain.AccessTarget.Predicates.declaredIn; +import static com.tngtech.archunit.core.domain.JavaCall.Predicates.target; import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.classes; import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.methods; import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.noClasses; @@ -20,6 +22,8 @@ import com.tngtech.archunit.lang.ArchRule; import com.tngtech.archunit.lang.ConditionEvents; import com.tngtech.archunit.lang.SimpleConditionEvent; import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.messaging.event.IntegrationEventDraft; +import dev.caskeleton.application.messaging.event.IntegrationEventDrafts; import dev.caskeleton.application.security.RequiresPermission; import dev.caskeleton.application.security.SystemInvoked; import dev.caskeleton.application.usecase.CommandUseCase; @@ -442,6 +446,44 @@ class ApplicationBoundaryArchitectureTest { + "Authentication, so the gate would be decoration") .allowEmptyShould(true); + /** + * A {@link SystemInvoked} use case is a write with no permission check, so no transport may reach + * it. A bean of one fails startup ({@code PermissionEnforcementGuard}); this closes the other + * path, an inbound adapter that constructs or calls one itself (review 2026-11-03 RR-01-02). + */ + @ArchTest + static final ArchRule INBOUND_ADAPTERS_DO_NOT_REACH_SYSTEM_INVOKED_USE_CASES = + noClasses() + .that() + .resideInAPackage("..adapter.inbound..") + .should() + .dependOnClassesThat() + .areAnnotatedWith(SystemInvoked.class) + .as( + "INBOUND_ADAPTERS_DO_NOT_REACH_SYSTEM_INVOKED_USE_CASES: a @SystemInvoked use case " + + "declares no permission because only the system calls it; an inbound adapter " + + "that reaches it runs a write with no authorization check. Its composition " + + "constructs it and hands it to the scheduler that calls it.") + .allowEmptyShould(true); + + /** + * {@link IntegrationEventDrafts} is where contract id, version, destination and correlation are + * decided for every feature. A feature that calls the ten-field draft constructor itself restates + * those rules, which is the drift review 2026-11-01 01-09 removed (RR-01-03). + */ + @ArchTest + static final ArchRule INTEGRATION_EVENT_DRAFTS_ARE_BUILT_BY_THE_SHARED_HELPER = + noClasses() + .that() + .doNotBelongToAnyOf(IntegrationEventDrafts.class, IntegrationEventDraft.class) + .should() + .callConstructorWhere(target(declaredIn(IntegrationEventDraft.class))) + .as( + "INTEGRATION_EVENT_DRAFTS_ARE_BUILT_BY_THE_SHARED_HELPER: production code builds an " + + "IntegrationEventDraft only through IntegrationEventDrafts.draft(...), which " + + "owns the contract fields, the correlation fallback and the single-tenant scope") + .allowEmptyShould(true); + @ArchTest static final ArchRule APPLICATION_AND_DOMAIN_DO_NOT_DEPEND_ON_SPRING_SECURITY = noClasses() diff --git a/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchRuleDiscoveryContractTest.java b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchRuleDiscoveryContractTest.java index 916d9945..b4da42bc 100644 --- a/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchRuleDiscoveryContractTest.java +++ b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchRuleDiscoveryContractTest.java @@ -114,9 +114,11 @@ class ArchRuleDiscoveryContractTest { "IDENTIFIER_ADAPTER_DOES_NOT_DEPEND_ON_OTHER_ADAPTERS_OR_BOOTSTRAP", "INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES", "INBOUND_ADAPTERS_DO_NOT_DEPEND_ON_OUTBOUND_ADAPTERS", + "INBOUND_ADAPTERS_DO_NOT_REACH_SYSTEM_INVOKED_USE_CASES", "INBOUND_PORT_IMPLEMENTATIONS_DECLARE_A_CAPABILITY", "INBOUND_PORT_IMPLEMENTATIONS_DECLARE_CAPABILITY", "INBOUND_PORT_IMPLEMENTATIONS_END_WITH_USE_CASE", + "INTEGRATION_EVENT_DRAFTS_ARE_BUILT_BY_THE_SHARED_HELPER", "MANAGEMENT_SECURITY_DOES_NOT_DEPEND_ON_HEALTH_INTERNALS", "MAPPERS_DO_NOT_DEPEND_ON_WEB_OR_APPLICATION_BOUNDARIES", "MESSAGING_OUTBOUND_PUBLIC_INSTANCE_METHODS_DO_NOT_LEAK_ADAPTER_TYPES_THROUGH_GENERICS", diff --git a/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchitectureViolationFixtureTest.java b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchitectureViolationFixtureTest.java index 369966ef..2eb2d35f 100644 --- a/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchitectureViolationFixtureTest.java +++ b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchitectureViolationFixtureTest.java @@ -8,16 +8,19 @@ import com.tngtech.archunit.lang.EvaluationResult; import dev.caskeleton.application.architecture.violations.ApplicationDiagnosticFrameworkViolation; import dev.caskeleton.bootstrap.architecture.allowed.application.CleanProjectionQueryPort; import dev.caskeleton.bootstrap.architecture.fixtures.application.RootWriteTransactionBoundaryUseCase; +import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.SystemInvokedCallingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.outbound.RawExternalResponseFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.outbound.messaging.GenericTypeLeakingAdapterFixture; import dev.caskeleton.bootstrap.architecture.violations.application.BulkWriteWithoutWriteAccessUseCase; import dev.caskeleton.bootstrap.architecture.violations.application.FixtureRepository; import dev.caskeleton.bootstrap.architecture.violations.application.GenericLeakQueryPort; +import dev.caskeleton.bootstrap.architecture.violations.application.HandBuiltIntegrationEventDraftFixture; import dev.caskeleton.bootstrap.architecture.violations.application.JakartaValidationApplicationFixture; import dev.caskeleton.bootstrap.architecture.violations.application.MissingTransactionBoundaryUseCase; import dev.caskeleton.bootstrap.architecture.violations.application.OutboundWithoutPermissionUseCase; import dev.caskeleton.bootstrap.architecture.violations.application.RawLeakQueryPort; import dev.caskeleton.bootstrap.architecture.violations.application.ReadOnlyRepositoryWriteUseCase; +import dev.caskeleton.bootstrap.architecture.violations.application.SystemInvokedFixtureUseCase; import dev.caskeleton.bootstrap.architecture.violations.domain.AnnotatedPublicNoArgValueObjectFixture; import dev.caskeleton.bootstrap.architecture.violations.domain.JakartaValidationDomainFixture; import dev.caskeleton.bootstrap.architecture.violations.domain.entity.FakeDomainEntity; @@ -154,6 +157,35 @@ class ArchitectureViolationFixtureTest { new ClassFileImporter() .importPackages("dev.caskeleton.bootstrap.architecture.violations.secrets"); + // Review 2026-11-03 RR-01-02: the controller and the use case it reaches, and nothing else, so + // the rule is proven on the one dependency it forbids. + private static final JavaClasses SYSTEM_INVOKED_FROM_TRANSPORT_FIXTURE_ONLY = + new ClassFileImporter() + .importClasses( + SystemInvokedCallingControllerFixture.class, SystemInvokedFixtureUseCase.class); + + // Review 2026-11-03 RR-01-03: one class that builds the draft by hand. + private static final JavaClasses HAND_BUILT_DRAFT_FIXTURE_ONLY = + new ClassFileImporter().importClasses(HandBuiltIntegrationEventDraftFixture.class); + + @Test + void anInboundAdapterThatCallsASystemInvokedUseCaseIsCaught() { + EvaluationResult result = + ApplicationBoundaryArchitectureTest.INBOUND_ADAPTERS_DO_NOT_REACH_SYSTEM_INVOKED_USE_CASES + .evaluate(SYSTEM_INVOKED_FROM_TRANSPORT_FIXTURE_ONLY); + + assertThat(result.hasViolation()).isTrue(); + } + + @Test + void aFeatureThatBuildsAnIntegrationEventDraftByHandIsCaught() { + EvaluationResult result = + ApplicationBoundaryArchitectureTest.INTEGRATION_EVENT_DRAFTS_ARE_BUILT_BY_THE_SHARED_HELPER + .evaluate(HAND_BUILT_DRAFT_FIXTURE_ONLY); + + assertThat(result.hasViolation()).isTrue(); + } + @Test void noRefreshScopeAnywhereCatchesRefreshScopeAnnotation() { EvaluationResult result = diff --git a/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/JpaProductionArchitectureTest.java b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/JpaProductionArchitectureTest.java index db694f5a..2411cf6e 100644 --- a/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/JpaProductionArchitectureTest.java +++ b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/JpaProductionArchitectureTest.java @@ -6,6 +6,7 @@ import com.tngtech.archunit.core.domain.JavaClasses; import com.tngtech.archunit.core.importer.ClassFileImporter; import dev.caskeleton.adapter.outbound.persistence.testkit.arch.JpaArchitectureRules; import jakarta.persistence.Entity; +import java.util.List; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -101,6 +102,29 @@ class JpaProductionArchitectureTest { JpaArchitectureRules.associationsAreMappedFromOneSideOnly().check(productionClasses); } + @Test + @DisplayName("알림·파일 서버 저장소 leaf의 엔티티는 그 leaf 밖의 운영 코드가 쓰지 않는다") + void capabilityPersistenceEntitiesAreUsedOnlyInsideTheirLeaf() { + // given: both leaves' entities are in the import, so the rule cannot pass by finding nothing + List leaves = + List.of( + "dev.caskeleton.adapter.outbound.persistence.notification", + "dev.caskeleton.adapter.outbound.persistence.fileserver"); + assertThat(leaves) + .allSatisfy( + leaf -> + assertThat(productionClasses.stream()) + .anyMatch( + type -> + type.getPackageName().startsWith(leaf) + && type.isAnnotatedWith(Entity.class))); + + // then + leaves.forEach( + leaf -> + JpaArchitectureRules.entitiesAreUsedOnlyInsideTheirLeaf(leaf).check(productionClasses)); + } + @Test @DisplayName("no entity uses two audit mechanisms at once") void noEntityUsesTwoAuditMechanismsAtOnce() { diff --git a/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/SystemInvokedCallingControllerFixture.java b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/SystemInvokedCallingControllerFixture.java new file mode 100644 index 00000000..dfe5f538 --- /dev/null +++ b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/SystemInvokedCallingControllerFixture.java @@ -0,0 +1,17 @@ +package dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller; + +import dev.caskeleton.bootstrap.architecture.violations.application.SystemInvokedFixtureUseCase; + +/** + * Negative fixture for {@code INBOUND_ADAPTERS_DO_NOT_REACH_SYSTEM_INVOKED_USE_CASES}: an inbound + * adapter that calls a {@code @SystemInvoked} use case, a write with no permission gate, straight + * from a transport (review 2026-11-03 RR-01-02). + */ +public class SystemInvokedCallingControllerFixture { + + private final SystemInvokedFixtureUseCase useCase = new SystemInvokedFixtureUseCase(); + + public void trigger() { + useCase.handle(); + } +} diff --git a/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/HandBuiltIntegrationEventDraftFixture.java b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/HandBuiltIntegrationEventDraftFixture.java new file mode 100644 index 00000000..d95d3634 --- /dev/null +++ b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/HandBuiltIntegrationEventDraftFixture.java @@ -0,0 +1,40 @@ +package dev.caskeleton.bootstrap.architecture.violations.application; + +import dev.caskeleton.application.messaging.contract.ContractId; +import dev.caskeleton.application.messaging.contract.IntegrationPayload; +import dev.caskeleton.application.messaging.contract.LogicalDestinationId; +import dev.caskeleton.application.messaging.event.AggregateIdentity; +import dev.caskeleton.application.messaging.event.AggregateOrder; +import dev.caskeleton.application.messaging.event.EventId; +import dev.caskeleton.application.messaging.event.IntegrationEventDraft; +import java.time.Instant; +import java.util.Optional; + +/** + * Negative fixture for {@code INTEGRATION_EVENT_DRAFTS_ARE_BUILT_BY_THE_SHARED_HELPER}: a feature + * that assembles the ten-field draft itself and so restates the correlation and tenant rules {@code + * IntegrationEventDrafts} owns (review 2026-11-01 01-09, 2026-11-03 RR-01-03). + */ +public final class HandBuiltIntegrationEventDraftFixture { + + private HandBuiltIntegrationEventDraftFixture() {} + + public static

    IntegrationEventDraft

    draft( + EventId eventId, + ContractId contractId, + LogicalDestinationId destination, + AggregateIdentity aggregate, + P payload) { + return new IntegrationEventDraft<>( + eventId, + contractId, + 1, + destination, + aggregate, + AggregateOrder.first(), + Instant.EPOCH, + eventId.value(), + Optional.empty(), + payload); + } +} diff --git a/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/SystemInvokedFixtureUseCase.java b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/SystemInvokedFixtureUseCase.java new file mode 100644 index 00000000..00e5fc4d --- /dev/null +++ b/src/app-bootstrap/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/SystemInvokedFixtureUseCase.java @@ -0,0 +1,16 @@ +package dev.caskeleton.bootstrap.architecture.violations.application; + +import dev.caskeleton.application.security.SystemInvoked; + +/** + * A use case only the system calls. Not a violation on its own: it is what {@code + * SystemInvokedCallingControllerFixture} reaches, the transport path {@code + * INBOUND_ADAPTERS_DO_NOT_REACH_SYSTEM_INVOKED_USE_CASES} forbids. + */ +@SystemInvoked(reason = "fixture: a scheduler calls it") +public final class SystemInvokedFixtureUseCase { + + public void handle() { + // fixture + } +} diff --git a/src/app-bootstrap/src/main/java/dev/caskeleton/bootstrap/runtime/SecretSourceValidator.java b/src/app-bootstrap/src/main/java/dev/caskeleton/bootstrap/runtime/SecretSourceValidator.java index 807cda18..7d4c1662 100644 --- a/src/app-bootstrap/src/main/java/dev/caskeleton/bootstrap/runtime/SecretSourceValidator.java +++ b/src/app-bootstrap/src/main/java/dev/caskeleton/bootstrap/runtime/SecretSourceValidator.java @@ -6,8 +6,10 @@ import dev.caskeleton.adapter.outbound.support.startup.StartupPhase; import dev.caskeleton.adapter.outbound.support.startup.StartupRule; import dev.caskeleton.application.secret.SecretSourcePort; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Supplier; @@ -32,6 +34,25 @@ public final class SecretSourceValidator implements StartupRule { /** Prefix marking a dev/local fake credential; forbidden to reach the prod profile. */ static final String LOCAL_DEV_SENTINEL_PREFIX = "__LOCAL_DEV_"; + /** + * Prefix of the passwords this repository writes down for its lanes: the Redis ACL fixtures in + * {@code infra/redis-sdk/acl} and the Compose lanes' {@code REDIS_LANE_*} values that connect + * with them. They are public — anyone who can read the repository has them — so under {@code + * prod} they are refused like the sentinel (review 2026-11-03 RR-07-04). A deployment that copied + * a lane's environment — which is also what an application needs to authenticate against a Redis + * that loaded the lane ACL file — would start with credentials it does not own; this is the one + * place that sees every value before anything connects. It cannot see a Redis server that loaded + * that file while the application uses real passwords; the ACL README says why no deployment + * loads it. + */ + static final String REPOSITORY_FIXTURE_PREFIX = "fixture-"; + + /** Values that are public by construction and so can never be a production secret. */ + private static final Map PUBLIC_CREDENTIAL_PREFIXES = + Map.of( + LOCAL_DEV_SENTINEL_PREFIX, "sentinel credential", + REPOSITORY_FIXTURE_PREFIX, "repository fixture password (public, infra/redis-sdk/acl)"); + /** * Secret keys that must be injected under the {@code prod} profile whatever the deployment * selected. The relational credential is not one of them: persistence-jpa contributes it while it @@ -68,7 +89,8 @@ public final class SecretSourceValidator implements StartupRule { @Override public String remedy() { return "inject real non-blank secrets from the secret manager / mounted environment; " - + "local/dev sentinel credentials are forbidden in prod."; + + "local/dev sentinel credentials and the repository's fixture-* passwords are forbidden" + + " in prod."; } @Override @@ -77,31 +99,44 @@ public final class SecretSourceValidator implements StartupRule { return List.of(); } List violations = new ArrayList<>(); - collectLocalDevSentinelViolations(environment, violations); + collectPublicCredentialViolations(environment, violations); collectMissingSecretViolations(environment, violations); return violations; } - private static void collectLocalDevSentinelViolations( + /** + * Names every property whose value is public by construction. Every property source is read, the + * process environment included, because a credential reference ({@code + * secret://@environment/}) may point at any variable name. The value itself is + * never put in the message. + */ + private static void collectPublicCredentialViolations( Environment environment, List violations) { if (!(environment instanceof ConfigurableEnvironment configurable)) { return; } - Set offenders = new LinkedHashSet<>(); + Map offenders = new LinkedHashMap<>(); for (PropertySource source : configurable.getPropertySources()) { if (source instanceof EnumerablePropertySource enumerable) { for (String name : enumerable.getPropertyNames()) { - Object value = enumerable.getProperty(name); - if (value instanceof String text && text.startsWith(LOCAL_DEV_SENTINEL_PREFIX)) { - offenders.add(name); + if (enumerable.getProperty(name) instanceof String text) { + PUBLIC_CREDENTIAL_PREFIXES.keySet().stream() + .filter(text::startsWith) + .findFirst() + .ifPresent(prefix -> offenders.putIfAbsent(name, prefix)); } } } } - for (String offender : offenders) { - violations.add( - "'" + LOCAL_DEV_SENTINEL_PREFIX + "' sentinel credential is configured for " + offender); - } + offenders.forEach( + (offender, prefix) -> + violations.add( + "'" + + prefix + + "' " + + PUBLIC_CREDENTIAL_PREFIXES.get(prefix) + + " is configured for " + + offender)); } private void collectMissingSecretViolations(Environment environment, List violations) { diff --git a/src/app-bootstrap/src/main/resources/config/observability.yml b/src/app-bootstrap/src/main/resources/config/observability.yml index 5b5b6d1a..6aedc565 100644 --- a/src/app-bootstrap/src/main/resources/config/observability.yml +++ b/src/app-bootstrap/src/main/resources/config/observability.yml @@ -111,8 +111,9 @@ logging: # DEBUG here prints SQL once JPA/jdbc is wired in org.hibernate.SQL: ${APP_LOG_LEVEL_SQL:INFO} -# Module-scoped knobs. Each block is bound into a *Settings @ConfigurationProperties -# record in the corresponding module, which is where allowed-value validation lives. +# Module-scoped knobs, each read and validated by the module that owns it (a settings record bound +# by StrictSettingsBinder, which refuses an unknown key). `app.logging` has no settings record: +# logback-spring.xml reads it () and LoggingConfigurationRule validates it. app: logging: file: diff --git a/src/app-bootstrap/src/main/resources/config/runtime.yml b/src/app-bootstrap/src/main/resources/config/runtime.yml index 3309b75c..e1ca9d8f 100644 --- a/src/app-bootstrap/src/main/resources/config/runtime.yml +++ b/src/app-bootstrap/src/main/resources/config/runtime.yml @@ -19,7 +19,8 @@ app: async: # feature-background-job-async-contract D7 — @Async ThreadPoolTaskExecutor pool sizing. # Registry SSOT: docs/registries/env-keys.yaml (APP_ASYNC_EXECUTOR_* rows 1337-1377). - # Bound into AsyncExecutorSettings @ConfigurationProperties(prefix = "app.async.executor"). + # Bound into the AsyncExecutorSettings record by StrictSettingsBinder (prefix "app.async.executor"), + # which refuses an unknown key under the prefix instead of ignoring it. # Bounded queue is mandatory (D7) — an unbounded queue makes max-size unreachable. executor: # int >= 1 (positive_int). Always-alive worker count. diff --git a/src/app-bootstrap/src/systemTest/java/dev/caskeleton/bootstrap/composition/BootstrapCapabilityReachabilitySystemTest.java b/src/app-bootstrap/src/systemTest/java/dev/caskeleton/bootstrap/composition/BootstrapCapabilityReachabilitySystemTest.java index adce9ed6..f6a89f02 100644 --- a/src/app-bootstrap/src/systemTest/java/dev/caskeleton/bootstrap/composition/BootstrapCapabilityReachabilitySystemTest.java +++ b/src/app-bootstrap/src/systemTest/java/dev/caskeleton/bootstrap/composition/BootstrapCapabilityReachabilitySystemTest.java @@ -27,7 +27,7 @@ class BootstrapCapabilityReachabilitySystemTest { try (ConfigurableApplicationContext context = ShippedCompositionHarness.shippedComposition("local") .run(ShippedCompositionHarness.allOffArguments())) { - assertThat(context.containsBean("outboxLeaderElection")).isFalse(); + assertThat(context.containsBean("outboxSettings")).isFalse(); assertThat(context.containsBean("distributedLockProvider")).isFalse(); assertThat(context.getBeansOfType(IdempotencyExecutor.class)).isEmpty(); assertThat( diff --git a/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/contract/DeveloperExperienceContractTest.java b/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/contract/DeveloperExperienceContractTest.java index 8ff57184..546ec57d 100644 --- a/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/contract/DeveloperExperienceContractTest.java +++ b/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/contract/DeveloperExperienceContractTest.java @@ -8,6 +8,7 @@ import dev.caskeleton.bootstrap.contract.support.RepositoryContractResources; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.List; import java.util.Map; import java.util.regex.Matcher; @@ -15,6 +16,7 @@ import java.util.regex.Pattern; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.springframework.boot.convert.DurationStyle; import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.SafeConstructor; @@ -407,7 +409,13 @@ class DeveloperExperienceContractTest { // given Map app = requireMap(requireMap(parseYamlMap(read("docker-compose.yml")), "services"), "app"); - int webDrainSeconds = 30; + // The web drain is the value this service runs with — APP_SERVER_SHUTDOWN_TIMEOUT's fallback in + // the same Compose service — not a number copied here, so a changed default changes the sum + // (review 2026-11-03 RR-06-08). + long webDrainSeconds = + composeFallback( + requireString(requireMap(app, "environment"), "APP_SERVER_SHUTDOWN_TIMEOUT")) + .toSeconds(); int preStopSeconds = 5; int marginSeconds = 5; @@ -417,7 +425,7 @@ class DeveloperExperienceContractTest { // then // Spring finishes the lifecycle phase (the web server's graceful drain) before it destroys the // async executor, which then waits for its queue: the two run one after the other (07-07). - assertThat(Integer.parseInt(grace.replace("s", ""))) + assertThat(Long.parseLong(grace.replace("s", ""))) .isGreaterThanOrEqualTo( webDrainSeconds + AsyncExecutorConfig.AWAIT_TERMINATION_SECONDS @@ -621,6 +629,15 @@ class DeveloperExperienceContractTest { return (List) value; } + /** The fallback of a Compose interpolation {@code ${NAME:-30s}}, as a duration. */ + private static Duration composeFallback(String interpolation) { + Matcher fallback = Pattern.compile("^\\$\\{[A-Z0-9_]+:-([^}]+)}$").matcher(interpolation); + assertThat(fallback.matches()) + .as("a ${NAME:-default} interpolation: " + interpolation) + .isTrue(); + return DurationStyle.detectAndParse(fallback.group(1)); + } + private static List requireStringList(Map owner, String key) { List values = requireList(owner, key); assertThat(values).allMatch(String.class::isInstance); diff --git a/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/contract/SampleRuntimeDefaultsContractTest.java b/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/contract/SampleRuntimeDefaultsContractTest.java index 543e31cb..805a300f 100644 --- a/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/contract/SampleRuntimeDefaultsContractTest.java +++ b/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/contract/SampleRuntimeDefaultsContractTest.java @@ -4,12 +4,21 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.bootstrap.contract.support.RepositoryContractResources; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Stream; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -19,8 +28,10 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; /** * The sample keeps the default composition's runtime defaults, so a number measured on the sample — - * thread count, pool wait — says something about the composition a service ships (09-08). It reads - * two configuration files, so it runs in the pull-request {@code repository-contract} lane (08-03). + * thread count, pool wait — says something about the composition a service ships (09-08), and it + * carries only settings something on its classpath reads. It reads configuration files, build + * scripts and production sources, so it runs in the pull-request {@code repository-contract} lane + * (08-03). */ @Tag("repository-contract") class SampleRuntimeDefaultsContractTest { @@ -69,6 +80,101 @@ class SampleRuntimeDefaultsContractTest { assertThat(connectionTimeout).isNull(); } + @Test + @DisplayName("샘플 application.yml의 app.* 설정 묶음은 샘플 classpath에 있는 모듈 코드가 읽는다") + void everySampleAppSettingGroupIsReadByAModuleOnTheSampleClasspath() throws IOException { + // given + Set groups = new TreeSet<>(); + for (Object document : loadAll("src/sample-portfolio/src/main/resources/application.yml")) { + if (document instanceof Map map && map.get("app") instanceof Map app) { + app.keySet().forEach(key -> groups.add(String.valueOf(key))); + } + } + Set classpath = sampleRuntimeModules(); + + // when + Set unread = new TreeSet<>(); + for (String group : groups) { + Pattern reader = Pattern.compile("app\\." + Pattern.quote(group) + "[\".}]"); + if (classpath.stream().noneMatch(module -> anyMainSourceMatches(module, reader))) { + unread.add("app." + group); + } + } + + // then + // The sample is copied first. A block nothing on its classpath reads — the HTTP client's + // timeouts with no HTTP client module, a messaging broker with no messaging module — is a + // setting the copier believes is in force and is not (review 2026-11-03 RR-06-06, 09-08). + assertThat(unread) + .as("app.* groups in the sample's application.yml that nothing reads") + .isEmpty(); + } + + /** The source directories of the sample and every project its production configurations reach. */ + private static Set sampleRuntimeModules() throws IOException { + String registry = + Files.readString(REPOSITORY_ROOT.resolve("src/config/architecture/modules.json")); + Map sourceOf = new LinkedHashMap<>(); + Matcher module = + Pattern.compile("\"gradle_path\":\\s*\"([^\"]+)\",\\s*\"source_path\":\\s*\"([^\"]+)\"") + .matcher(registry); + while (module.find()) { + sourceOf.put(module.group(1), module.group(2)); + } + sourceOf.put(":sample-portfolio", "src/sample-portfolio"); + Pattern edge = + Pattern.compile( + "(?m)^\\s*(?:api|implementation|runtimeOnly)\\s*\\(?\\s*project\\(\\s*['\"](:[^'\"]+)['\"]"); + Set reached = new LinkedHashSet<>(); + Deque pending = new ArrayDeque<>(List.of(":sample-portfolio")); + Set seen = new LinkedHashSet<>(); + while (!pending.isEmpty()) { + String path = pending.pop(); + String source = sourceOf.get(path); + if (!seen.add(path) || source == null) { + continue; + } + Path directory = REPOSITORY_ROOT.resolve(source); + reached.add(directory); + Matcher dependency = edge.matcher(Files.readString(directory.resolve("build.gradle"))); + while (dependency.find()) { + pending.add(dependency.group(1)); + } + } + return reached; + } + + private static boolean anyMainSourceMatches(Path module, Pattern reader) { + Path sources = module.resolve("src/main/java"); + if (!Files.isDirectory(sources)) { + return false; + } + try (Stream files = Files.walk(sources)) { + return files + .filter(file -> file.toString().endsWith(".java")) + .anyMatch( + file -> { + try { + return reader.matcher(Files.readString(file)).find(); + } catch (IOException failure) { + throw new UncheckedIOException(failure); + } + }); + } catch (IOException failure) { + throw new UncheckedIOException(failure); + } + } + + private static List loadAll(String relative) throws IOException { + LoaderOptions options = new LoaderOptions(); + options.setMaxAliasesForCollections(0); + List documents = new ArrayList<>(); + new Yaml(new SafeConstructor(options)) + .loadAll(Files.readString(REPOSITORY_ROOT.resolve(relative))) + .forEach(documents::add); + return documents; + } + private static String defaultOf(Object value) { if (value == null) { return null; diff --git a/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/contract/outbox/OutboxStatusRegistryContractTest.java b/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/contract/outbox/OutboxStatusRegistryContractTest.java index d8bf639c..eed9079e 100644 --- a/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/contract/outbox/OutboxStatusRegistryContractTest.java +++ b/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/contract/outbox/OutboxStatusRegistryContractTest.java @@ -3,24 +3,28 @@ package dev.caskeleton.bootstrap.contract.outbox; import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.application.outbox.OutboxEventStatus; +import dev.caskeleton.application.outbox.OutboxRelayResult; import dev.caskeleton.bootstrap.contract.support.RepositoryContractResources; import dev.caskeleton.shared.error.OperationalError; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.yaml.snakeyaml.Yaml; /** - * Contract test — {@link OutboxEventStatus} 5 values ↔ {@code metrics.yaml} {@code - * outbox.pending.size} allowed_values, and {@code OUTBOX_*} 2 error codes ↔ {@code - * error-codes.yaml} category/retryable. + * Contract test — {@link OutboxEventStatus} values ↔ {@code metrics.yaml} {@code + * outbox.pending.size} allowed_values, {@link OutboxRelayResult.Outcome} values ↔ {@code + * outbox.publisher.published.total} {@code outcome} allowed_values, and {@code OUTBOX_*} 2 error + * codes ↔ {@code error-codes.yaml} category/retryable. */ @Tag("document-contract") class OutboxStatusRegistryContractTest { @@ -79,6 +83,43 @@ class OutboxStatusRegistryContractTest { .containsExactlyInAnyOrderElementsOf(registryStatuses); } + /** + * The relay counts every cycle outcome under {@code outcome}; the registry is what a dashboard or + * an alert reads the allowed values from. {@code RELEASED} was counted and not registered, and + * {@code IN_FLIGHT}, which no cycle produces, was registered (re-review 2026-11-03 RR-05-06). + */ + @Test + @DisplayName("relay 회차 결과 값과 metrics.yaml의 published.total outcome 허용 값이 같다") + void relayOutcomesMatchMetricsYamlPublishedTotalOutcomeAllowedValues() { + // given + Set enumOutcomes = + Arrays.stream(OutboxRelayResult.Outcome.values()) + .map(Enum::name) + .collect(Collectors.toSet()); + + // when + List registered = allowedValues("outbox.publisher.published.total", "outcome"); + + // then + assertThat(registered).containsExactlyInAnyOrderElementsOf(enumOutcomes); + } + + @SuppressWarnings("unchecked") + private static List allowedValues(String metric, String tag) { + List> metrics = (List>) metricsRoot.get("metrics"); + Map row = + metrics.stream() + .filter(m -> metric.equals(m.get("name"))) + .findFirst() + .orElseThrow(() -> new AssertionError(metric + " must exist in metrics.yaml")); + return ((List>) row.get("tags")) + .stream() + .filter(t -> tag.equals(t.get("name"))) + .findFirst() + .map(t -> (List) t.get("allowed_values")) + .orElseThrow(() -> new AssertionError(metric + " must have a '" + tag + "' tag")); + } + @Test void outboxErrorCodesMatchRegistryCategoryAndRetryable() { Map> codeIndex = diff --git a/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/runtime/ErrorDetailExposureSafetyValidatorTest.java b/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/runtime/ErrorDetailExposureSafetyValidatorTest.java index 8b347bbe..a44749ba 100644 --- a/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/runtime/ErrorDetailExposureSafetyValidatorTest.java +++ b/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/runtime/ErrorDetailExposureSafetyValidatorTest.java @@ -68,7 +68,8 @@ class ErrorDetailExposureSafetyValidatorTest { static class ValidatorConfig { @Bean - ErrorDetailExposureSafetyValidator errorDetailExposureSafetyValidator(Environment environment) { + static ErrorDetailExposureSafetyValidator errorDetailExposureSafetyValidator( + Environment environment) { return new ErrorDetailExposureSafetyValidator(); } } diff --git a/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/runtime/RuntimeNumericBoundsValidatorTest.java b/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/runtime/RuntimeNumericBoundsValidatorTest.java index 0d594af4..9a242e87 100644 --- a/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/runtime/RuntimeNumericBoundsValidatorTest.java +++ b/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/runtime/RuntimeNumericBoundsValidatorTest.java @@ -78,7 +78,7 @@ class RuntimeNumericBoundsValidatorTest { @Configuration static class ValidatorConfig { @Bean - RuntimeNumericBoundsValidator runtimeNumericBoundsValidator(Environment environment) { + static RuntimeNumericBoundsValidator runtimeNumericBoundsValidator(Environment environment) { return new RuntimeNumericBoundsValidator(); } } diff --git a/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/runtime/SecretSourceValidatorTest.java b/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/runtime/SecretSourceValidatorTest.java index 4b193fcf..aaeabb20 100644 --- a/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/runtime/SecretSourceValidatorTest.java +++ b/src/app-bootstrap/src/test/java/dev/caskeleton/bootstrap/runtime/SecretSourceValidatorTest.java @@ -103,6 +103,53 @@ class SecretSourceValidatorTest { .run(context -> assertThat(context).hasNotFailed()); } + // ---- repository fixture passwords (review 2026-11-03 RR-07-04) -------------------------- + + @Test + @DisplayName("prod에서 비밀값이 저장소 fixture 비밀번호(fixture-)면 그 변수를 짚어 기동을 거부한다") + void prodProfileWithARepositoryFixturePasswordFailsStartup() { + // given + ApplicationContextRunner prodWithLaneFixture = + runner + .withInitializer(ctx -> ctx.getEnvironment().setActiveProfiles("prod")) + .withPropertyValues(allRequiredSecretsPresent()) + // The Compose Redis lanes' value: a public password from infra/redis-sdk/acl, which a + // deployment that copied the lane's environment would carry into production. + .withPropertyValues("REDIS_LANE_APPLICATION_PASSWORD=fixture-application"); + + // when + prodWithLaneFixture.run( + context -> { + // then + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()) + .isInstanceOfSatisfying( + StartupFailureException.class, + e -> + assertThat(e.errorCode()) + .isEqualTo(StartupErrorCode.STARTUP_VALIDATION_FAILED)) + .hasStackTraceContaining("fixture-") + .hasStackTraceContaining("REDIS_LANE_APPLICATION_PASSWORD"); + }); + } + + @Test + @DisplayName("prod가 아니면 저장소 fixture 비밀번호(fixture-)로도 기동한다") + void nonProdProfileWithARepositoryFixturePasswordStarts() { + // given + ApplicationContextRunner localWithLaneFixture = + runner + .withInitializer(ctx -> ctx.getEnvironment().setActiveProfiles("local")) + .withPropertyValues("REDIS_LANE_APPLICATION_PASSWORD=fixture-application"); + + // when + localWithLaneFixture.run( + context -> { + // then + assertThat(context).hasNotFailed(); + }); + } + // ---- required-secret presence guard (§테스트 계약) ----------------------- @Test diff --git a/src/application-core/CLAUDE.md b/src/application-core/CLAUDE.md index 917d0a4b..93780a35 100644 --- a/src/application-core/CLAUDE.md +++ b/src/application-core/CLAUDE.md @@ -69,7 +69,9 @@ Package root: `dev.caskeleton.application`. | `capability.Idempotency` | `IDEMPOTENT` / `KEYED` / `NOT_IDEMPOTENT`. | | `capability.RepositoryAccess` | `NONE` / `READ_REPOSITORY` / `WRITE_REPOSITORY`. | | `messaging.contract.IntegrationEventContractContribution

    ` | Closed exact-record payload type, canonical component order, local schema identity/hash and provider-neutral descriptor contribution. | -| `messaging.event.IntegrationEventDraft

    ` | Typed semantic event before local encoding; never JSON, Kafka or persistence state. | +| `messaging.event.IntegrationEventDraft

    ` | Typed semantic event before local encoding; never JSON, Kafka or persistence state. Built only through `IntegrationEventDrafts` (`INTEGRATION_EVENT_DRAFTS_ARE_BUILT_BY_THE_SHARED_HELPER`). | +| `messaging.event.IntegrationEventDrafts` | The one way a feature builds a draft: contract id, version and destination from the contract, correlation from the request when canonical (else the event id), `SINGLE_TENANT_SCOPE` for a feature without tenants. | +| `messaging.event.IntegrationIdentity` | The one grammar and the length bounds of event, correlation/causation and aggregate ids. Test a value with `isCanonical(value, bound)`; never copy the pattern. | | `messaging.event.ValidatedIntegrationEvent` | Stable semantic identities plus immutable exact encoded bytes and hashes, ready for a later durable append boundary. | | `messaging.event.IntegrationEventEncoderPort` | Framework-free local draft-to-validated-event boundary implemented by an outbound adapter (the template's: `adapter:outbound:support` `JsonIntegrationEventEncoder`). | | `outbox.OutboxAppendPort` | The one outbox append: a `ValidatedIntegrationEvent`, in the caller's write transaction. | @@ -77,9 +79,10 @@ Package root: `dev.caskeleton.application`. ## Notification (NOTIF-ADR-005) -`..notification.platform..` (leaf `application-notification`) is canonical. The R1 types directly -under `dev.caskeleton.application.notification` were removed on 2026-09-25; only the simple -`NotificationPort`, `Notification` and `NotificationChannel` remain there. +None of the notification types are in this leaf. `..notification.platform..` and the simple +`NotificationPort`, `Notification` and `NotificationChannel` directly under +`dev.caskeleton.application.notification` all belong to leaf `application-notification`; the shared +package prefix is the only link. The R1 orchestration types were removed on 2026-09-25. ## Naming convention @@ -197,9 +200,12 @@ application-core never self-registers with a DI framework. It is **not final**: declares `@RequiresPermission`, the web adapter enforces it through a CGLIB proxy, and CGLIB cannot subclass a final class (`RequiresPermissionProxyabilityTest`). It is registered as a bean for the same reason — a use case built with `new` is never checked. A use case only the system calls (a -scheduler, a relay) carries `@SystemInvoked(reason = ...)` instead of a permission and is not a bean. -A composition with `@RequiresPermission` beans but no enforcing advisor fails startup -(`composition:service-runtime` `PermissionEnforcementGuard`). +scheduler, a relay) carries `@SystemInvoked(reason = ...)` instead of a permission and is not a bean: +it is a write with no permission check, so a transport must not reach it. Both halves are enforced — +a `@SystemInvoked` bean fails startup, and an inbound adapter that depends on one fails +`INBOUND_ADAPTERS_DO_NOT_REACH_SYSTEM_INVOKED_USE_CASES`. A composition with `@RequiresPermission` +beans but no enforcing advisor fails startup too (`composition:service-runtime` +`PermissionEnforcementGuard`, both checks). ## Allowed transactional shapes diff --git a/src/application-core/README.md b/src/application-core/README.md index a11b353b..b86059ed 100644 --- a/src/application-core/README.md +++ b/src/application-core/README.md @@ -32,7 +32,16 @@ Cache도 같은 원칙을 따른다. `cache` 패키지에는 cache-redis가 구 - `IntegrationEventDraft`는 canonical event/aggregate/order/correlation identity와 typed payload를 보유한다. tenant가 없는 모드도 null 대신 canonical system tenant scope를 `AggregateIdentity`에 넣어 dedupe/order identity가 PostgreSQL nullable uniqueness에 기대지 - 않게 한다. + 않게 한다. 새 event의 event id와 causation id는 정규 소문자 UUIDv7이어야 한다 + (`IntegrationIdentity.isCanonicalUuidV7`). messaging platform의 `MessageId`가 그것만 바꾸지 않고 + 싣기 때문이다. draft가 거부하므로 그런 id는 업무 트랜잭션 안에서 실패하고, 커밋된 뒤 relay에서 + DEAD가 되지 않는다(리뷰 2026-11-01 06-06, 재리뷰 2026-11-03 RR-05-04). 저장 문법 `EventId`는 + 이 규칙 전에 쓰인 행을 읽으려고 넓게 남아 있다. +- draft는 `IntegrationEventDrafts.draft(...)`로만 만든다. 계약 id·버전·목적지, correlation + fallback(요청 값이 `IntegrationIdentity` 문법에 맞지 않으면 event id), tenant 없는 기능의 + `SINGLE_TENANT_SCOPE`가 거기 한 번 있다. 운영 코드가 생성자를 직접 부르면 + `INTEGRATION_EVENT_DRAFTS_ARE_BUILT_BY_THE_SHARED_HELPER`가 막는다. 식별자 문법·길이는 + `IntegrationIdentity`가 정본이고, 값을 미리 검사할 때는 `isCanonical(value, bound)`를 부른다. - `IntegrationEventEncoderPort` 뒤의 adapter가 deterministic encoding을 수행하고 `ValidatedIntegrationEvent`를 돌려준다. 템플릿의 구현은 `adapter:outbound:support`의 `JsonIntegrationEventEncoder`다(envelope v1 + payload record의 결정적 JSON, 선언한 schema hash를 @@ -43,9 +52,38 @@ Cache도 같은 원칙을 따른다. `cache` 패키지에는 cache-redis가 구 payload/envelope byte limit, sensitivity classification, same-event requeue horizon만 표현한다. physical topic, Kafka cluster/security topology는 deployment binding의 책임이다. -이 단계의 완성 범위는 **framework-free semantic contract R1**이다. JSON Schema validator와 -deterministic writer, Kafka ACK producer, PostgreSQL outbox append/relay는 후속 R2 작업이며 여기서 -구현되었거나 검증됐다고 주장하지 않는다. +이 절의 타입은 framework-free semantic contract다. 결정적 writer는 `adapter:outbound:support`의 +`JsonIntegrationEventEncoder`, outbox append·claim은 `adapter:outbound:persistence-jpa`, relay 조립은 +`composition:outbox-relay`, 발행은 `adapter:outbound:messaging`의 platform bridge가 맡는다. payload를 +JSON Schema로 매번 검증하는 validator는 템플릿에 없다(서비스가 encoder bean을 바꿔 둔다). + +### payload version 올리기 (리뷰 2026-11-01 06-10) + +payload schema는 version마다 `contracts/messaging//v.schema.json` 파일 하나다. 한 번 +출시한 version 파일은 고치지 않는다. 이미 그 version으로 쓰인 event가 outbox·broker·DLQ에 남아 있고, +소비자는 그 schema로 만들어져 있다. 그래서 schema를 가진 leaf는 출시한 version의 SHA-256을 +`src/test/resources/contracts/released-payload-schemas.sha256`에 고정하고, testFixtures의 +`ReleasedPayloadSchemaContract`를 상속한 테스트가 두 가지를 막는다. + +- 출시한 version 파일의 바이트가 바뀌면 실패한다(필드 삭제·타입 변경 같은 깨지는 변경 포함). +- 고정되지 않은 version 파일이 있으면 실패한다. 새 version은 manifest에 한 줄을 더하는 순간 출시된다. + +필드를 바꾸거나 지우거나 더하는 변경은 새 version으로 낸다(설계 문서 2026-07-28 §14.4 STRICT_VERSIONED). + +1. 새 version 추가: `v.schema.json`, 새 payload record, 새 `IntegrationEventContractContribution`. + 옛 version의 contribution은 그대로 둔다. encoder catalog는 `(contractId, payloadVersion)`마다 하나라 + 두 version이 함께 등록된다. +2. 소비자 배포: 모든 소비자가 v과 v을 둘 다 읽게 한다. outbox에 이미 쌓인 v 행은 저장된 + 바이트 그대로 나가므로, 생산자를 바꾼 뒤에도 한동안 두 version이 함께 도착한다. +3. 생산자 전환: draft를 v contract로 만든다. 이 커밋에서 manifest에 v 줄을 더한다. +4. 옛 version 제거: outbox·broker·DLQ·archive에 v이 더 남지 않았을 때만 contribution과 파일을 지우고 + manifest 줄도 지운다. 지우는 줄은 리뷰에서 보이는 변경이다. + +고른 방법과 버린 대안: 출시한 version 파일의 해시 고정(위). 버린 대안 — 직전 version과 새 version의 +JSON Schema diff로 호환성을 검사하는 fixture: payload schema가 닫혀 있어(`unevaluatedProperties: false`) +필드 하나를 더해도 옛 소비자의 검증이 깨지므로, diff는 거의 모든 새 version을 "깨짐"으로 판정한다. +설계도 diff heuristic으로 호환성을 주장하지 않기로 했다(§14.4). 외부 schema registry 도입 — 이 템플릿 +범위에 비해 운영 비용이 크다. --- @@ -367,9 +405,14 @@ claim → 트랜잭션 밖에서 발행 → at-least-once 보장. - 발행 성공(`CONFIRMED`) → `inWrite { markPublished }` → `PUBLISHED`. - 확정 거부(`REJECTED_BEFORE_SEND`·`REJECTED_AFTER_BROKER`) → 시도 예산과 무관하게 바로 `markDead`. - 일시적 실패(`AMBIGUOUS`·`REJECTED_RETRYABLE`, 즉 `OutboxPublishOutcome.retryable()`) → - 시도 예산을 쓰지 않고 항상 `markDeferred(nextAttemptAt)`(FAILED 전이 + 미룬 시도로 기록). broker 정지·종료 중·permit 부족· + 시도 예산을 쓰지 않고 항상 `markDeferred(nextAttemptAt, failureCode)`(FAILED 전이 + 미룬 시도로 기록). broker 정지·종료 중·permit 부족· credential 교체는 모든 행을 같은 방식으로 실패시키고 저절로 끝나므로, 예산을 쓰게 하면 몇 분 - 장애가 aggregate 머리마다 DEAD 행을 남긴다(2026-11-01 리뷰 06-01·06-03). + 장애가 aggregate 머리마다 DEAD 행을 남긴다(2026-11-01 리뷰 06-01·06-03). 대가로 끝이 없다 — + 저절로 풀리지 않는 원인(진짜 credential 오류, Kafka adapter가 이름을 모르는 broker 거부)이면 + 영원히 재시도된다. 그래서 모든 실패 전이는 원인 코드를 함께 적는다(`OutboxPublishAttempt.failureCode`: + publisher가 붙인 코드, 없으면 결과 이름, 던진 예외면 그 타입 이름 → 행 `last_failure_code`, 보고 + `failureCode`). 오래 미룬 행은 `OutboxStorePort.longRetries`로 세고, 운영자는 그 행을 `PARKED`로 + 보류할 수 있다(재리뷰 2026-11-03 RR-05-03, publish-failed runbook §4.1). - 발행 예외(`RuntimeException`, adapter가 분류하지 못한 실패): `budgetedAttemptCount() >= maxAttempts` 면 `markDead`, 아니면 `markFailed(nextAttemptAt)`. 예산은 이 경우에만 쓴다 — payload 자체 결함일 수 있는 실패가 aggregate를 영원히 막지 않게 하는 장치다. @@ -411,6 +454,10 @@ claim → 트랜잭션 밖에서 발행 → at-least-once 보장. `@RequiresPermission("outbox:relay")`는 D4 규칙을 통과시키려고 붙인 장식이었다(빈이 아니라 한 번도 검사되지 않았고, 빈이었다면 매 tick 거부됐다). ArchUnit D4는 `@SystemInvoked`를 권한 선언 대신 받고, `SYSTEM_INVOKED_USE_CASES_DECLARE_NO_PERMISSION`이 둘을 함께 다는 것을 막는다. + 면제의 전제(전송 계층이 닿지 않는다)도 코드가 지킨다: `@SystemInvoked` 빈은 + `PermissionEnforcementGuard`가 기동을 실패시키고, inbound adapter가 그 클래스에 의존하면 + `INBOUND_ADAPTERS_DO_NOT_REACH_SYSTEM_INVOKED_USE_CASES`가 빌드를 실패시킨다(review 2026-11-03 + RR-01-02). ### OutboxBackoffPolicy @@ -439,7 +486,9 @@ claim → 트랜잭션 밖에서 발행 → at-least-once 보장. - `OutboxEventStatus` — 행 상태 머신(D5). `PENDING → IN_FLIGHT → {PUBLISHED | FAILED | DEAD}`, `FAILED → IN_FLIGHT`(재claim), `IN_FLIGHT 고아 → IN_FLIGHT`(I6). `DEAD` 는 FIFO 형제를 막는다 — - 같은 aggregate 의 뒤 행 claim 을 차단하므로 runbook 개입이 필요하다. + 같은 aggregate 의 뒤 행 claim 을 차단하므로 runbook 개입이 필요하다. `PARKED` 는 운영자만 만든다 + (`FAILED → PARKED`, runbook SQL): claim 되지 않고 DEAD 처럼 FIFO 형제를 막는다. 재개는 + `PARKED → FAILED`, 버리기는 `PARKED → PUBLISHED`. relay 의 어떤 전이도 `PARKED` 로 가거나 나오지 않는다. - append 는 `OutboxAppendPort.append(ValidatedIntegrationEvent)` 하나다(2026-09-28, 소유자 결정 7). use case 가 `IntegrationEventDraft` 를 만들고 `IntegrationEventEncoderPort` 로 encode 한 뒤 같은 write 트랜잭션 안에서 append 한다. consumer 의 중복 제거 키는 event id 다. @@ -448,11 +497,11 @@ claim → 트랜잭션 밖에서 발행 → at-least-once 보장. 다음 실패 시 dead-letter 여부를 결정한다. 옛 `OutboxEvent`/`NewOutboxEvent`/`LegacyOutboxAppendPort` (free-text payload 모델)는 삭제됐다. - `OutboxRelayResult` — relay 한 사이클의 결과. `claimedCount` + per-event outcome 목록. 내부 - `Outcome` enum 이 `OutboxEventStatus` 의 5값을 재사용하지 않고 4값(PUBLISHED/FAILED/DEAD/ - STALE — 마지막은 lease 를 잃어 결과를 쓰지 못한 경우)을 + `Outcome` enum 이 `OutboxEventStatus` 를 재사용하지 않고 5값(PUBLISHED/FAILED/DEAD/STALE/RELEASED — + STALE 은 lease 를 잃어 결과를 쓰지 못한 경우, RELEASED 는 멈추는 relay 가 쓰지 않은 claim 을 놓은 경우)을 따로 두는 이유: relay 관점은 "이번 실행에서 무슨 일이 있었나" 이지 "행의 현재 영속 상태" 가 - 아니다(후자는 store 가 5값으로 추적). 별도 enum 이 메트릭·스케줄러 로깅 consumer 에게 relay API - 를 깔끔하게 유지해 준다. + 아니다(후자는 store 가 `OutboxEventStatus` 로 추적). 이 5값이 `outbox.publisher.published.total` 의 + `outcome` label 이다(`docs/registries/metrics.yaml`, 계약 테스트가 비교). - `PublishPendingOutboxEventsCommand` — relay 커맨드 마커. 스케줄러 구동이라 caller 파라미터가 없고, 모든 운영 파라미터는 생성 시점에 주입된다(IdempotencyExecutor 선례). 호출마다 새 인스턴스를 만들 필요가 없게 `INSTANCE` 싱글톤을 제공한다. @@ -522,40 +571,12 @@ application 계층은 락 획득/해제 계약만 알고, 실제 구현은 adapt --- -## Notification R1 +## 알림 타입은 이 leaf에 없다 (application-notification) -R1 오케스트레이션 타입(`dev.caskeleton.application.notification` 직속, production consumer 0건)은 -2026-09-25에 삭제했다. canonical은 `..notification.platform..`이다(NOTIF-ADR-005). 직속 패키지에는 -단순 알림 포트 `NotificationPort`·`Notification`·`NotificationChannel`만 남았다. - -## 알림 포트 (notification) - -### NotificationPort - -- named channel + 논리 route 로 알림을 전달하는 outbound 포트. -- 2-인자 overload 는 관례적 `"default"` route 를 쓰는 편의 메서드다. adapter 가 - `app.notification.routes..` 에서 구체 provider 목록을 해석하므로, application - 계층은 provider 선택과 분리된다(호출부에 provider id 가 없다). -- fan-out(route 당 여러 provider)은 adapter 관심사다. 어떤 provider 가 실패하면 adapter 가 - fail-open 정책(관측만, 전파 안 함)을 적용하고, route 자체가 unbound 면 - `AdapterDisabledException("notification", ...)` 을 던진다. -- outbound 포트의 `*Port` 접미사 명명 규칙을 따른다(CLAUDE.md). - -### Notification (값 객체) - -- transport-중립 알림 값으로, routing target 과 내용만 운반한다. -- **절대 logger(또는 dependency logger)에 넘기면 안 된다** — recipient 주소·body 가 로그 라인에 - 닿을 수 없게 하기 위함(PII 계약). -- `adapter-outbound` 에서 `application-core` 로 옮긴 이유: 유스케이스가 adapter 타입을 import 하지 - 않고도 알림을 만들어 `NotificationPort` 로 전달할 수 있게 하기 위함(clean-architecture - HARD-STOP #3). - -### Channel - -- 알림 채널 판별자(`EMAIL`/`SLACK`). provider 와 독립적으로 전달 매체를 식별한다. -- caller 가 `Channel` 값(compile-safe) + 논리 route 이름을 `NotificationPort` 에 넘기면 adapter 가 - `app.notification.routes..` 로 provider 목록을 해석한다 — 도메인은 provider - 세부를 들여다보지 않는다(HARD-STOP #4). +`NotificationPort`·`Notification`·`NotificationChannel`과 알림 플랫폼(`..notification.platform..`, +NOTIF-ADR-005)은 leaf `application-notification`(`src/application-notification`)이 소유한다. 패키지 +이름이 `dev.caskeleton.application.notification`이라 이 leaf의 것처럼 보일 뿐이다. 설명과 규칙은 그 +leaf의 코드 Javadoc과 `docs/notification/`이 정본이다. R1 오케스트레이션 타입은 2026-09-25에 삭제했다. --- @@ -593,9 +614,11 @@ R1 오케스트레이션 타입(`dev.caskeleton.application.notification` 직속 벽시계(예전 샘플의 epoch 마이크로초)는 인스턴스 사이 시계 차이·NTP 되돌림·같은 마이크로초 두 건에서 커밋 순서와 달라진다. - 계약(`OutboxAggregateSequenceContract`, fake `InMemoryOutboxAggregateSequences`와 JPA 구현이 공유): - aggregate마다 1부터 하나씩 커지고 aggregate끼리 독립이다. JPA 구현(`JpaOutboxStore`, V21 - `outbox_aggregate_sequence`)은 counter row lock으로 같은 aggregate 쓰기를 직렬화하고, 기존 outbox - row의 최대 번호 다음에서 시작하며, reaper가 row를 지워도 번호를 다시 쓰지 않는다. 같은 + aggregate마다 첫 번호부터 하나씩 커지고, 첫 쓰기 두 개가 겹쳐도 둘 다 성공하며, aggregate끼리 + 독립이다. 첫 번호는 구현마다 다르다(fake는 1). JPA 구현(`JpaOutboxStore`, V21 + `outbox_aggregate_sequence`)은 counter row lock으로 같은 aggregate 쓰기를 직렬화하고, 남은 outbox + row의 최대 번호와 V21이 기록한 floor(적용 시각의 epoch 마이크로초) 중 큰 값 다음에서 시작한다. + 그래서 reaper가 지웠거나 identity가 바뀐 aggregate도 옛 벽시계 번호보다 뒤에서 시작한다. 같은 `(tenant_scope, aggregate_type, aggregate_id, aggregate_sequence, event_index)` 두 번째 append는 V21 유일 인덱스가 append 시점에 거부한다. - aggregate가 자체 version 열(낙관적 락)을 가지면 그 값을 써도 된다. 이 포트는 counter가 없는 diff --git a/src/application-core/build.gradle b/src/application-core/build.gradle index 2317c633..b077bf43 100644 --- a/src/application-core/build.gradle +++ b/src/application-core/build.gradle @@ -28,7 +28,8 @@ strictTestLanes { description = 'Runs exact Messaging application contract qualification tests.' requires 'dev.caskeleton.application.messaging.contract.IntegrationEventContractContributionTest', 'dev.caskeleton.application.messaging.event.IntegrationEventDraftTest', - 'dev.caskeleton.application.messaging.event.ValidatedIntegrationEventTest' + 'dev.caskeleton.application.messaging.event.ValidatedIntegrationEventTest', + 'dev.caskeleton.application.messaging.contract.ApplicationPayloadSchemaReleaseTest' rejectSkipped = true } } diff --git a/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyCompletionUnknownException.java b/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyCompletionUnknownException.java new file mode 100644 index 00000000..2d067c1b --- /dev/null +++ b/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyCompletionUnknownException.java @@ -0,0 +1,32 @@ +package dev.caskeleton.application.idempotency; + +import dev.caskeleton.shared.error.ApiErrorCarrier; +import dev.caskeleton.shared.error.ApiErrorCode; +import dev.caskeleton.shared.error.OperationalError; + +/** + * The action ran, and the store did not answer when its result was being recorded. + * + *

    Not {@link IdempotencyUnavailableException}: that one is raised before the action, so the + * request certainly did not apply. Here it may have. Effects inside the action's own transaction + * roll back with this exception; anything else the action did — a call to another system, a message + * — has happened. Telling the client "not applied" would invite a resend under a new key, which is + * the duplicate the key exists to prevent (re-review 2026-11-03, RR-02-03). + * + *

    A resend with the same key is safe: the server judges it as a replay when the result was in + * fact recorded, as in-flight while the attempt's lease lasts, and as recovery-required after that. + */ +public final class IdempotencyCompletionUnknownException extends RuntimeException + implements ApiErrorCarrier { + + private static final long serialVersionUID = 1L; + + public IdempotencyCompletionUnknownException() { + super("idempotency store did not answer while recording a completed action; outcome unknown"); + } + + @Override + public ApiErrorCode errorCode() { + return OperationalError.IDEMPOTENCY_COMPLETION_UNKNOWN; + } +} diff --git a/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyExecutor.java b/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyExecutor.java index f7f2ab9c..6dae5a62 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyExecutor.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyExecutor.java @@ -16,9 +16,26 @@ import java.util.Optional; * *

    The action runs only after a confirmed start. This preserves request-replay evidence; it does * not create a cross-store exactly-once boundary, and nothing here should be read as claiming one. + * + *

    A store that does not answer is reported by when it stopped answering: before the action it is + * {@link IdempotencyUnavailableException} (not applied), after it {@link + * IdempotencyCompletionUnknownException} (outcome unknown). {@code docs/web/http-contract.md} §2 + * gives the client behaviour for each. */ public final class IdempotencyExecutor { + /** + * How long a refused duplicate is told to wait before resending with the same key: the {@code + * Retry-After} of the in-flight 409, and the one value every transport and the lease check read. + * The executor does not wait itself; it refuses at once (docs/web/http-contract.md §2). + * + *

    It must stay below the processing lease, so the resend arrives while the first attempt can + * still finish; an early one costs only another 409. {@code IdempotencySettings} refuses to start + * with a lease no longer than this, so the rule is enforced where the lease is configured rather + * than remembered (re-review 2026-11-03, RR-02-05). + */ + public static final Duration IN_FLIGHT_RETRY_AFTER = Duration.ofSeconds(2); + private final IdempotencyStorePort store; private final Duration processingLeaseTtl; private final Duration replayTtl; @@ -230,7 +247,8 @@ public final class IdempotencyExecutor { return switch (store.complete( owner, response, replayTtl, request.claimAttempt().operationId())) { case COMPLETED, ALREADY_COMPLETED_SAME_RESULT -> result; - case UNAVAILABLE -> throw new IdempotencyUnavailableException(); + // The action has run. Reporting "unavailable" here would tell the client it did not. + case UNAVAILABLE -> throw new IdempotencyCompletionUnknownException(); case INDETERMINATE -> reconcileCompletion(request, result, codec, alreadyRetried); default -> throw recovery("completion was not confirmed"); }; @@ -265,7 +283,7 @@ public final class IdempotencyExecutor { } yield complete(request, requireOwner(inspection), result, codec, true); } - case UNAVAILABLE -> throw new IdempotencyUnavailableException(); + case UNAVAILABLE -> throw new IdempotencyCompletionUnknownException(); default -> throw recovery("the completion response is indeterminate and was not reconciled"); }; } diff --git a/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyInFlightException.java b/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyInFlightException.java index 802cce5b..dbb8d236 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyInFlightException.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyInFlightException.java @@ -8,7 +8,8 @@ import dev.caskeleton.shared.error.OperationalError; * Raised when another attempt still holds the same {@link IdempotencyScopeDigest}. Framework-free, * application-owned; every transport maps it to a {@code IDEMPOTENT_IN_FLIGHT} 409 (retryable=true * — the result is unknown and the client resends with the same key, never a new one; {@code - * docs/web/http-contract.md}). The message carries only the opaque scope digest, for logs. + * docs/web/http-contract.md}). The message carries only the opaque scope digest, for logs. How long + * the client waits before resending is {@link IdempotencyExecutor#IN_FLIGHT_RETRY_AFTER}. */ public class IdempotencyInFlightException extends RuntimeException implements ApiErrorCarrier { diff --git a/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyUnavailableException.java b/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyUnavailableException.java index 7d0352f2..53249eea 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyUnavailableException.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyUnavailableException.java @@ -4,7 +4,12 @@ import dev.caskeleton.shared.error.ApiErrorCarrier; import dev.caskeleton.shared.error.ApiErrorCode; import dev.caskeleton.shared.error.OperationalError; -/** Fail-closed signal raised before an action when request-replay coordination is unavailable. */ +/** + * Fail-closed signal raised before an action when request-replay coordination is unavailable. + * + *

    Only before the action: the request certainly did not apply. A store that stops answering + * after the action ran is {@link IdempotencyCompletionUnknownException} instead. + */ public final class IdempotencyUnavailableException extends RuntimeException implements ApiErrorCarrier { diff --git a/src/application-core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventDraft.java b/src/application-core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventDraft.java index b09a2360..c949d2ab 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventDraft.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventDraft.java @@ -6,7 +6,16 @@ import dev.caskeleton.application.messaging.contract.LogicalDestinationId; import java.time.Instant; import java.util.Optional; -/** Feature-owned semantic event before local encoding and schema validation. */ +/** + * Feature-owned semantic event before local encoding and schema validation. + * + *

    A new event's id and causation id are canonical lower-case UUIDv7s ({@link + * IntegrationIdentity#isCanonicalUuidV7}). The messaging bridge sends the event under its id as the + * platform's message id and refuses any other spelling; refused there, the event was already + * committed and the relay could only park it, with every later event of its aggregate behind it. + * Refused here, the write that announces it fails instead (review 06-06, re-review 2026-11-03 + * RR-05-04). + */ public record IntegrationEventDraft

    ( EventId eventId, ContractId contractId, @@ -33,11 +42,24 @@ public record IntegrationEventDraft

    ( if (occurredAt == null) { throw new IllegalArgumentException("occurredAt must not be null"); } + if (!IntegrationIdentity.isCanonicalUuidV7(eventId.value())) { + throw new IllegalArgumentException( + "eventId must be a canonical lower-case UUIDv7, the id the messaging platform carries;" + + " was " + + eventId.value()); + } IntegrationIdentity.requireCorrelation("correlationId", correlationId); if (causationId == null) { throw new IllegalArgumentException("causationId Optional must not be null"); } - causationId.ifPresent(value -> IntegrationIdentity.requireCorrelation("causationId", value)); + causationId.ifPresent( + value -> { + if (!IntegrationIdentity.isCanonicalUuidV7(value)) { + throw new IllegalArgumentException( + "causationId must be a canonical lower-case UUIDv7, the id of the event that caused" + + " this one"); + } + }); if (featurePayload == null) { throw new IllegalArgumentException("featurePayload must not be null"); } diff --git a/src/application-core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java b/src/application-core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java index 5ead377c..535a146d 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java @@ -1,16 +1,31 @@ package dev.caskeleton.application.messaging.event; -import dev.caskeleton.application.outbox.OutboxPublishOutcome; +import dev.caskeleton.application.outbox.OutboxPublishAttempt; +import java.time.Duration; +import java.util.Optional; import java.util.concurrent.CompletionStage; /** * Application-owned boundary for publishing one fully validated integration event. * *

    The application exposes its canonical event model and outcome vocabulary only. Broker, - * transport, codec, and messaging-platform types remain on the outbound side of the boundary. + * transport, codec, and messaging-platform types remain on the outbound side of the boundary; a + * publisher that knows why an attempt was not confirmed translates its reason into the attempt's + * failure code ({@link OutboxPublishAttempt}) rather than handing the platform's type across. */ @FunctionalInterface public interface IntegrationEventPublishPort { - CompletionStage publish(ValidatedIntegrationEvent event); + CompletionStage publish(ValidatedIntegrationEvent event); + + /** + * The longest one publish can take before the returned stage completes, when the publisher knows + * it. The outbox relay sizes its batch against it (see {@code + * OutboxMessagePublishPort#longestAttempt}). + * + * @return the longest attempt, or empty when unknown + */ + default Optional longestAttempt() { + return Optional.empty(); + } } diff --git a/src/application-core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java b/src/application-core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java index dd1663bd..adc509c3 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java @@ -48,6 +48,31 @@ public final class IntegrationIdentity { && PATTERN.matcher(value).matches(); } + /** + * The spelling of a new event's id and causation id: a canonical lower-case UUIDv7, the only id + * the messaging platform's {@code MessageId} carries unchanged. A UUID in upper case or in the + * short form reads back as a different string, and any other version is refused outright. + */ + public static final String UUID_V7_GRAMMAR = + "[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"; + + private static final Pattern UUID_V7 = Pattern.compile(UUID_V7_GRAMMAR); + + /** + * Whether {@code value} is a canonical lower-case UUIDv7 ({@link #UUID_V7_GRAMMAR}). + * + *

    The stored identity grammar ({@link #GRAMMAR}) is wider, because rows appended before this + * rule may hold other ids and must still be read back. A new event is held to this one where it + * is drafted ({@link IntegrationEventDraft}), so an id the platform would refuse fails in the + * writer's transaction instead of parking the committed event in the relay (review 06-06). + * + * @param value the candidate; {@code null} is not one + * @return true for a canonical lower-case UUIDv7 + */ + public static boolean isCanonicalUuidV7(String value) { + return value != null && UUID_V7.matcher(value).matches(); + } + /** Refuses a correlation or causation id outside the grammar or the bound. */ static void requireCorrelation(String field, String value) { if (!isCanonical(value, CORRELATION_MAX_LENGTH)) { diff --git a/src/application-core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java b/src/application-core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java index 5f3e7469..e5d8719c 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java @@ -17,8 +17,10 @@ import dev.caskeleton.application.transaction.TransactionPort; import dev.caskeleton.application.usecase.CommandUseCase; import dev.caskeleton.shared.digest.Sha256; import java.time.Clock; +import java.time.Instant; import java.util.Objects; import java.util.Optional; +import java.util.UUID; /** * Accepts long-running work: one transaction that writes the operation row and the outbox row. @@ -141,7 +143,7 @@ public class SubmitDurableOperationUseCase */ private static IntegrationEventDraft submittedEvent( DurableOperation operation) { - String identity = canonicalIdentity(operation.operationId()); + String identity = canonicalIdentity(operation.operationId(), operation.submittedAt()); // Single-tenant scope even when the command names a tenant: the event only wakes a worker, the // tenant travels in the stored operation, and mapping a TenantId into the lower-case scope // grammar is the messaging platform's decision, not this use case's. @@ -158,18 +160,34 @@ public class SubmitDurableOperationUseCase } /** - * The operation id itself when it fits the integration-event identity grammar — every id the HTTP - * adapter mints does — and otherwise a digest of it. + * The operation id itself when it is a canonical lower-case UUIDv7 — every id the HTTP adapter + * mints is — and otherwise a UUIDv7 derived from it. * - *

    An operation id may contain {@code ~} or run to 128 characters, and an event id may do - * neither. The digest keeps the event deterministic for the operation, so a retry still produces - * the same event, and the payload carries the id unchanged for the worker. + *

    The event id is the messaging platform's message id, which is a canonical lower-case UUIDv7; + * the draft refuses anything else, so an operation id a client chose ({@code ~report}, a v4, an + * upper-case spelling) cannot be the event id. The derived id keeps the event deterministic for + * the operation — the same id and submission time always give the same event id — and the payload + * carries the operation id unchanged for the worker. Its time field is the submission time, so it + * orders like a minted one; its other 74 bits are the SHA-256 of the operation id. + * + *

    It replaced an {@code operation-} digest, which fit the stored identity grammar and + * passed the append, and which the bridge then refused, parking the committed event (review + * 06-06, re-review 2026-11-03 RR-05-04). */ - static String canonicalIdentity(DurableOperationId operationId) { + static String canonicalIdentity(DurableOperationId operationId, Instant submittedAt) { String value = operationId.value(); - if (IntegrationIdentity.isCanonical(value, IntegrationIdentity.EVENT_ID_MAX_LENGTH)) { + if (IntegrationIdentity.isCanonicalUuidV7(value)) { return value; } - return "operation-" + Sha256.ofUtf8(value).hex(); + byte[] digest = Sha256.ofUtf8(value).bytes(); + long millis = submittedAt.toEpochMilli() & 0xFFFF_FFFF_FFFFL; + long randA = ((digest[0] & 0x0FL) << 8) | (digest[1] & 0xFFL); + long mostSignificant = (millis << 16) | (0x7L << 12) | randA; + long randB = 0; + for (int i = 2; i < 10; i++) { + randB = (randB << 8) | (digest[i] & 0xFFL); + } + long leastSignificant = (randB & 0x3FFF_FFFF_FFFF_FFFFL) | 0x8000_0000_0000_0000L; + return new UUID(mostSignificant, leastSignificant).toString(); } } diff --git a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxAggregateSequencePort.java b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxAggregateSequencePort.java index a042012f..73a17469 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxAggregateSequencePort.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxAggregateSequencePort.java @@ -14,7 +14,9 @@ import dev.caskeleton.application.messaging.event.AggregateIdentity; * version column already counts its writes may use that instead; this port is for the aggregates * that keep no counter. * - *

    Contract, which {@code OutboxAggregateSequenceContract} holds every implementation to: + *

    Contract, which {@code OutboxAggregateSequenceContract} holds every implementation to (one + * case per item; the reaper and seed half of the second item needs outbox rows and is a PostgreSQL + * case, {@code PostgreSqlOutboxAggregateSequenceIntegrationTest}): * *

      *
    • Called inside the caller's write transaction ({@code TransactionPort.inWrite}), the same @@ -22,8 +24,14 @@ import dev.caskeleton.application.messaging.event.AggregateIdentity; * a second writer of the same aggregate waits for the first to commit or roll back; a rolled * back number is handed out again. *
    • Strictly greater than every number this port returned for the aggregate before, including - * for events the retention reaper has since deleted, and greater than any number an earlier - * outbox row of the aggregate carries. + * for events the retention reaper has since deleted, and greater than any number an outbox + * row of the aggregate carries when the call is made. + *
    • Where an implementation replaced an earlier numbering (the sample's wall clock), greater + * than any number that numbering could have sent for the aggregate — also for rows deleted + * since, or written under an identity that has since changed. The first number is therefore + * not necessarily 1: the JPA counter starts above the moment its migration ran, in epoch + * microseconds (review RR-03-02). + *
    • Two first calls for one aggregate that overlap both succeed, with consecutive numbers. *
    • Independent per aggregate identity: one aggregate's numbers say nothing about another's. *
    */ diff --git a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxEventStatus.java b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxEventStatus.java index 355633b6..b4b13a0d 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxEventStatus.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxEventStatus.java @@ -1,8 +1,9 @@ package dev.caskeleton.application.outbox; /** - * Status state machine for a transactional outbox event row. {@code DEAD} blocks its aggregate's - * FIFO siblings until manually resolved. See README for the full transition diagram. + * Status state machine for a transactional outbox event row. {@code DEAD} and {@code PARKED} block + * their aggregate's FIFO siblings until manually resolved. See README for the full transition + * diagram. */ public enum OutboxEventStatus { @@ -25,6 +26,9 @@ public enum OutboxEventStatus { /** * Publish failed transiently; {@code next_attempt_at} carries the backoff deadline. The row will * be re-claimed once {@code now >= next_attempt_at}. + * + *

    A row that failed for a reason of the moment (deferred, {@link + * OutboxStorePort#markDeferred}) stays here without end; {@code last_failure_code} says why. */ FAILED, @@ -33,5 +37,16 @@ public enum OutboxEventStatus { * see {@code OperationalError.OUTBOX_DEAD_LETTER}. While a row is DEAD its aggregate's FIFO queue * is blocked. */ - DEAD + DEAD, + + /** + * Held by an operator: a FAILED row the relay would retry without end, taken out of the retry + * loop while its cause is found ({@code runbook://outbox/publish-failed}, "끝나지 않는 재시도"). + * + *

    No relay transition leads here and none leaves: only the runbook's park and resume SQL do. + * The claim never takes a parked row, and the row still blocks its aggregate's later events, so + * parking stops the retries without letting the aggregate's order slip. Resuming returns it to + * FAILED, due at once; skipping marks it PUBLISHED with the same approval a DEAD skip needs. + */ + PARKED } diff --git a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxLongRetries.java b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxLongRetries.java new file mode 100644 index 00000000..8de5f6ad --- /dev/null +++ b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxLongRetries.java @@ -0,0 +1,24 @@ +package dev.caskeleton.application.outbox; + +/** + * The outbox rows that have been retrying for a long time ({@link OutboxStorePort#longRetries}). + * + * @param rows how many FAILED rows have collected at least the asked-for number of deferred claims + * @param oldestAgeSeconds the oldest such row's age since it occurred, in seconds; 0 when there is + * none + */ +public record OutboxLongRetries(long rows, long oldestAgeSeconds) { + + /** No row is retrying long. */ + public static final OutboxLongRetries NONE = new OutboxLongRetries(0, 0); + + public OutboxLongRetries { + if (rows < 0 || oldestAgeSeconds < 0) { + throw new IllegalArgumentException( + "rows and oldestAgeSeconds must be >= 0, were " + rows + " and " + oldestAgeSeconds); + } + if (rows == 0 && oldestAgeSeconds != 0) { + throw new IllegalArgumentException("no row has no age, was " + oldestAgeSeconds); + } + } +} diff --git a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxMessagePublishPort.java b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxMessagePublishPort.java index 39cb6972..3255b3d7 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxMessagePublishPort.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxMessagePublishPort.java @@ -1,5 +1,8 @@ package dev.caskeleton.application.outbox; +import java.time.Duration; +import java.util.Optional; + /** * Outbound port for publishing a claimed outbox event to the message broker. * Fail-closed: any publish failure MUST be surfaced by throwing a {@link @@ -19,7 +22,8 @@ public interface OutboxMessagePublishPort { void publish(ClaimedOutboxEvent event); /** - * Publishes and reports what was achieved. + * Publishes and reports what was achieved, and the publisher's code for why when it was not a + * confirmation. * *

    The throwing method above cannot express an ambiguous publish: a broker that accepted the * frame and never confirmed it is reported the same way as one that refused it outright, and the @@ -29,11 +33,26 @@ public interface OutboxMessagePublishPort { * @param event the claimed outbox event * @return what the attempt achieved */ - default OutboxPublishOutcome publishForOutcome(ClaimedOutboxEvent event) { + default OutboxPublishAttempt publishForOutcome(ClaimedOutboxEvent event) { publish(event); // A method that either returns or throws can only report these two. Reporting CONFIRMED here // is honest for such an adapter — it is what "returned without throwing" has always meant — // and an adapter that can tell an ambiguous publish apart overrides this. - return OutboxPublishOutcome.CONFIRMED; + return OutboxPublishAttempt.confirmed(); + } + + /** + * The longest one publish can take before this port answers, when the port knows it. + * + *

    The relay holds a whole batch under one lease while it publishes the rows one after another, + * so a batch whose worst case outlasts the lease is reclaimed and sent again by another instance. + * Only the port knows its worst case, so the relay's assembly asks it and refuses a batch size + * that does not fit (review 2026-11-01 06-12). Empty when the port cannot say; the check is then + * skipped, not guessed. + * + * @return the longest attempt, or empty when unknown + */ + default Optional longestAttempt() { + return Optional.empty(); } } diff --git a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxPublishAttempt.java b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxPublishAttempt.java new file mode 100644 index 00000000..af161a42 --- /dev/null +++ b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxPublishAttempt.java @@ -0,0 +1,103 @@ +package dev.caskeleton.application.outbox; + +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Pattern; + +/** + * What one publish attempt achieved, and — when it did not achieve a confirmation — the publisher's + * code for why. + * + *

    The outcome decides what the relay does next; the code is for the operator. A refusal for now + * and an unanswered publish are retried without end (review 2026-11-01 README 15), so a row can + * keep failing for a reason no one sees: a credential that is really wrong, a topic policy the + * broker client has no name for. The outcome alone cannot tell those apart — a shutdown, a closed + * transport and a refused credential are all {@link OutboxPublishOutcome#REJECTED_RETRYABLE}. The + * code can, and the relay writes it on the row and into its failure report (re-review 2026-11-03 + * RR-05-03). + * + *

    The code is the publisher's stable, machine-readable name for the failure — a messaging + * platform failure code such as {@code KAFKA_AUTHENTICATIONEXCEPTION} or {@code + * PUBLISH_DEADLINE_EXCEEDED} — never a message: a message can carry an endpoint, a statement or a + * credential, and the row and the log are read by more people than the broker is. + * + * @param outcome what the attempt achieved + * @param failureCode the publisher's code for an attempt that was not confirmed; empty when the + * publisher named none, and always empty for a confirmed attempt + */ +public record OutboxPublishAttempt(OutboxPublishOutcome outcome, Optional failureCode) { + + /** The longest code a row keeps: the width of {@code outbox_event.last_failure_code}. */ + public static final int FAILURE_CODE_MAX_LENGTH = 120; + + /** Letters and digits, then any of {@code . _ : -}: a name, not a sentence. */ + private static final Pattern FAILURE_CODE = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._:-]*"); + + public OutboxPublishAttempt { + Objects.requireNonNull(outcome, "outcome must not be null"); + Objects.requireNonNull(failureCode, "failureCode must not be null"); + if (outcome == OutboxPublishOutcome.CONFIRMED && failureCode.isPresent()) { + throw new IllegalArgumentException("a confirmed publish has no failure code"); + } + failureCode.ifPresent( + code -> { + if (!isFailureCode(code)) { + throw new IllegalArgumentException( + "failureCode must be 1-" + + FAILURE_CODE_MAX_LENGTH + + " characters matching " + + FAILURE_CODE.pattern()); + } + }); + } + + /** A publish the broker acknowledged. */ + public static OutboxPublishAttempt confirmed() { + return new OutboxPublishAttempt(OutboxPublishOutcome.CONFIRMED, Optional.empty()); + } + + /** + * An attempt whose publisher reported only the outcome. + * + * @param outcome what the attempt achieved + * @return the attempt + */ + public static OutboxPublishAttempt of(OutboxPublishOutcome outcome) { + return new OutboxPublishAttempt(outcome, Optional.empty()); + } + + /** + * An attempt that was not confirmed, with the publisher's code for why. + * + * @param outcome what the attempt achieved; not {@link OutboxPublishOutcome#CONFIRMED} + * @param failureCode the publisher's code, see {@link #isFailureCode} + * @return the attempt + */ + public static OutboxPublishAttempt failed(OutboxPublishOutcome outcome, String failureCode) { + return new OutboxPublishAttempt(outcome, Optional.of(failureCode)); + } + + /** + * Whether {@code candidate} can be kept as a failure code. A publisher translating a foreign code + * asks first and falls back to a name of its own when the answer is no. + * + * @param candidate the code; {@code null} is not one + * @return true when it is 1-{@value #FAILURE_CODE_MAX_LENGTH} characters of the code grammar + */ + public static boolean isFailureCode(String candidate) { + return candidate != null + && !candidate.isEmpty() + && candidate.length() <= FAILURE_CODE_MAX_LENGTH + && FAILURE_CODE.matcher(candidate).matches(); + } + + /** + * The code the relay records for this attempt: the publisher's, or the outcome's own name when + * the publisher gave none. + * + * @return a code that is never empty + */ + public String recordedFailureCode() { + return failureCode.orElse(outcome.name()); + } +} diff --git a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxPublishOutcome.java b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxPublishOutcome.java index d895ef23..5539682c 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxPublishOutcome.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxPublishOutcome.java @@ -12,12 +12,14 @@ package dev.caskeleton.application.outbox; * two outbox models is real: * *

      - *
    • the legacy {@code OutboxEventStatus.FAILED} is retryable and {@code DEAD} is - * terminal; - *
    • the platform's {@code OutboxStatus.FAILED} is a definite rejection and terminal, - * while {@code AMBIGUOUS} is the retryable one. + *
    • this application's {@code OutboxEventStatus.FAILED} ({@code outbox_event}) is + * retryable and {@code DEAD} is terminal; + *
    • the vendored platform's {@code OutboxStatus.FAILED} ({@code messaging_outbox}) is a + * definite rejection and terminal, while {@code AMBIGUOUS} is the retryable one. *
    * + *

    The full correspondence is one table, at the head of {@code docs/messaging/outbox-inbox.md}. + * *

    Mapping those by name inverts both meanings: a definitely-rejected message would be retried * forever and an unknown one would be parked. This type exists so no bridge is ever tempted to map * enum names. @@ -43,7 +45,7 @@ public enum OutboxPublishOutcome { * Refused before anything was transmitted — validation, routing, an unknown destination. * *

    Definite: nothing was stored anywhere, so a retry of the same payload fails the same way. - * This is what the legacy model calls {@code DEAD}, not what it calls {@code FAILED}. + * This is what this application's outbox calls {@code DEAD}, not what it calls {@code FAILED}. */ REJECTED_BEFORE_SEND, diff --git a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxRelayFailureReport.java b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxRelayFailureReport.java index f0723dc6..405d5c82 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxRelayFailureReport.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxRelayFailureReport.java @@ -17,6 +17,12 @@ import java.util.Objects; * survives is the exception's class name and, when the exception carries a platform code, that one: * enough to group failures and route a runbook, and nothing that is business content. The bounded * code is {@link #code()}, which this allowlist already owned. + * + *

    {@link #failureCode()} is the code the row records for the failure ({@code + * last_failure_code}): the publisher's own name for a refusal or an unanswered publish, or the + * outcome or exception type when it gave none. The cause type of a returned outcome is one of this + * package's carrier exceptions and says only which outcome it was; the failure code is what tells a + * refused credential from a closed transport (re-review 2026-11-03 RR-05-03). */ public record OutboxRelayFailureReport( OperationalError code, @@ -26,7 +32,8 @@ public record OutboxRelayFailureReport( String correlationId, int attemptCount, Instant nextAttemptAt, - String causeType) { + String causeType, + String failureCode) { public OutboxRelayFailureReport { Objects.requireNonNull(code, "code must not be null"); @@ -35,6 +42,9 @@ public record OutboxRelayFailureReport( requireNonBlank(aggregateId, "aggregateId"); requireNonBlank(correlationId, "correlationId"); requireNonBlank(causeType, "causeType"); + if (!OutboxPublishAttempt.isFailureCode(failureCode)) { + throw new IllegalArgumentException("failureCode must be a bounded failure code"); + } if (attemptCount < 1) { throw new IllegalArgumentException("attemptCount must be >= 1, was " + attemptCount); } @@ -58,7 +68,8 @@ public record OutboxRelayFailureReport( String correlationId, int attemptCount, Instant nextAttemptAt, - RuntimeException cause) { + RuntimeException cause, + String failureCode) { return new OutboxRelayFailureReport( OperationalError.OUTBOX_PUBLISH_FAILED, eventId, @@ -67,7 +78,8 @@ public record OutboxRelayFailureReport( correlationId, attemptCount, nextAttemptAt, - causeTypeOf(cause)); + causeTypeOf(cause), + failureCode); } public static OutboxRelayFailureReport deadLetter( @@ -76,7 +88,8 @@ public record OutboxRelayFailureReport( String aggregateId, String correlationId, int attemptCount, - RuntimeException cause) { + RuntimeException cause, + String failureCode) { return new OutboxRelayFailureReport( OperationalError.OUTBOX_DEAD_LETTER, eventId, @@ -85,7 +98,8 @@ public record OutboxRelayFailureReport( correlationId, attemptCount, null, - causeTypeOf(cause)); + causeTypeOf(cause), + failureCode); } /** diff --git a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxRelayResult.java b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxRelayResult.java index 3f2ce264..90be859a 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxRelayResult.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxRelayResult.java @@ -5,8 +5,9 @@ import java.util.Objects; /** * Result returned by {@link PublishPendingOutboxEventsUseCase} after one relay cycle: the number of - * events claimed and a per-event {@link Outcome} list. See README for why {@code Outcome} is a - * separate 3-value enum rather than reusing {@link OutboxEventStatus}. + * events claimed and a per-event {@link Outcome} list. {@code Outcome} is the cycle's result, not a + * row status, which is why it does not reuse {@link OutboxEventStatus}: a row is never left {@code + * STALE} or {@code RELEASED}, and no cycle ends {@code IN_FLIGHT} or {@code PARKED}. * * @param claimedCount total number of events claimed from the store in this cycle * @param outcomes per-event outcomes in the order they were processed @@ -47,20 +48,21 @@ public record OutboxRelayResult(int claimedCount, List outcomes) { /** * Per-event relay cycle outcome. * - *

    Three values mirror the three terminal states reachable in a single relay cycle: broker ack - * ({@link #PUBLISHED}), transient failure below the retry cap ({@link #FAILED}), or retry cap - * exhausted ({@link #DEAD}). The fourth, {@link #STALE}, records a result this relay was no - * longer allowed to write. + *

    Three values are the row transitions a cycle writes: broker ack ({@link #PUBLISHED}), a + * failure that will be retried ({@link #FAILED}), or one that will not ({@link #DEAD}). {@link + * #STALE} records a result this relay was no longer allowed to write, and {@link #RELEASED} a + * claim a stopping relay gave back unused. The counter {@code outbox.publisher.published.total} + * carries exactly these five as its {@code outcome} label. */ public enum Outcome { /** Broker acknowledged the publish; row transitions to PUBLISHED. */ PUBLISHED, - /** Publish failed transiently; row transitions to FAILED with a backoff window. */ + /** Publish failed and will be retried; row transitions to FAILED with a backoff window. */ FAILED, - /** All retry attempts exhausted; row transitions to DEAD. */ + /** Refused for good, or the attempt budget is spent; row transitions to DEAD. */ DEAD, /** diff --git a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxStorePort.java b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxStorePort.java index fc2b49fc..b96ac1da 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxStorePort.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/outbox/OutboxStorePort.java @@ -7,11 +7,12 @@ import java.util.Map; /** * Outbound port for relay-side access to the outbox store. All mutating operations ({@link - * #claimBatch}, {@link #markPublished}, {@link #markFailed}, {@link #markDead}) must be called - * inside a {@code TransactionPort.inWrite(...)} boundary owned by the relay; implementations must - * not open their own transaction. A completion applies only while the relay's {@link OutboxLease} - * still holds the row, so a relay that stalled past its claim cannot overwrite the relay that took - * the row over. See README for the claim/FIFO/in-flight-timeout/lease semantics. + * #claimBatch}, {@link #markPublished}, {@link #markFailed}, {@link #markDeferred}, {@link + * #releaseLease}, {@link #markDead}) must be called inside a {@code TransactionPort.inWrite(...)} + * boundary owned by the relay; implementations must not open their own transaction. A completion + * applies only while the relay's {@link OutboxLease} still holds the row, so a relay that stalled + * past its claim cannot overwrite the relay that took the row over. See README for the + * claim/FIFO/in-flight-timeout/lease semantics. */ public interface OutboxStorePort { @@ -47,9 +48,11 @@ public interface OutboxStorePort { * * @param lease the claim this relay holds * @param nextAttemptAt the earliest instant at which the event may be re-claimed + * @param failureCode why the attempt failed, kept on the row for the operator (see {@link + * OutboxPublishAttempt}); never a message * @return {@link OutboxCompletion#STALE}, writing nothing, when the lease no longer holds the row */ - OutboxCompletion markFailed(OutboxLease lease, Instant nextAttemptAt); + OutboxCompletion markFailed(OutboxLease lease, Instant nextAttemptAt, String failureCode); /** * Marks the leased event as failed for a reason that belongs to the moment — the broker did not @@ -59,11 +62,16 @@ public interface OutboxStorePort { * ClaimedOutboxEvent#budgetedAttemptCount()}). Must be called inside {@code * TransactionPort.inWrite(...)}. * + *

    A deferred row is retried without end, so its failure code is what tells an operator a row + * that waits out an outage from one the broker will never take — a credential that is really + * wrong, a topic policy the broker client has no name for (re-review 2026-11-03 RR-05-03). + * * @param lease the claim this relay holds * @param nextAttemptAt the earliest instant at which the event may be re-claimed + * @param failureCode the publisher's code for the refusal or the unanswered publish * @return {@link OutboxCompletion#STALE}, writing nothing, when the lease no longer holds the row */ - OutboxCompletion markDeferred(OutboxLease lease, Instant nextAttemptAt); + OutboxCompletion markDeferred(OutboxLease lease, Instant nextAttemptAt, String failureCode); /** * Gives a claimed row back without having tried to publish it: the row is claimable again from @@ -84,9 +92,11 @@ public interface OutboxStorePort { * manually resolved (see README). * * @param lease the claim this relay holds + * @param failureCode why the row was dead-lettered: the refusal's code, or the type of the last + * unclassified failure * @return {@link OutboxCompletion#STALE}, writing nothing, when the lease no longer holds the row */ - OutboxCompletion markDead(OutboxLease lease); + OutboxCompletion markDead(OutboxLease lease, String failureCode); /** * Returns a count of unpublished rows grouped by {@link OutboxEventStatus} — gauge source for the @@ -107,4 +117,21 @@ public interface OutboxStorePort { * @return map from event type to oldest-unpublished-row age in seconds; absent when none */ Map oldestUnpublishedAgeSecondsByEventType(Instant now); + + /** + * The rows that have been retrying for a long time — FAILED, with at least {@code + * minDeferredClaims} claims deferred — counted, with the age of the oldest. Gauge source for + * {@code outbox.retry.long.size} and {@code outbox.retry.long.age}. Called outside a transaction + * (read-only). + * + *

    Deferred claims are the ones that do not spend the attempt budget, so they are also the ones + * with no end: a row that keeps collecting them is retried until someone looks. Counting them is + * how someone is told to look (re-review 2026-11-03 RR-05-03). + * + * @param minDeferredClaims how many deferred claims make a row long-retrying; at least 1 + * @param now current wall-clock time used to compute age + * @return the count and the oldest row's age since it occurred; {@link OutboxLongRetries#NONE} + * when no row qualifies + */ + OutboxLongRetries longRetries(int minDeferredClaims, Instant now); } diff --git a/src/application-core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java b/src/application-core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java index cb7060e0..68ae05cf 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java @@ -43,6 +43,9 @@ import java.util.function.Supplier; public final class PublishPendingOutboxEventsUseCase implements CommandUseCase { + /** The failure code of a thrown failure whose type has no simple name. */ + static final String UNNAMED_FAILURE = "UNNAMED_EXCEPTION"; + private final OutboxStorePort store; private final OutboxMessagePublishPort publishPort; private final OutboxRelayFailureReportPort failureReporter; @@ -168,34 +171,56 @@ public final class PublishPendingOutboxEventsUseCase */ private OutboxRelayResult.Outcome publishOne(OutboxLease lease) { ClaimedOutboxEvent event = lease.event(); - OutboxPublishOutcome achieved; + OutboxPublishAttempt attempt; try { // The outcome-valued call, not the throwing one. An adapter that cannot tell an ambiguous // publish apart still throws and is handled below exactly as before; one that can no longer // has to pretend a message it may have stored was definitely not stored. - achieved = publishPort.publishForOutcome(event); + attempt = publishPort.publishForOutcome(event); } catch (RuntimeException publishEx) { // Publish failure: drive FAILED/DEAD state machine + typed report; do NOT rethrow. return handlePublishFailure(lease, clock.instant(), publishEx); } + OutboxPublishOutcome achieved = attempt.outcome(); if (achieved == OutboxPublishOutcome.AMBIGUOUS) { // The message left this process and no acknowledgement arrived, so it may be stored. Treated // as retryable and reported as such: the duplicate risk is the caller's to bound with an // idempotency key, and the alternative — parking it — loses a message that was delivered. // It does not spend the attempt budget: a broker that stopped answering fails every row this // way, and parking them after three tries is what turned a short outage into DEAD rows. - return retryLater(lease, clock.instant(), new OutboxPublishAmbiguousException(), false); + // + // The same branch has no end for a refusal the broker client could not name. The Kafka + // adapter classifies a failure it does not recognise as ambiguous, so a record the broker + // will never take — a topic timestamp policy, a topic that does not exist — is retried here + // five minutes apart for as long as nobody looks, and its aggregate waits behind it. That is + // the price of not parking a delivered message, and it is paid in the open: the publisher's + // code is written on the row, outbox.retry.long.size counts the row once it has been deferred + // long enough, and the publish-failed runbook parks it (re-review 2026-11-03 RR-05-03). + return retryLater( + lease, + clock.instant(), + new OutboxPublishAmbiguousException(), + attempt.recordedFailureCode(), + false); } if (achieved == OutboxPublishOutcome.REJECTED_RETRYABLE) { // Refused for a reason that is about the moment — shutdown, saturation, a closed transport, // a credential being rotated — not about the message. Retried like an ambiguous outcome and - // for the same reason; the cause says it was a refusal, so no duplicate is suspected. - return retryLater(lease, clock.instant(), new OutboxPublishRefusedException(achieved), false); + // for the same reason; the cause says it was a refusal, so no duplicate is suspected. A + // credential that is really wrong ends up here too, for ever; the failure code is what tells + // it from a rotation. + return retryLater( + lease, + clock.instant(), + new OutboxPublishRefusedException(achieved), + attempt.recordedFailureCode(), + false); } if (achieved != OutboxPublishOutcome.CONFIRMED) { // Definite refusal. Retrying the same payload fails the same way, so the attempt budget is // not spent on it; the row is dead-lettered now rather than after the timer runs out. - return deadLetter(lease, new OutboxPublishRefusedException(achieved)); + return deadLetter( + lease, new OutboxPublishRefusedException(achieved), attempt.recordedFailureCode()); } // markPublished failure (if any) propagates: the row stays IN_FLIGHT and is // recovered via the orphan visibility-timeout reclaim path. @@ -220,11 +245,22 @@ public final class PublishPendingOutboxEventsUseCase */ private OutboxRelayResult.Outcome handlePublishFailure( OutboxLease lease, Instant failedAt, RuntimeException cause) { + String failureCode = failureCodeOf(cause); if (lease.event().budgetedAttemptCount() >= backoffPolicy.maxAttempts()) { // All attempts exhausted — DEAD-letter the event. - return deadLetter(lease, cause); + return deadLetter(lease, cause, failureCode); } - return retryLater(lease, failedAt, cause, true); + return retryLater(lease, failedAt, cause, failureCode, true); + } + + /** + * The code a thrown failure is recorded under: its type's simple name, which is a type and not + * content. A type with no usable name — an anonymous class — is recorded as {@value + * #UNNAMED_FAILURE}. + */ + private static String failureCodeOf(RuntimeException cause) { + String name = cause.getClass().getSimpleName(); + return OutboxPublishAttempt.isFailureCode(name) ? name : UNNAMED_FAILURE; } /** @@ -234,15 +270,19 @@ public final class PublishPendingOutboxEventsUseCase * only a {@code budgeted} failure is recorded as spending the budget. */ private OutboxRelayResult.Outcome retryLater( - OutboxLease lease, Instant failedAt, RuntimeException cause, boolean budgeted) { + OutboxLease lease, + Instant failedAt, + RuntimeException cause, + String failureCode, + boolean budgeted) { ClaimedOutboxEvent event = lease.event(); Instant nextAttemptAt = backoffPolicy.nextAttemptAt(event.attemptCount(), failedAt); OutboxCompletion completion = tx.inWrite( () -> budgeted - ? store.markFailed(lease, nextAttemptAt) - : store.markDeferred(lease, nextAttemptAt)); + ? store.markFailed(lease, nextAttemptAt, failureCode) + : store.markDeferred(lease, nextAttemptAt, failureCode)); if (completion == OutboxCompletion.STALE) { return stale(lease, OutboxRelayResult.Outcome.FAILED); } @@ -255,13 +295,15 @@ public final class PublishPendingOutboxEventsUseCase event.correlationId(), event.attemptCount(), nextAttemptAt, - cause)); + cause, + failureCode)); return OutboxRelayResult.Outcome.FAILED; } - private OutboxRelayResult.Outcome deadLetter(OutboxLease lease, RuntimeException cause) { + private OutboxRelayResult.Outcome deadLetter( + OutboxLease lease, RuntimeException cause, String failureCode) { ClaimedOutboxEvent event = lease.event(); - if (tx.inWrite(() -> store.markDead(lease)) == OutboxCompletion.STALE) { + if (tx.inWrite(() -> store.markDead(lease, failureCode)) == OutboxCompletion.STALE) { return stale(lease, OutboxRelayResult.Outcome.DEAD); } reportFailure( @@ -272,7 +314,8 @@ public final class PublishPendingOutboxEventsUseCase event.aggregateId(), event.correlationId(), event.attemptCount(), - cause)); + cause, + failureCode)); return OutboxRelayResult.Outcome.DEAD; } diff --git a/src/application-core/src/main/java/dev/caskeleton/application/security/SystemInvoked.java b/src/application-core/src/main/java/dev/caskeleton/application/security/SystemInvoked.java index 3b642c04..54b9cb18 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/security/SystemInvoked.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/security/SystemInvoked.java @@ -16,8 +16,13 @@ import java.lang.annotation.Target; * architecture rule that otherwise requires a permission on every {@code WRITE_REPOSITORY} use case * accepts it in place of one. * - *

    A use case marked this way must not become a Spring bean that a transport can reach; the - * composition that runs it constructs it directly. + *

    A use case marked this way is a write with no permission check, so a transport must never + * reach it. The composition that runs it constructs it directly and hands it to its system caller. + * Both ways a transport could reach it are refused: a bean of it fails startup ({@code + * composition:service-runtime} {@code PermissionEnforcementGuard}), and an inbound adapter that + * depends on it fails the architecture rule {@code + * INBOUND_ADAPTERS_DO_NOT_REACH_SYSTEM_INVOKED_USE_CASES}. It is not a way to silence the rule that + * a mutating use case declares a permission: a write a user triggers declares one. */ @Documented @Retention(RetentionPolicy.RUNTIME) diff --git a/src/application-core/src/test/java/dev/caskeleton/application/idempotency/IdempotencyExecutorTest.java b/src/application-core/src/test/java/dev/caskeleton/application/idempotency/IdempotencyExecutorTest.java index 9975c742..73f2062d 100644 --- a/src/application-core/src/test/java/dev/caskeleton/application/idempotency/IdempotencyExecutorTest.java +++ b/src/application-core/src/test/java/dev/caskeleton/application/idempotency/IdempotencyExecutorTest.java @@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.application.transaction.OperationId; +import dev.caskeleton.shared.error.ApiErrorCarrier; import java.time.Duration; import java.time.Instant; import java.util.Optional; @@ -156,6 +157,94 @@ class IdempotencyExecutorTest { assertThat(store.completeCalls).isEqualTo(1); } + @Test + @DisplayName("claim 단계에서 저장소가 응답하지 않으면 action 을 돌리지 않고 실행 전 거부(IDEMPOTENCY_UNAVAILABLE)로 알린다") + void anUnavailableClaimIsARefusalBeforeTheActionRuns() { + // given + FakeStore store = new FakeStore(); + store.claim = new IdempotencyClaimOutcome.Unavailable(); + AtomicBoolean ran = new AtomicBoolean(); + + // when + Throwable thrown = + catchThrowable( + () -> + executor(store) + .execute( + SCOPE, + FINGERPRINT, + ATTEMPT, + () -> { + ran.set(true); + return new IdempotentAction.Outcome.Success<>("created"); + }, + codec())); + + // then + assertThat(errorCodeOf(thrown)).isEqualTo("IDEMPOTENCY_UNAVAILABLE"); + assertThat(ran).isFalse(); + } + + /** + * The action already ran when completion fails, so "refused before the action" would be false. + * Effects in the action's own transaction roll back with the exception; anything else it did has + * happened. The client must hear "outcome unknown" and resend with the same key, never "not + * applied" (re-review 2026-11-03 decision, RR-02-03). + */ + @Test + @DisplayName( + "action 뒤 complete 에서 저장소가 응답하지 않으면 실행 전 거부가 아니라 결과 불명(IDEMPOTENCY_COMPLETION_UNKNOWN)이다") + void anUnavailableCompletionAfterTheActionRanIsAnUnknownOutcome() { + // given + FakeStore store = startedStore(); + store.complete = IdempotencyCompleteOutcome.UNAVAILABLE; + AtomicBoolean ran = new AtomicBoolean(); + + // when + Throwable thrown = + catchThrowable( + () -> + executor(store) + .execute( + SCOPE, + FINGERPRINT, + ATTEMPT, + () -> { + ran.set(true); + return new IdempotentAction.Outcome.Success<>("created"); + }, + codec())); + + // then + assertThat(ran).isTrue(); + assertThat(errorCodeOf(thrown)).isEqualTo("IDEMPOTENCY_COMPLETION_UNKNOWN"); + } + + @Test + @DisplayName("응답을 잃은 complete 를 확인하려는데 저장소가 응답하지 않아도 결과 불명(IDEMPOTENCY_COMPLETION_UNKNOWN)이다") + void anUnavailableInspectionOfALostCompletionIsAnUnknownOutcome() { + // given + FakeStore store = startedStore(); + store.complete = IdempotencyCompleteOutcome.INDETERMINATE; + store.inspectionAfterComplete = + IdempotencyInspection.outcome(IdempotencyInspectionOutcome.UNAVAILABLE); + + // when + Throwable thrown = + catchThrowable( + () -> + executor(store) + .execute( + SCOPE, + FINGERPRINT, + ATTEMPT, + () -> new IdempotentAction.Outcome.Success<>("created"), + codec())); + + // then + assertThat(errorCodeOf(thrown)).isEqualTo("IDEMPOTENCY_COMPLETION_UNKNOWN"); + } + @Test @DisplayName("a replayed response that disagrees with this caller's is a recovery, not a result") void aConflictingReplayIsNotReturned() { @@ -282,6 +371,11 @@ class IdempotencyExecutorTest { return store; } + private static String errorCodeOf(Throwable thrown) { + assertThat(thrown).isInstanceOf(ApiErrorCarrier.class); + return ((ApiErrorCarrier) thrown).errorCode().code(); + } + private static IdempotencyExecutor executor(FakeStore store) { return new IdempotencyExecutor( store, Duration.ofSeconds(30), Duration.ofHours(1), Duration.ofHours(1), "json-v2", 1); diff --git a/src/application-core/src/test/java/dev/caskeleton/application/idempotency/IdempotencyFailureCodeTest.java b/src/application-core/src/test/java/dev/caskeleton/application/idempotency/IdempotencyFailureCodeTest.java index 951216b0..d778f7b5 100644 --- a/src/application-core/src/test/java/dev/caskeleton/application/idempotency/IdempotencyFailureCodeTest.java +++ b/src/application-core/src/test/java/dev/caskeleton/application/idempotency/IdempotencyFailureCodeTest.java @@ -21,6 +21,9 @@ class IdempotencyFailureCodeTest { OperationalError.IDEMPOTENT_REQUEST_MISMATCH), Arguments.of( new IdempotencyUnavailableException(), OperationalError.IDEMPOTENCY_UNAVAILABLE), + Arguments.of( + new IdempotencyCompletionUnknownException(), + OperationalError.IDEMPOTENCY_COMPLETION_UNKNOWN), Arguments.of( new IdempotencyRecoveryRequiredException("outcome unknown"), OperationalError.IDEMPOTENCY_RECOVERY_REQUIRED)); diff --git a/src/application-core/src/test/java/dev/caskeleton/application/messaging/contract/ApplicationPayloadSchemaReleaseTest.java b/src/application-core/src/test/java/dev/caskeleton/application/messaging/contract/ApplicationPayloadSchemaReleaseTest.java new file mode 100644 index 00000000..7703a689 --- /dev/null +++ b/src/application-core/src/test/java/dev/caskeleton/application/messaging/contract/ApplicationPayloadSchemaReleaseTest.java @@ -0,0 +1,18 @@ +package dev.caskeleton.application.messaging.contract; + +import dev.caskeleton.application.messaging.contract.testkit.ReleasedPayloadSchemaContract; +import java.nio.file.Path; + +/** The payload schemas application-core publishes ({@code durable-operation.submitted}). */ +class ApplicationPayloadSchemaReleaseTest extends ReleasedPayloadSchemaContract { + + @Override + protected Path schemaRoot() { + return Path.of("src/main/resources/contracts/messaging"); + } + + @Override + protected Path manifest() { + return Path.of("src/test/resources/contracts/released-payload-schemas.sha256"); + } +} diff --git a/src/application-core/src/test/java/dev/caskeleton/application/messaging/event/IntegrationEventDraftTest.java b/src/application-core/src/test/java/dev/caskeleton/application/messaging/event/IntegrationEventDraftTest.java index 96155e73..ca0e5d39 100644 --- a/src/application-core/src/test/java/dev/caskeleton/application/messaging/event/IntegrationEventDraftTest.java +++ b/src/application-core/src/test/java/dev/caskeleton/application/messaging/event/IntegrationEventDraftTest.java @@ -2,16 +2,83 @@ package dev.caskeleton.application.messaging.event; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.application.messaging.contract.ContractId; import dev.caskeleton.application.messaging.contract.IntegrationPayload; import dev.caskeleton.application.messaging.contract.LogicalDestinationId; import java.time.Instant; import java.util.Optional; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; class IntegrationEventDraftTest { + /** A canonical lower-case UUIDv7: the only spelling the messaging platform carries. */ + private static final String EVENT_ID = "01994e11-4d88-7000-8000-0000000000e1"; + + private static final String CAUSATION_ID = "01994e11-4d88-7000-8000-0000000000c1"; + + /** + * The bridge sends an event under its id as the platform's {@code MessageId}, a canonical + * lower-case UUIDv7, and refuses any other — after the business transaction committed, as a DEAD + * row that blocks the aggregate. A draft is where such an id is refused first, in the writer's + * transaction (review 06-06, re-review 2026-11-03 RR-05-04). + */ + @ParameterizedTest + @ValueSource( + strings = { + "event-1", + "01994E11-4D88-7000-8000-0000000000E1", + "6f1c2a3b-4d5e-4f60-8a7b-9c0d1e2f3a4b", + "1-2-3-4-5" + }) + @DisplayName("정규 소문자 UUIDv7이 아닌 event id로는 draft를 만들지 않는다") + void aDraftRefusesAnEventIdThatIsNotACanonicalUuidV7(String eventId) { + // given + EventId id = new EventId(eventId); + + // when + Throwable thrown = catchThrowable(() -> draftWithIds(id, Optional.empty())); + + // then + assertThat(thrown) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("eventId must be a canonical lower-case UUIDv7"); + } + + @Test + @DisplayName("정규 소문자 UUIDv7이 아닌 causation id로는 draft를 만들지 않는다") + void aDraftRefusesACausationIdThatIsNotACanonicalUuidV7() { + // given + Optional causation = Optional.of("cause-1"); + + // when + Throwable thrown = catchThrowable(() -> draftWithIds(new EventId(EVENT_ID), causation)); + + // then + assertThat(thrown) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("causationId must be a canonical lower-case UUIDv7"); + } + + private static IntegrationEventDraft draftWithIds( + EventId eventId, Optional causationId) { + return new IntegrationEventDraft<>( + eventId, + new ContractId("portfolio.worklog.reserved"), + 1, + new LogicalDestinationId("portfolio-domain-events"), + new AggregateIdentity("tenant-a", "worklog", "worklog-42"), + new AggregateOrder(17, 0), + Instant.parse("2026-07-28T05:10:30.123Z"), + "corr-1", + causationId, + new TestPayload("worklog-42")); + } + @Test void eventIdAcceptsOnlyBoundedCanonicalUsAscii() { assertThat(new EventId("A0:event.id-1").value()).isEqualTo("A0:event.id-1"); @@ -53,14 +120,14 @@ class IntegrationEventDraftTest { @Test void draftRequiresPositiveVersionTimeCanonicalCorrelationAndTypedPayload() { TestPayload payload = new TestPayload("worklog-42"); - IntegrationEventDraft draft = draft(payload, Optional.of("cause-1")); + IntegrationEventDraft draft = draft(payload, Optional.of(CAUSATION_ID)); assertThat(draft.featurePayload()).isSameAs(payload); assertThat(draft.occurredAt()).isEqualTo(Instant.parse("2026-07-28T05:10:30.123Z")); assertThatThrownBy( () -> new IntegrationEventDraft<>( - new EventId("event-1"), + new EventId(EVENT_ID), new ContractId("portfolio.worklog.reserved"), 0, new LogicalDestinationId("portfolio-domain-events"), @@ -74,7 +141,7 @@ class IntegrationEventDraftTest { assertThatThrownBy( () -> new IntegrationEventDraft<>( - new EventId("event-1"), + new EventId(EVENT_ID), new ContractId("portfolio.worklog.reserved"), 1, new LogicalDestinationId("portfolio-domain-events"), @@ -103,7 +170,7 @@ class IntegrationEventDraftTest { private static IntegrationEventDraft draft( TestPayload payload, Optional causationId) { return new IntegrationEventDraft<>( - new EventId("event-1"), + new EventId(EVENT_ID), new ContractId("portfolio.worklog.reserved"), 1, new LogicalDestinationId("portfolio-domain-events"), diff --git a/src/application-core/src/test/java/dev/caskeleton/application/messaging/event/IntegrationEventDraftsTest.java b/src/application-core/src/test/java/dev/caskeleton/application/messaging/event/IntegrationEventDraftsTest.java index 2e73b071..adedd81d 100644 --- a/src/application-core/src/test/java/dev/caskeleton/application/messaging/event/IntegrationEventDraftsTest.java +++ b/src/application-core/src/test/java/dev/caskeleton/application/messaging/event/IntegrationEventDraftsTest.java @@ -13,7 +13,7 @@ import org.junit.jupiter.params.provider.ValueSource; class IntegrationEventDraftsTest { - private static final EventId EVENT_ID = new EventId("event-1"); + private static final EventId EVENT_ID = new EventId("01994e11-4d88-7000-8000-0000000000e1"); private static final Instant OCCURRED_AT = Instant.parse("2026-11-01T00:00:00.000001Z"); private static final AggregateIdentity AGGREGATE = new AggregateIdentity(IntegrationEventDrafts.SINGLE_TENANT_SCOPE, "kit", "kit-1"); diff --git a/src/application-core/src/test/java/dev/caskeleton/application/messaging/event/IntegrationIdentityTest.java b/src/application-core/src/test/java/dev/caskeleton/application/messaging/event/IntegrationIdentityTest.java index e044c662..39e42db9 100644 --- a/src/application-core/src/test/java/dev/caskeleton/application/messaging/event/IntegrationIdentityTest.java +++ b/src/application-core/src/test/java/dev/caskeleton/application/messaging/event/IntegrationIdentityTest.java @@ -74,7 +74,7 @@ class IntegrationIdentityTest { private static IntegrationEventDraft draftWithCorrelation(String correlationId) { return new IntegrationEventDraft<>( - new EventId("event-1"), + new EventId("01994e11-4d88-7000-8000-0000000000e1"), new ContractId("portfolio.worklog.reserved"), 1, new LogicalDestinationId("portfolio-domain-events"), diff --git a/src/application-core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java b/src/application-core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java index d26132f1..ac469e2d 100644 --- a/src/application-core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java +++ b/src/application-core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import dev.caskeleton.application.idempotency.RequestFingerprint; +import dev.caskeleton.application.messaging.event.IntegrationIdentity; import dev.caskeleton.application.messaging.event.ValidatedIntegrationEvent; import dev.caskeleton.application.messaging.event.testkit.FakeIntegrationEventEncoder; import dev.caskeleton.application.operation.testkit.InMemoryDurableOperationStore; @@ -18,6 +19,8 @@ import java.time.ZoneOffset; import java.util.List; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * The two writes that must not come apart. @@ -30,7 +33,11 @@ import org.junit.jupiter.api.Test; class SubmitDurableOperationUseCaseTest { private static final Instant NOW = Instant.parse("2026-08-25T10:00:00Z"); - private static final DurableOperationId ID = new DurableOperationId("op-1"); + + /** An id the HTTP adapter mints: a canonical lower-case UUIDv7, used as the event id itself. */ + private static final String OPERATION_UUID = "01994e11-4d88-7000-8000-0000000000a1"; + + private static final DurableOperationId ID = new DurableOperationId(OPERATION_UUID); // Both stores join the in-memory port's transactions and undo their writes when one rolls back, // as the database does. A port that simply ran the action would let rollbackLeavesNothing pass @@ -151,8 +158,8 @@ class SubmitDurableOperationUseCaseTest { e -> e.contractId(), e -> e.logicalDestinationId()) .containsExactly( - "op-1", - "op-1", + OPERATION_UUID, + OPERATION_UUID, DurableOperationSubmittedContract.CONTRACT_ID, DurableOperationSubmittedContract.DESTINATION); } @@ -166,21 +173,46 @@ class SubmitDurableOperationUseCaseTest { // The worker reads what to run from the submission it claims; the request never travels. assertThat(encoder.drafts()) .extracting(draft -> draft.featurePayload()) - .containsExactly(new DurableOperationSubmitted("op-1", "reports.generate")); + .containsExactly(new DurableOperationSubmitted(OPERATION_UUID, "reports.generate")); + } + + /** + * An event id is the platform's message id, a canonical lower-case UUIDv7, or the bridge parks + * the event after the operation committed. An operation id that is anything else — a client's own + * id, {@code ~} included — becomes a UUIDv7 derived from it and the submission time: the same + * operation always gets the same event id (review 06-06, re-review 2026-11-03 RR-05-04). + */ + @ParameterizedTest + @ValueSource(strings = {"~report", "op-1", "01994E11-4D88-7000-8000-0000000000A1"}) + @DisplayName("UUIDv7이 아닌 operation id는 그 id에서 만든 같은 UUIDv7 event id가 된다") + void anOperationIdThatIsNotAUuidV7BecomesAStableUuidV7DerivedFromIt(String operationId) { + // given + DurableOperationId id = new DurableOperationId(operationId); + + // when + String identity = SubmitDurableOperationUseCase.canonicalIdentity(id, NOW); + + // then + assertThat(List.of(identity, SubmitDurableOperationUseCase.canonicalIdentity(id, NOW))) + .allMatch(IntegrationIdentity::isCanonicalUuidV7) + .containsOnly(identity) + .doesNotContain(operationId); } @Test - void anOperationIdOutsideTheEventIdGrammarBecomesAStableDigestIdentity() { + @DisplayName("UUIDv7이 아닌 operation id로 받은 작업도 outbox에 발행할 수 있는 event로 남는다") + void aSubmissionUnderAClientChosenIdStillAppendsAPublishableEvent() { // given - DurableOperationId tilde = new DurableOperationId("~report"); + DurableOperationId clientChosen = new DurableOperationId("~report"); // when - String identity = SubmitDurableOperationUseCase.canonicalIdentity(tilde); + useCase.handle(command(clientChosen, "c".repeat(64))); // then - assertThat(List.of(identity, SubmitDurableOperationUseCase.canonicalIdentity(tilde))) - .allMatch(value -> value.matches("operation-[0-9a-f]{64}")) - .containsOnly(identity); + assertThat(outbox.appended()) + .singleElement() + .extracting(event -> event.eventId().value()) + .matches(IntegrationIdentity::isCanonicalUuidV7); } /** The in-memory outbox, failing its next append when told to. */ diff --git a/src/application-core/src/test/java/dev/caskeleton/application/outbox/OutboxPublishOutcomeTest.java b/src/application-core/src/test/java/dev/caskeleton/application/outbox/OutboxPublishOutcomeTest.java index a64468b8..4b486185 100644 --- a/src/application-core/src/test/java/dev/caskeleton/application/outbox/OutboxPublishOutcomeTest.java +++ b/src/application-core/src/test/java/dev/caskeleton/application/outbox/OutboxPublishOutcomeTest.java @@ -72,6 +72,6 @@ class OutboxPublishOutcomeTest { assertThat(legacy.publishForOutcome(null)) .as("'returned without throwing' has always meant confirmed for such an adapter") - .isEqualTo(OutboxPublishOutcome.CONFIRMED); + .isEqualTo(OutboxPublishAttempt.confirmed()); } } diff --git a/src/application-core/src/test/java/dev/caskeleton/application/outbox/OutboxRelayFailureReportTest.java b/src/application-core/src/test/java/dev/caskeleton/application/outbox/OutboxRelayFailureReportTest.java index 008f9a52..f01055af 100644 --- a/src/application-core/src/test/java/dev/caskeleton/application/outbox/OutboxRelayFailureReportTest.java +++ b/src/application-core/src/test/java/dev/caskeleton/application/outbox/OutboxRelayFailureReportTest.java @@ -7,12 +7,14 @@ import dev.caskeleton.shared.error.OperationalError; import java.lang.reflect.RecordComponent; import java.time.Instant; import java.util.Arrays; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; class OutboxRelayFailureReportTest { private static final Instant NEXT_ATTEMPT_AT = Instant.parse("2026-07-25T01:02:03Z"); private static final RuntimeException CAUSE = new RuntimeException("broker unavailable"); + private static final String CODE = "KAFKA_TIMEOUTEXCEPTION"; @Test void recordComponentsAreAnExactSafeAllowlist() { @@ -29,14 +31,16 @@ class OutboxRelayFailureReportTest { "nextAttemptAt", // Not "cause": a Throwable component is a Throwable in the operational JSON, and the // allowlist that guards every other field had no say over its message or stack. - "causeType"); + "causeType", + // A bounded code, never a message (re-review 2026-11-03 RR-05-03). + "failureCode"); } @Test void retryableFailureFactoryCreatesPublishFailedReport() { OutboxRelayFailureReport report = OutboxRelayFailureReport.retryableFailure( - "evt-1", "WorkLogReserved", "agg-1", "corr-1", 2, NEXT_ATTEMPT_AT, CAUSE); + "evt-1", "WorkLogReserved", "agg-1", "corr-1", 2, NEXT_ATTEMPT_AT, CAUSE, CODE); assertThat(report.code()).isEqualTo(OperationalError.OUTBOX_PUBLISH_FAILED); assertThat(report.eventId()).isEqualTo("evt-1"); @@ -54,7 +58,7 @@ class OutboxRelayFailureReportTest { void deadLetterFactoryCreatesTerminalReportWithoutRetryTime() { OutboxRelayFailureReport report = OutboxRelayFailureReport.deadLetter( - "evt-1", "WorkLogReserved", "agg-1", "corr-1", 3, CAUSE); + "evt-1", "WorkLogReserved", "agg-1", "corr-1", 3, CAUSE, CODE); assertThat(report.code()).isEqualTo(OperationalError.OUTBOX_DEAD_LETTER); assertThat(report.nextAttemptAt()).isNull(); @@ -75,7 +79,8 @@ class OutboxRelayFailureReportTest { "corr-1", 1, null, - CAUSE.getClass().getName())) + CAUSE.getClass().getName(), + CODE)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("code"); } @@ -83,19 +88,26 @@ class OutboxRelayFailureReportTest { @Test void rejectsBlankIdentifiersAndEventType() { assertThatThrownBy( - () -> OutboxRelayFailureReport.deadLetter(" ", "Event", "agg-1", "corr-1", 1, CAUSE)) + () -> + OutboxRelayFailureReport.deadLetter( + " ", "Event", "agg-1", "corr-1", 1, CAUSE, CODE)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("eventId"); assertThatThrownBy( - () -> OutboxRelayFailureReport.deadLetter("evt-1", "", "agg-1", "corr-1", 1, CAUSE)) + () -> + OutboxRelayFailureReport.deadLetter("evt-1", "", "agg-1", "corr-1", 1, CAUSE, CODE)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("eventType"); assertThatThrownBy( - () -> OutboxRelayFailureReport.deadLetter("evt-1", "Event", "\t", "corr-1", 1, CAUSE)) + () -> + OutboxRelayFailureReport.deadLetter( + "evt-1", "Event", "\t", "corr-1", 1, CAUSE, CODE)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("aggregateId"); assertThatThrownBy( - () -> OutboxRelayFailureReport.deadLetter("evt-1", "Event", "agg-1", "\n", 1, CAUSE)) + () -> + OutboxRelayFailureReport.deadLetter( + "evt-1", "Event", "agg-1", "\n", 1, CAUSE, CODE)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("correlationId"); } @@ -104,7 +116,8 @@ class OutboxRelayFailureReportTest { void rejectsAttemptCountBelowOne() { assertThatThrownBy( () -> - OutboxRelayFailureReport.deadLetter("evt-1", "Event", "agg-1", "corr-1", 0, CAUSE)) + OutboxRelayFailureReport.deadLetter( + "evt-1", "Event", "agg-1", "corr-1", 0, CAUSE, CODE)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("attemptCount"); } @@ -121,7 +134,8 @@ class OutboxRelayFailureReportTest { "corr-1", 1, null, - CAUSE.getClass().getName())) + CAUSE.getClass().getName(), + CODE)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("nextAttemptAt"); } @@ -138,15 +152,29 @@ class OutboxRelayFailureReportTest { "corr-1", 1, NEXT_ATTEMPT_AT, - CAUSE.getClass().getName())) + CAUSE.getClass().getName(), + CODE)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("nextAttemptAt"); } + @Test + @DisplayName("원인 코드 자리에 문장을 넣으면 보고를 만들지 않는다") + void aSentenceIsRefusedAsAFailureCode() { + assertThatThrownBy( + () -> + OutboxRelayFailureReport.deadLetter( + "evt-1", "Event", "agg-1", "corr-1", 1, CAUSE, "connect to 10.0.0.7 failed")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("failureCode"); + } + @Test void causeIsRequired() { assertThatThrownBy( - () -> OutboxRelayFailureReport.deadLetter("evt-1", "Event", "agg-1", "corr-1", 1, null)) + () -> + OutboxRelayFailureReport.deadLetter( + "evt-1", "Event", "agg-1", "corr-1", 1, null, CODE)) .isInstanceOf(NullPointerException.class) .hasMessageContaining("cause"); } diff --git a/src/application-core/src/test/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCaseTest.java b/src/application-core/src/test/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCaseTest.java index 5c664c0d..e6d7d2a0 100644 --- a/src/application-core/src/test/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCaseTest.java +++ b/src/application-core/src/test/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCaseTest.java @@ -131,7 +131,8 @@ class PublishPendingOutboxEventsUseCaseTest { "correlation-evt-fail", 1, nextAttemptAt, - brokerDown)); + brokerDown, + "RuntimeException")); } @Test @@ -155,7 +156,8 @@ class PublishPendingOutboxEventsUseCaseTest { "agg-1", "correlation-evt-dead", 3, - brokerDown)); + brokerDown, + "RuntimeException")); } // ---- failure NOT swallowed ---- @@ -476,15 +478,15 @@ class PublishPendingOutboxEventsUseCaseTest { } @Override - public OutboxCompletion markFailed(OutboxLease lease, Instant retryAt) { + public OutboxCompletion markFailed(OutboxLease lease, Instant retryAt, String failureCode) { markFailedCalls++; - return super.markFailed(lease, retryAt); + return super.markFailed(lease, retryAt, failureCode); } @Override - public OutboxCompletion markDead(OutboxLease lease) { + public OutboxCompletion markDead(OutboxLease lease, String failureCode) { markDeadCalls++; - return super.markDead(lease); + return super.markDead(lease, failureCode); } OutboxEventStatus currentStatus() { @@ -519,19 +521,20 @@ class PublishPendingOutboxEventsUseCaseTest { } @Override - public OutboxCompletion markFailed(OutboxLease lease, Instant nextAttemptAt) { + public OutboxCompletion markFailed( + OutboxLease lease, Instant nextAttemptAt, String failureCode) { if (markFailedException != null) { throw markFailedException; } - return super.markFailed(lease, nextAttemptAt); + return super.markFailed(lease, nextAttemptAt, failureCode); } @Override - public OutboxCompletion markDead(OutboxLease lease) { + public OutboxCompletion markDead(OutboxLease lease, String failureCode) { if (markDeadException != null) { throw markDeadException; } - return super.markDead(lease); + return super.markDead(lease, failureCode); } } diff --git a/src/application-core/src/test/java/dev/caskeleton/application/outbox/TransientPublishFailureRelayTest.java b/src/application-core/src/test/java/dev/caskeleton/application/outbox/TransientPublishFailureRelayTest.java index 04fbc520..40104494 100644 --- a/src/application-core/src/test/java/dev/caskeleton/application/outbox/TransientPublishFailureRelayTest.java +++ b/src/application-core/src/test/java/dev/caskeleton/application/outbox/TransientPublishFailureRelayTest.java @@ -85,6 +85,58 @@ class TransientPublishFailureRelayTest { .isNotNull(); } + @Test + @DisplayName("다시 시도할 거부는 publisher가 붙인 원인 코드를 행과 실패 보고에 남긴다") + void aRetryableRefusalLeavesThePublishersFailureCodeOnTheRowAndInTheReport() { + // given: the broker refused this producer's credentials + store.addClaimable(OutboxEvents.claimed("evt-credential", START.minusSeconds(1), 1)); + port.answering( + OutboxPublishAttempt.failed( + OutboxPublishOutcome.REJECTED_RETRYABLE, "KAFKA_AUTHENTICATIONEXCEPTION")); + + // when + relay.handle(PublishPendingOutboxEventsCommand.INSTANCE); + + // then: what the outcome cannot say — a credential, not a shutdown — is on both + assertThat( + List.of( + store.lastFailureCode("evt-credential"), + reports.getFirst().failureCode(), + reports.getFirst().causeType())) + .containsExactly( + "KAFKA_AUTHENTICATIONEXCEPTION", + "KAFKA_AUTHENTICATIONEXCEPTION", + OutboxPublishRefusedException.class.getName()); + } + + @Test + @DisplayName("원인 코드 없이 응답이 없던 발행은 결과 이름을 원인 코드로 남긴다") + void anUnansweredPublishWithoutACodeIsRecordedUnderItsOutcomesName() { + // given + store.addClaimable(OutboxEvents.claimed("evt-silent", START.minusSeconds(1), 1)); + port.answering(OutboxPublishAttempt.of(OutboxPublishOutcome.AMBIGUOUS)); + + // when + relay.handle(PublishPendingOutboxEventsCommand.INSTANCE); + + // then + assertThat(store.lastFailureCode("evt-silent")).isEqualTo("AMBIGUOUS"); + } + + @Test + @DisplayName("분류되지 않고 던져진 실패는 예외 타입 이름을 원인 코드로 남긴다") + void anUnclassifiedThrownFailureIsRecordedUnderItsTypesName() { + // given + store.addClaimable(OutboxEvents.claimed("evt-thrown", START.minusSeconds(1), 1)); + port.throwing(new IllegalStateException("driver said: host=10.0.0.7 password=hunter2")); + + // when + relay.handle(PublishPendingOutboxEventsCommand.INSTANCE); + + // then: the type, never the message + assertThat(store.lastFailureCode("evt-thrown")).isEqualTo("IllegalStateException"); + } + @Test @DisplayName("일시적 실패는 시도 예산을 다 쓴 행도 DEAD로 보내지 않는다") void aTransientFailureNeverDeadLettersEvenPastTheAttemptBudget() { diff --git a/src/application-core/src/test/java/dev/caskeleton/application/outbox/testkit/FakeOutboxMessagePublisherContractTest.java b/src/application-core/src/test/java/dev/caskeleton/application/outbox/testkit/FakeOutboxMessagePublisherContractTest.java index 96ff3a23..f9f82d6b 100644 --- a/src/application-core/src/test/java/dev/caskeleton/application/outbox/testkit/FakeOutboxMessagePublisherContractTest.java +++ b/src/application-core/src/test/java/dev/caskeleton/application/outbox/testkit/FakeOutboxMessagePublisherContractTest.java @@ -37,7 +37,7 @@ class FakeOutboxMessagePublisherContractTest extends OutboxMessagePublishContrac .answering(CLAIMED_EVENT_ID, OutboxPublishOutcome.REJECTED_BEFORE_SEND); // when - OutboxPublishOutcome outcome = publisher.publishForOutcome(claimedRow()); + OutboxPublishOutcome outcome = publisher.publishForOutcome(claimedRow()).outcome(); // then assertThat(outcome).isEqualTo(OutboxPublishOutcome.REJECTED_BEFORE_SEND); @@ -68,7 +68,7 @@ class FakeOutboxMessagePublisherContractTest extends OutboxMessagePublishContrac .answering(OutboxPublishOutcome.REJECTED_RETRYABLE); // when - OutboxPublishOutcome outcome = publisher.publishForOutcome(claimedRow()); + OutboxPublishOutcome outcome = publisher.publishForOutcome(claimedRow()).outcome(); // then assertThat(outcome).isEqualTo(OutboxPublishOutcome.REJECTED_RETRYABLE); diff --git a/src/application-core/src/test/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxAggregateSequenceContractTest.java b/src/application-core/src/test/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxAggregateSequenceContractTest.java index 499a43cb..c8fdde54 100644 --- a/src/application-core/src/test/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxAggregateSequenceContractTest.java +++ b/src/application-core/src/test/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxAggregateSequenceContractTest.java @@ -6,7 +6,9 @@ import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; class InMemoryOutboxAggregateSequenceContractTest extends OutboxAggregateSequenceContract { - private final InMemoryOutboxAggregateSequences sequences = new InMemoryOutboxAggregateSequences(); + private final DirectTransactionPort transactions = new DirectTransactionPort(); + private final InMemoryOutboxAggregateSequences sequences = + new InMemoryOutboxAggregateSequences(transactions); @Override protected OutboxAggregateSequencePort sequences() { @@ -15,6 +17,6 @@ class InMemoryOutboxAggregateSequenceContractTest extends OutboxAggregateSequenc @Override protected TransactionPort transactions() { - return new DirectTransactionPort(); + return transactions; } } diff --git a/src/application-core/src/test/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxStoreLeaseFencingContractTest.java b/src/application-core/src/test/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxStoreLeaseFencingContractTest.java index 49f51433..be1a30cc 100644 --- a/src/application-core/src/test/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxStoreLeaseFencingContractTest.java +++ b/src/application-core/src/test/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxStoreLeaseFencingContractTest.java @@ -29,4 +29,9 @@ class InMemoryOutboxStoreLeaseFencingContractTest extends OutboxLeaseFencingCont protected OutboxEventStatus statusOf(String eventId) { return store.status(eventId); } + + @Override + protected String lastFailureCodeOf(String eventId) { + return store.lastFailureCode(eventId); + } } diff --git a/src/application-core/src/test/resources/contracts/released-payload-schemas.sha256 b/src/application-core/src/test/resources/contracts/released-payload-schemas.sha256 new file mode 100644 index 00000000..b2f1e29a --- /dev/null +++ b/src/application-core/src/test/resources/contracts/released-payload-schemas.sha256 @@ -0,0 +1,4 @@ +# SHA-256 (LF-normalized) of every released payload schema version in src/main/resources/contracts/messaging. +# A released version is never edited: a change goes into a new version (application-core README). +# Add a line when a version is released; change a line only for a version no one can have consumed. +4d2f780bf6c23fa548fc2b0e6ef02302e17fca44a20f21ebf5ba07aaac11a245 durable-operation.submitted/v1.schema.json diff --git a/src/application-core/src/testFixtures/java/dev/caskeleton/application/cache/testkit/CacheRegionContract.java b/src/application-core/src/testFixtures/java/dev/caskeleton/application/cache/testkit/CacheRegionContract.java index 3d03e322..63fbd552 100644 --- a/src/application-core/src/testFixtures/java/dev/caskeleton/application/cache/testkit/CacheRegionContract.java +++ b/src/application-core/src/testFixtures/java/dev/caskeleton/application/cache/testkit/CacheRegionContract.java @@ -179,6 +179,25 @@ public abstract class CacheRegionContract { assertThat(valueOf(region.lookup("k"))).isEqualTo("at-10"); } + /** + * 2^53 + 1 against 2^53: a floating-point parse reads both as the same number, so a provider that + * compares revisions as doubles refuses the newer one (review 2026-11-03 RR-01-06). + */ + @Test + @DisplayName("ONLY_IF_SOURCE_REVISION_NEWER는 2^53보다 큰 revision도 한 자리 차이를 구분한다") + void onlyIfSourceRevisionNewerTellsApartRevisionsAboveTwoToTheFiftyThird() { + // given + CacheRegionPort region = region(); + region.record("k", "at-2^53", upsert("9007199254740992")); + + // when + CacheRecordOutcome outcome = + region.record("k", "at-2^53+1", newerThanStored("9007199254740993")); + + // then + assertThat(outcome).isEqualTo(CacheRecordOutcome.RECORDED); + } + @Test @DisplayName("ONLY_IF_SOURCE_REVISION_NEWER는 비어 있는 키에 기록한다") void onlyIfSourceRevisionNewerRecordsIntoAnEmptyKey() { diff --git a/src/application-core/src/testFixtures/java/dev/caskeleton/application/idempotency/testkit/IdempotencyStoreContract.java b/src/application-core/src/testFixtures/java/dev/caskeleton/application/idempotency/testkit/IdempotencyStoreContract.java index 297ce65f..29bfef2f 100644 --- a/src/application-core/src/testFixtures/java/dev/caskeleton/application/idempotency/testkit/IdempotencyStoreContract.java +++ b/src/application-core/src/testFixtures/java/dev/caskeleton/application/idempotency/testkit/IdempotencyStoreContract.java @@ -22,6 +22,7 @@ import dev.caskeleton.application.idempotency.StoredResponse; import dev.caskeleton.application.transaction.OperationId; import java.time.Duration; import java.util.function.Supplier; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -34,7 +35,10 @@ import org.junit.jupiter.api.Test; * trusts, and those cases stay with each store's own tests. The one timed case is the one a caller * relies on whichever store is composed — {@code renew} really extends the lease (review 05-02: the * Redis store answered a renew as already applied and let the lease run out). A store reaches time - * through {@link #elapse(Duration)}; the in-memory fake has no expiry and leaves it a no-op. + * through {@link #elapse(Duration)}. A store that cannot move its clock — the in-memory fake, which + * models no expiry, and the Redis store over the command fake — keeps the default, which aborts the + * timed case so the report shows it skipped instead of passing a case that never ran its clock + * (review 2026-11-03 RR-01-06). * *

    Every call goes through {@link #inTransaction(Supplier)}, the way the store's callers make it: * the PostgreSQL store refuses to run outside an application-owned write transaction. @@ -81,12 +85,16 @@ public abstract class IdempotencyStoreContract { } /** - * Lets {@code duration} pass on the clock the store trusts. A store without lease expiry keeps - * the default no-op. + * Lets {@code duration} pass on the clock the store trusts. The default aborts the calling test: + * a store without a clock the contract can move cannot run a timed case, and a no-op here would + * let that case pass without time ever passing. * * @param duration how long to let pass */ - protected void elapse(Duration duration) {} + protected void elapse(Duration duration) { + Assumptions.abort( + "this store has no clock the contract can move, so the timed case does not run against it"); + } private IdempotencyStorePort subject() { return new EachCallInTransaction(store()); @@ -360,6 +368,30 @@ public abstract class IdempotencyStoreContract { assertThat(other).isInstanceOf(IdempotencyClaimOutcome.InProgress.class); } + @Test + @DisplayName("renew한 처리 lease는 claim이 정한 replay TTL보다 길어도 그동안 기록이 사라져 다른 요청에 넘어가지 않는다") + void aLeaseRenewedPastTheReplayTtlKeepsTheRecord() { + // given + IdempotencyStorePort store = subject(); + Duration lease = processingLease(); + IdempotencyClaimRequest shortReplay = + new IdempotencyClaimRequest( + SCOPE, FINGERPRINT, store.newClaimAttempt(OPERATION), lease, lease, "json-v2", 1); + IdempotencyOwner claimed = + ((IdempotencyClaimOutcome.Acquired) store.claim(shortReplay)).owner(); + IdempotencyOwner executing = + store.markExecutionStarted(claimed, OPERATION).owner().orElseThrow(); + store.renew(executing, lease.multipliedBy(3), OPERATION); + elapse(lease.multipliedBy(2)); + + // when + IdempotencyClaimOutcome other = + store.claim(claim(store.newClaimAttempt(OTHER_OPERATION), FINGERPRINT, lease)); + + // then + assertThat(other).isInstanceOf(IdempotencyClaimOutcome.InProgress.class); + } + /** Routes every call of the store through {@link #inTransaction(Supplier)}. */ private final class EachCallInTransaction implements IdempotencyStorePort { diff --git a/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/contract/testkit/ReleasedPayloadSchemaContract.java b/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/contract/testkit/ReleasedPayloadSchemaContract.java new file mode 100644 index 00000000..fc35a68e --- /dev/null +++ b/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/contract/testkit/ReleasedPayloadSchemaContract.java @@ -0,0 +1,115 @@ +package dev.caskeleton.application.messaging.contract.testkit; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.shared.digest.Sha256; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.TreeMap; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * A released payload schema version is never edited (review 2026-11-01 06-10). + * + *

    An event's payload schema is {@code contracts/messaging//v.schema.json}. Once a + * version has been published, consumers have been built against it and the outbox and the broker + * hold events written under it, so changing its bytes — a field removed, a type changed — breaks + * them in an order that depends on who deploys first. The encoder checks only that the declared + * hash is the file's hash, and both move together in one edit. The rule the messaging design sets + * (strict versioning: "같은 version file의 checksum 변경은 CI 실패") had nothing enforcing it. + * + *

    The leaf that owns schemas pins each released version's SHA-256 in a manifest, as the + * persistence leaves pin applied migrations: an edited version fails, and a new version fails until + * it is pinned, which is the moment it is released. A change goes into a new version, and the + * version procedure (application-core README "payload version 올리기") is how consumers get it. + * + *

    What this deliberately does not do is diff a version against the one before. A new version may + * change anything — that is why it is a new version, read by consumers that support it before a + * producer emits it — and the design does not claim compatibility from a JSON Schema diff. + */ +public abstract class ReleasedPayloadSchemaContract { + + private static final Pattern SCHEMA_FILE = Pattern.compile("[^/]+/v[1-9][0-9]*\\.schema\\.json"); + + /** The directory holding {@code /v.schema.json}, relative to the leaf. */ + protected abstract Path schemaRoot(); + + /** + * The manifest of released versions: {@code /v.schema.json} per line, + * {@code #} comments allowed. + */ + protected abstract Path manifest(); + + @Test + @DisplayName("이미 출시한 payload schema 버전 파일은 바이트가 바뀌지 않는다") + void aReleasedPayloadSchemaVersionKeepsItsBytes() { + // given + Map released = released(); + + // when + Map current = current(); + + // then + assertThat(current) + .as( + "a released payload schema changed; put the change in a new version (%s)", + "application-core README, payload version 올리기") + .containsAllEntriesOf(released); + } + + @Test + @DisplayName("모든 payload schema 버전 파일은 출시 목록에 고정돼 있다") + void everyPayloadSchemaVersionIsPinnedInTheManifest() { + // given + Map released = released(); + + // when + Map current = current(); + + // then + assertThat(released.keySet()) + .as("pin a new payload schema version in %s when it is released", manifest()) + .containsAll(current.keySet()); + } + + private Map released() { + Map pinned = new TreeMap<>(); + try { + for (String line : Files.readAllLines(manifest(), StandardCharsets.UTF_8)) { + String trimmed = line.strip(); + if (trimmed.isEmpty() || trimmed.startsWith("#")) { + continue; + } + String[] fields = trimmed.split("\\s+", 2); + pinned.put(fields[1], fields[0]); + } + } catch (IOException exception) { + throw new UncheckedIOException(exception); + } + return pinned; + } + + /** Every version file under the root, by relative path, with the SHA-256 of its LF bytes. */ + private Map current() { + Map hashes = new TreeMap<>(); + try (Stream files = Files.walk(schemaRoot())) { + for (Path file : files.filter(Files::isRegularFile).toList()) { + String relative = schemaRoot().relativize(file).toString().replace('\\', '/'); + if (SCHEMA_FILE.matcher(relative).matches()) { + String content = Files.readString(file, StandardCharsets.UTF_8).replace("\r\n", "\n"); + hashes.put(relative, Sha256.ofUtf8(content).hex()); + } + } + } catch (IOException exception) { + throw new UncheckedIOException(exception); + } + return hashes; + } +} diff --git a/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java b/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java index 4d2a5ad4..b7fe8a5c 100644 --- a/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java +++ b/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java @@ -2,6 +2,7 @@ package dev.caskeleton.application.messaging.event.testkit; import dev.caskeleton.application.messaging.event.IntegrationEventPublishPort; import dev.caskeleton.application.messaging.event.ValidatedIntegrationEvent; +import dev.caskeleton.application.outbox.OutboxPublishAttempt; import dev.caskeleton.application.outbox.OutboxPublishOutcome; import java.util.ArrayList; import java.util.List; @@ -17,21 +18,32 @@ import java.util.concurrent.CompletionStage; public final class FakeIntegrationEventPublisher implements IntegrationEventPublishPort { private final List published = new ArrayList<>(); - private OutboxPublishOutcome answer = OutboxPublishOutcome.CONFIRMED; + private OutboxPublishAttempt answer = OutboxPublishAttempt.confirmed(); /** - * Answers every later publish with {@code outcome}. + * Answers every later publish with {@code outcome}, and for anything but a confirmation the + * outcome's own name as its failure code. * * @param outcome the outcome to report * @return this fake */ public FakeIntegrationEventPublisher answering(OutboxPublishOutcome outcome) { - this.answer = Objects.requireNonNull(outcome, "outcome"); + return answering(attemptOf(outcome)); + } + + /** + * Answers every later publish with {@code attempt}, failure code included. + * + * @param attempt the attempt to report + * @return this fake + */ + public FakeIntegrationEventPublisher answering(OutboxPublishAttempt attempt) { + this.answer = Objects.requireNonNull(attempt, "attempt"); return this; } @Override - public CompletionStage publish(ValidatedIntegrationEvent event) { + public CompletionStage publish(ValidatedIntegrationEvent event) { published.add(Objects.requireNonNull(event, "event must not be null")); return CompletableFuture.completedFuture(answer); } @@ -57,4 +69,17 @@ public final class FakeIntegrationEventPublisher implements IntegrationEventPubl case REFUSES_AFTER_RECEIVING -> OutboxPublishOutcome.REJECTED_AFTER_BROKER; }; } + + /** + * The attempt a publisher that names its failures reports for {@code outcome}: the outcome, and + * unless it is a confirmation, a code — the outcome's name, standing in for a broker's own. + * + * @param outcome the outcome + * @return the attempt + */ + public static OutboxPublishAttempt attemptOf(OutboxPublishOutcome outcome) { + return outcome == OutboxPublishOutcome.CONFIRMED + ? OutboxPublishAttempt.confirmed() + : OutboxPublishAttempt.failed(outcome, outcome.name()); + } } diff --git a/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/IntegrationEventEncoderContract.java b/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/IntegrationEventEncoderContract.java index 2f5a4182..510937b9 100644 --- a/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/IntegrationEventEncoderContract.java +++ b/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/IntegrationEventEncoderContract.java @@ -132,7 +132,7 @@ public abstract class IntegrationEventEncoderContract { LogicalDestinationId destination, IntegrationPayload payload) { ContractId contractId = EncoderContractKitContract.CONTRACT_ID; return new IntegrationEventDraft<>( - new EventId("event-1"), + new EventId("01994e11-4d88-7000-8000-0000000000e1"), contractId, 1, destination, diff --git a/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/IntegrationEventPublishContract.java b/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/IntegrationEventPublishContract.java index 7863777b..a1231f37 100644 --- a/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/IntegrationEventPublishContract.java +++ b/src/application-core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/IntegrationEventPublishContract.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.application.messaging.event.IntegrationEventPublishPort; import dev.caskeleton.application.messaging.event.ValidatedIntegrationEvent; +import dev.caskeleton.application.outbox.OutboxPublishAttempt; import dev.caskeleton.application.outbox.OutboxPublishOutcome; import dev.caskeleton.application.outbox.testkit.OutboxEvents; import java.time.Instant; @@ -15,12 +16,16 @@ import org.junit.jupiter.api.Test; * What every {@link IntegrationEventPublishPort} promises, run by the platform bridge's test and * the fake's test alike (review 2026-11-01 TEST-F-1). * - *

    The relay acts on the outcome alone: it marks a row published, retries it, or parks it. So the + *

    The relay acts on the outcome: it marks a row published, retries it, or parks it. So the * promise is the translation — each thing a broker can do with a message arrives as the one outcome * that leads the relay to the right action. A bridge that reported a refusal of the moment as a * definite rejection parked a committed event and its aggregate behind it (06-01); the bridge's own * test caught it, the fake every use-case test ran on did not have to. * + *

    An attempt that was not confirmed also carries the publisher's code for why. The relay writes + * it on the row, and it is the only thing that tells a refused credential from a closed transport + * once both have been translated into the same outcome (re-review 2026-11-03 RR-05-03). + * *

    The event id is a canonical UUIDv7, the only spelling the platform carries. */ public abstract class IntegrationEventPublishContract { @@ -106,9 +111,22 @@ public abstract class IntegrationEventPublishContract { assertThat(outcome.retryable()).isFalse(); } - /** The contract's event, published and waited for. */ + @Test + @DisplayName("확인되지 않은 발행은 publisher가 붙인 원인 코드를 함께 알린다") + void anUnconfirmedPublishCarriesThePublishersFailureCode() { + // given + IntegrationEventPublishPort port = publisherOverABrokerThat(BrokerAnswer.REFUSES_FOR_NOW); + + // when + OutboxPublishAttempt attempt = port.publish(contractEvent()).toCompletableFuture().join(); + + // then + assertThat(attempt.failureCode()).isPresent(); + } + + /** The contract's event, published and waited for; the outcome is what these scenarios read. */ private static OutboxPublishOutcome publish(IntegrationEventPublishPort port) { - return port.publish(contractEvent()).toCompletableFuture().join(); + return port.publish(contractEvent()).toCompletableFuture().join().outcome(); } /** The event every scenario publishes. */ diff --git a/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/FakeOutboxMessagePublisher.java b/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/FakeOutboxMessagePublisher.java index e3ddc929..64cdac98 100644 --- a/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/FakeOutboxMessagePublisher.java +++ b/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/FakeOutboxMessagePublisher.java @@ -1,8 +1,10 @@ package dev.caskeleton.application.outbox.testkit; +import dev.caskeleton.application.messaging.event.testkit.FakeIntegrationEventPublisher; import dev.caskeleton.application.outbox.ClaimedOutboxEvent; import dev.caskeleton.application.outbox.OutboxMessagePublishPort; import dev.caskeleton.application.outbox.OutboxPublishAmbiguousException; +import dev.caskeleton.application.outbox.OutboxPublishAttempt; import dev.caskeleton.application.outbox.OutboxPublishOutcome; import dev.caskeleton.application.outbox.OutboxPublishRefusedException; import java.util.ArrayList; @@ -19,18 +21,19 @@ import java.util.function.Consumer; * it. * *

    The relay's tests script it four ways: one outcome for every publish ({@link - * #answering(OutboxPublishOutcome)}), one outcome for one event ({@link #answering(String, - * OutboxPublishOutcome)}), an exception the broker client raised instead of answering ({@link - * #throwing}), and something that happens while the broker call is in progress ({@link - * #whenPublishing}) — a clock that moves, a stop request that arrives. A per-event script wins over - * the shared answer, and a thrown exception wins over both. + * #answering(OutboxPublishOutcome)}, or {@link #answering(OutboxPublishAttempt)} with the + * publisher's own failure code; an outcome alone carries its name as the code), one outcome for one + * event ({@link #answering(String, OutboxPublishOutcome)}), an exception the broker client raised + * instead of answering ({@link #throwing}), and something that happens while the broker call is in + * progress ({@link #whenPublishing}) — a clock that moves, a stop request that arrives. A per-event + * script wins over the shared answer, and a thrown exception wins over both. */ public final class FakeOutboxMessagePublisher implements OutboxMessagePublishPort { private final List published = new ArrayList<>(); - private final Map answerPerEvent = new HashMap<>(); + private final Map answerPerEvent = new HashMap<>(); private final Map failurePerEvent = new HashMap<>(); - private OutboxPublishOutcome answer = OutboxPublishOutcome.CONFIRMED; + private OutboxPublishAttempt answer = OutboxPublishAttempt.confirmed(); private RuntimeException failure; private Consumer duringPublish = event -> {}; @@ -42,7 +45,18 @@ public final class FakeOutboxMessagePublisher implements OutboxMessagePublishPor * @return this fake */ public FakeOutboxMessagePublisher answering(OutboxPublishOutcome outcome) { - this.answer = Objects.requireNonNull(outcome, "outcome"); + return answering(FakeIntegrationEventPublisher.attemptOf(Objects.requireNonNull(outcome))); + } + + /** + * Answers every later publish with {@code attempt} — an outcome with the publisher's own failure + * code — and stops throwing a failure set by {@link #throwing(RuntimeException)}. + * + * @param attempt the attempt to report + * @return this fake + */ + public FakeOutboxMessagePublisher answering(OutboxPublishAttempt attempt) { + this.answer = Objects.requireNonNull(attempt, "attempt"); this.failure = null; return this; } @@ -57,7 +71,8 @@ public final class FakeOutboxMessagePublisher implements OutboxMessagePublishPor */ public FakeOutboxMessagePublisher answering(String eventId, OutboxPublishOutcome outcome) { answerPerEvent.put( - Objects.requireNonNull(eventId, "eventId"), Objects.requireNonNull(outcome, "outcome")); + Objects.requireNonNull(eventId, "eventId"), + FakeIntegrationEventPublisher.attemptOf(Objects.requireNonNull(outcome, "outcome"))); return this; } @@ -101,7 +116,7 @@ public final class FakeOutboxMessagePublisher implements OutboxMessagePublishPor @Override public void publish(ClaimedOutboxEvent event) { - OutboxPublishOutcome outcome = publishForOutcome(event); + OutboxPublishOutcome outcome = publishForOutcome(event).outcome(); if (outcome == OutboxPublishOutcome.CONFIRMED) { return; } @@ -112,7 +127,7 @@ public final class FakeOutboxMessagePublisher implements OutboxMessagePublishPor } @Override - public OutboxPublishOutcome publishForOutcome(ClaimedOutboxEvent event) { + public OutboxPublishAttempt publishForOutcome(ClaimedOutboxEvent event) { published.add(Objects.requireNonNull(event, "event must not be null")); // The answer is read after the side effect: a side effect that moves a clock past an outage or // re-scripts the fake decides what this very call reports, as a broker that came back would. diff --git a/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxAggregateSequences.java b/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxAggregateSequences.java index dff55d5b..bf05860a 100644 --- a/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxAggregateSequences.java +++ b/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxAggregateSequences.java @@ -2,16 +2,61 @@ package dev.caskeleton.application.outbox.testkit; import dev.caskeleton.application.messaging.event.AggregateIdentity; import dev.caskeleton.application.outbox.OutboxAggregateSequencePort; +import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; import java.util.HashMap; import java.util.Map; +import java.util.Objects; -/** In-memory {@link OutboxAggregateSequencePort}: one counter per aggregate identity. */ +/** + * In-memory {@link OutboxAggregateSequencePort}: one counter per aggregate identity. + * + *

    Built over a {@link DirectTransactionPort}, it keeps the JPA counter's transaction rules, and + * {@link OutboxAggregateSequenceContract} holds it to them: a number is refused outside a write + * transaction, and a number taken by a transaction that rolls back is handed out again. Built with + * no port, it hands out numbers with or without a transaction and keeps them on rollback — for + * tests that seed events directly and do not model transactions, as {@code new + * InMemoryOutboxStore()} does. + */ public final class InMemoryOutboxAggregateSequences implements OutboxAggregateSequencePort { private final Map last = new HashMap<>(); + private final DirectTransactionPort transactions; + + /** A counter that does not look at transactions. */ + public InMemoryOutboxAggregateSequences() { + this.transactions = null; + } + + /** + * A counter that joins {@code transactions} as the JPA counter joins the caller's. + * + * @param transactions the port the use case under test opens its transactions through + */ + public InMemoryOutboxAggregateSequences(DirectTransactionPort transactions) { + this.transactions = Objects.requireNonNull(transactions, "transactions"); + } @Override public synchronized long nextSequence(AggregateIdentity aggregate) { - return last.merge(aggregate, 1L, Long::sum); + Objects.requireNonNull(aggregate, "aggregate must not be null"); + if (transactions == null) { + return last.merge(aggregate, 1L, Long::sum); + } + if (!transactions.active() || transactions.readOnly()) { + throw new IllegalStateException( + "outbox nextSequence requires an active caller transaction (a write one)"); + } + Long previous = last.get(aggregate); + long next = last.merge(aggregate, 1L, Long::sum); + transactions.onRollback(() -> restore(aggregate, previous)); + return next; + } + + private synchronized void restore(AggregateIdentity aggregate, Long previous) { + if (previous == null) { + last.remove(aggregate); + } else { + last.put(aggregate, previous); + } } } diff --git a/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxStore.java b/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxStore.java index a9442c84..26311043 100644 --- a/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxStore.java +++ b/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/InMemoryOutboxStore.java @@ -6,6 +6,7 @@ import dev.caskeleton.application.outbox.OutboxAppendPort; import dev.caskeleton.application.outbox.OutboxCompletion; import dev.caskeleton.application.outbox.OutboxEventStatus; import dev.caskeleton.application.outbox.OutboxLease; +import dev.caskeleton.application.outbox.OutboxLongRetries; import dev.caskeleton.application.outbox.OutboxStorePort; import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; import java.time.Duration; @@ -126,7 +127,7 @@ public class InMemoryOutboxStore implements OutboxStorePort, OutboxAppendPort { @Override public OutboxCompletion markPublished(OutboxLease lease) { - OutboxCompletion completion = complete(lease, OutboxEventStatus.PUBLISHED, null); + OutboxCompletion completion = complete(lease, OutboxEventStatus.PUBLISHED, null, null); if (completion == OutboxCompletion.APPLIED) { published.add(lease.eventId()); } @@ -134,8 +135,9 @@ public class InMemoryOutboxStore implements OutboxStorePort, OutboxAppendPort { } @Override - public OutboxCompletion markFailed(OutboxLease lease, Instant nextAttemptAt) { - OutboxCompletion completion = complete(lease, OutboxEventStatus.FAILED, nextAttemptAt); + public OutboxCompletion markFailed(OutboxLease lease, Instant nextAttemptAt, String failureCode) { + OutboxCompletion completion = + complete(lease, OutboxEventStatus.FAILED, nextAttemptAt, failureCode); if (completion == OutboxCompletion.APPLIED) { failed.put(lease.eventId(), nextAttemptAt); } @@ -143,8 +145,10 @@ public class InMemoryOutboxStore implements OutboxStorePort, OutboxAppendPort { } @Override - public OutboxCompletion markDeferred(OutboxLease lease, Instant nextAttemptAt) { - OutboxCompletion completion = complete(lease, OutboxEventStatus.FAILED, nextAttemptAt); + public OutboxCompletion markDeferred( + OutboxLease lease, Instant nextAttemptAt, String failureCode) { + OutboxCompletion completion = + complete(lease, OutboxEventStatus.FAILED, nextAttemptAt, failureCode); if (completion == OutboxCompletion.APPLIED) { rows.get(lease.eventId()).deferredAttemptCount++; failed.put(lease.eventId(), nextAttemptAt); @@ -154,7 +158,7 @@ public class InMemoryOutboxStore implements OutboxStorePort, OutboxAppendPort { @Override public OutboxCompletion releaseLease(OutboxLease lease, Instant claimableAt) { - OutboxCompletion completion = complete(lease, OutboxEventStatus.PENDING, claimableAt); + OutboxCompletion completion = complete(lease, OutboxEventStatus.PENDING, claimableAt, null); if (completion == OutboxCompletion.APPLIED) { Row row = rows.get(lease.eventId()); row.attemptCount--; @@ -164,17 +168,20 @@ public class InMemoryOutboxStore implements OutboxStorePort, OutboxAppendPort { } @Override - public OutboxCompletion markDead(OutboxLease lease) { - OutboxCompletion completion = complete(lease, OutboxEventStatus.DEAD, null); + public OutboxCompletion markDead(OutboxLease lease, String failureCode) { + OutboxCompletion completion = complete(lease, OutboxEventStatus.DEAD, null, failureCode); if (completion == OutboxCompletion.APPLIED) { dead.add(lease.eventId()); } return completion; } - /** The same predicate the SQL completion carries: still IN_FLIGHT, same owner, same token. */ + /** + * The same predicate the SQL completion carries: still IN_FLIGHT, same owner, same token. A + * failure code, when given, replaces the row's last one, as the SQL completion's does. + */ private OutboxCompletion complete( - OutboxLease lease, OutboxEventStatus status, Instant nextAttemptAt) { + OutboxLease lease, OutboxEventStatus status, Instant nextAttemptAt, String failureCode) { Row row = rows.get(lease.eventId()); if (row == null || row.status != OutboxEventStatus.IN_FLIGHT @@ -187,6 +194,9 @@ public class InMemoryOutboxStore implements OutboxStorePort, OutboxAppendPort { if (nextAttemptAt != null) { row.nextAttemptAt = nextAttemptAt; } + if (failureCode != null) { + row.lastFailureCode = failureCode; + } return OutboxCompletion.APPLIED; } @@ -211,10 +221,38 @@ public class InMemoryOutboxStore implements OutboxStorePort, OutboxAppendPort { return ages; } + @Override + public OutboxLongRetries longRetries(int minDeferredClaims, Instant now) { + List retrying = + rows.values().stream() + .filter(row -> row.status == OutboxEventStatus.FAILED) + .filter(row -> row.deferredAttemptCount >= minDeferredClaims) + .toList(); + if (retrying.isEmpty()) { + return OutboxLongRetries.NONE; + } + long oldest = + retrying.stream() + .mapToLong(row -> Duration.between(row.event.occurredAt(), now).toSeconds()) + .max() + .orElseThrow(); + return new OutboxLongRetries(retrying.size(), Math.max(0, oldest)); + } + public OutboxEventStatus status(String eventId) { return row(eventId).status; } + /** + * The code the row's last failed completion recorded, or {@code null} before any failed. + * + * @param eventId the row + * @return the last failure code + */ + public String lastFailureCode(String eventId) { + return row(eventId).lastFailureCode; + } + public int attemptCount(String eventId) { return row(eventId).attemptCount; } @@ -262,6 +300,7 @@ public class InMemoryOutboxStore implements OutboxStorePort, OutboxAppendPort { private int attemptCount; private int deferredAttemptCount; private Instant nextAttemptAt; + private String lastFailureCode; private String leaseOwner; private long leaseToken; diff --git a/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/OutboxAggregateSequenceContract.java b/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/OutboxAggregateSequenceContract.java index 1a3cd869..27fce80a 100644 --- a/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/OutboxAggregateSequenceContract.java +++ b/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/OutboxAggregateSequenceContract.java @@ -1,10 +1,12 @@ package dev.caskeleton.application.outbox.testkit; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.application.messaging.event.AggregateIdentity; import dev.caskeleton.application.outbox.OutboxAggregateSequencePort; import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.transaction.testkit.OverlappingWrites; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -14,6 +16,10 @@ import org.junit.jupiter.api.Test; /** * What every {@link OutboxAggregateSequencePort} promises, run against the in-memory fake and the * JPA adapter alike. Each test uses aggregates of its own, so a subclass may share one store. + * + *

    Where an aggregate's numbers start is the implementation's: the fake starts at 1, the JPA + * counter above every number the replaced wall-clock scheme could have issued (V21). What every + * implementation promises is the step and the independence. */ public abstract class OutboxAggregateSequenceContract { @@ -22,8 +28,8 @@ public abstract class OutboxAggregateSequenceContract { protected abstract TransactionPort transactions(); @Test - @DisplayName("같은 aggregate의 순서 번호는 1부터 커밋마다 하나씩 커진다") - void anAggregatesNumbersStartAtOneAndGrowByOnePerCommittedWrite() { + @DisplayName("같은 aggregate의 순서 번호는 첫 번호부터 커밋마다 하나씩 커진다") + void anAggregatesNumbersGrowByOnePerCommittedWriteFromItsFirst() { // given AggregateIdentity aggregate = aggregate(); List numbers = new ArrayList<>(); @@ -34,23 +40,110 @@ public abstract class OutboxAggregateSequenceContract { } // then - assertThat(numbers).containsExactly(1L, 2L, 3L); + long first = numbers.get(0); + assertThat(List.of(first >= 1, numbers)) + .containsExactly(true, List.of(first, first + 1, first + 2)); } @Test - @DisplayName("aggregate마다 순서 번호를 따로 센다") - void eachAggregateCountsOnItsOwn() { + @DisplayName("다른 aggregate의 쓰기는 이 aggregate의 첫 번호를 바꾸지 않는다") + void anotherAggregatesWritesDoNotMoveThisAggregatesFirstNumber() { // given AggregateIdentity first = aggregate(); AggregateIdentity second = aggregate(); - transactions().inWrite(() -> sequences().nextSequence(first)); + long firstsFirst = transactions().inWrite(() -> sequences().nextSequence(first)); transactions().inWrite(() -> sequences().nextSequence(first)); // when long secondsFirst = transactions().inWrite(() -> sequences().nextSequence(second)); // then - assertThat(secondsFirst).isEqualTo(1L); + assertThat(secondsFirst).isEqualTo(firstsFirst); + } + + /** + * The first two writes of an aggregate overlap: both find no counter. A store that creates the + * counter with a plain insert fails the second one on the primary key (review RR-03-01), which + * rejects a valid business write that no caller retries. + */ + @Test + @DisplayName("카운터가 없는 aggregate에 두 쓰기가 겹쳐도 둘 다 성공하고 커밋 순서대로 연속 번호를 받는다") + void twoOverlappingFirstWritesBothSucceedWithConsecutiveNumbers() { + // given + AggregateIdentity aggregate = aggregate(); + + // when + List numbers = + OverlappingWrites.run( + work -> transactions().inWrite(work), + () -> sequences().nextSequence(aggregate), + () -> sequences().nextSequence(aggregate)); + + // then + assertThat(numbers.get(1)).isEqualTo(numbers.get(0) + 1); + } + + /** + * Outside the caller's write transaction the number would commit apart from the event it numbers + * (review 2026-11-03 RR-01-06). + */ + @Test + @DisplayName("쓰기 트랜잭션 밖에서는 순서 번호를 주지 않는다") + void aNumberIsRefusedOutsideTheCallersWriteTransaction() { + // given + AggregateIdentity aggregate = aggregate(); + + // when + Throwable thrown = catchThrowable(() -> sequences().nextSequence(aggregate)); + + // then + assertThat(thrown) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("requires an active caller transaction"); + } + + /** + * The number is taken under the counter's lock and goes back with a rollback, so the next commit + * reuses it rather than leaving a gap (review 2026-11-03 RR-01-06). Relative to the first number, + * so a store that seeds a new counter above 1 is held to the same case. + */ + @Test + @DisplayName("롤백된 쓰기가 받은 순서 번호는 다음 쓰기가 다시 받는다") + void aNumberTakenByARolledBackWriteIsHandedOutAgain() { + // given + AggregateIdentity aggregate = aggregate(); + long first = transactions().inWrite(() -> sequences().nextSequence(aggregate)); + long rolledBack = numberTakenByARolledBackWrite(aggregate); + + // when + long next = transactions().inWrite(() -> sequences().nextSequence(aggregate)); + + // then + assertThat(List.of(rolledBack, next)).containsExactly(first + 1, first + 1); + } + + private long numberTakenByARolledBackWrite(AggregateIdentity aggregate) { + long[] taken = new long[1]; + try { + transactions() + .inWrite( + () -> { + taken[0] = sequences().nextSequence(aggregate); + throw new RollBack(); + }); + } catch (RollBack expected) { + // the write rolled back, as this case needs + } + return taken[0]; + } + + /** Thrown inside a write so that its transaction rolls back. */ + private static final class RollBack extends RuntimeException { + private static final long serialVersionUID = 1L; + + RollBack() { + super("roll the write back"); + } } private static AggregateIdentity aggregate() { diff --git a/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/OutboxLeaseFencingContract.java b/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/OutboxLeaseFencingContract.java index ff17746b..279640f2 100644 --- a/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/OutboxLeaseFencingContract.java +++ b/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/OutboxLeaseFencingContract.java @@ -7,6 +7,7 @@ import dev.caskeleton.application.outbox.OutboxBackoffPolicy; import dev.caskeleton.application.outbox.OutboxCompletion; import dev.caskeleton.application.outbox.OutboxEventStatus; import dev.caskeleton.application.outbox.OutboxLease; +import dev.caskeleton.application.outbox.OutboxLongRetries; import dev.caskeleton.application.outbox.OutboxMessagePublishPort; import dev.caskeleton.application.outbox.OutboxRelayFailureReport; import dev.caskeleton.application.outbox.OutboxRelayFailureReportPort; @@ -54,6 +55,13 @@ public abstract class OutboxLeaseFencingContract { /** Reads the row's persisted status, bypassing anything the store caches. */ protected abstract OutboxEventStatus statusOf(String eventId); + /** + * Reads the row's persisted {@code last_failure_code}, bypassing anything the store caches. + * + * @return the code, or {@code null} when no failed completion wrote one + */ + protected abstract String lastFailureCodeOf(String eventId); + @Test void lateFailureFromASupersededRelayIsStaleAndThePublishedRowStays() { // given @@ -149,7 +157,8 @@ public abstract class OutboxLeaseFencingContract { transactions().inWrite(() -> store().markPublished(lease)); // when - OutboxCompletion second = transactions().inWrite(() -> store().markDead(lease)); + OutboxCompletion second = + transactions().inWrite(() -> store().markDead(lease, "REJECTED_BEFORE_SEND")); // then assertThat(Map.entry(second, statusOf(eventId))) @@ -201,7 +210,9 @@ public abstract class OutboxLeaseFencingContract { Instant retryAt = CLAIMED_AT.plusSeconds(30); // when - OutboxCompletion deferred = transactions().inWrite(() -> store().markDeferred(first, retryAt)); + OutboxCompletion deferred = + transactions() + .inWrite(() -> store().markDeferred(first, retryAt, "KAFKA_TRANSPORT_CLOSED")); // then OutboxLease second = claim("relay-a", retryAt); @@ -214,6 +225,95 @@ public abstract class OutboxLeaseFencingContract { .containsExactly(2, 1, 1); } + @Test + @DisplayName("미룬 시도는 publisher가 붙인 원인 코드를 행에 남긴다") + void aDeferredClaimLeavesThePublishersFailureCodeOnTheRow() { + // given + String eventId = appendEvent("deferred-code"); + OutboxLease lease = claim("relay-a", CLAIMED_AT); + + // when + transactions() + .inWrite( + () -> + store() + .markDeferred( + lease, CLAIMED_AT.plusSeconds(30), "KAFKA_AUTHENTICATIONEXCEPTION")); + + // then + assertThat(List.of(statusOf(eventId), lastFailureCodeOf(eventId))) + .containsExactly(OutboxEventStatus.FAILED, "KAFKA_AUTHENTICATIONEXCEPTION"); + } + + @Test + @DisplayName("DEAD로 옮긴 시도는 그 원인 코드를 행에 남긴다") + void aDeadLetteredClaimLeavesItsFailureCodeOnTheRow() { + // given + String eventId = appendEvent("dead-code"); + OutboxLease lease = claim("relay-a", CLAIMED_AT); + + // when + transactions().inWrite(() -> store().markDead(lease, "KAFKA_RECORDTOOLARGEEXCEPTION")); + + // then + assertThat(List.of(statusOf(eventId), lastFailureCodeOf(eventId))) + .containsExactly(OutboxEventStatus.DEAD, "KAFKA_RECORDTOOLARGEEXCEPTION"); + } + + @Test + @DisplayName("넘겨받힌 claim의 실패는 행의 원인 코드를 바꾸지 않는다") + void aSupersededLeasesFailureDoesNotOverwriteTheRowsFailureCode() { + // given + String eventId = appendEvent("stale-code"); + OutboxLease expired = claim("relay-a", CLAIMED_AT); + OutboxLease current = claim("relay-b", AFTER_EXPIRY); + transactions() + .inWrite(() -> store().markDeferred(current, AFTER_EXPIRY.plusSeconds(30), "CURRENT_CODE")); + + // when + OutboxCompletion late = + transactions() + .inWrite(() -> store().markFailed(expired, AFTER_EXPIRY.plusSeconds(30), "LATE_CODE")); + + // then + assertThat(List.of(late, lastFailureCodeOf(eventId))) + .containsExactly(OutboxCompletion.STALE, "CURRENT_CODE"); + } + + @Test + @DisplayName("미룬 claim이 기준 수 이상 쌓인 FAILED 행만 오래 재시도 중인 행으로 센다") + void onlyFailedRowsWithEnoughDeferredClaimsCountAsLongRetrying() { + // given: one row deferred twice (due again at +60s), one deferred once (claimed at +59s, when + // it is the only row due), measured an hour after both occurred + String twice = appendEvent("long-twice"); + deferTwice(twice); + appendEvent("long-once"); + OutboxLease once = claim("relay-a", CLAIMED_AT.plusSeconds(59)); + transactions().inWrite(() -> store().markDeferred(once, CLAIMED_AT.plusSeconds(89), "ONCE")); + Instant anHourLater = CLAIMED_AT.minusSeconds(1).plusSeconds(3600); + + // when + OutboxLongRetries longRetries = store().longRetries(2, anHourLater); + + // then + assertThat(longRetries).isEqualTo(new OutboxLongRetries(1, 3600)); + } + + /** Claims and defers the only claimable row twice; it ends FAILED with two deferred claims. */ + private void deferTwice(String eventId) { + Instant at = CLAIMED_AT; + for (int i = 0; i < 2; i++) { + OutboxLease lease = claim("relay-a", at); + if (!lease.eventId().equals(eventId)) { + throw new IllegalStateException( + "expected to claim " + eventId + ", got " + lease.eventId()); + } + Instant retryAt = at.plusSeconds(30); + transactions().inWrite(() -> store().markDeferred(lease, retryAt, "TWICE")); + at = retryAt; + } + } + private OutboxLease claim(String owner, Instant now) { return transactions() .inWrite(() -> store().claimBatch(owner, 1, now, IN_FLIGHT_TIMEOUT)) @@ -302,13 +402,15 @@ public abstract class OutboxLeaseFencingContract { } @Override - public OutboxCompletion markFailed(OutboxLease lease, Instant nextAttemptAt) { - return count(delegate.markFailed(lease, nextAttemptAt)); + public OutboxCompletion markFailed( + OutboxLease lease, Instant nextAttemptAt, String failureCode) { + return count(delegate.markFailed(lease, nextAttemptAt, failureCode)); } @Override - public OutboxCompletion markDeferred(OutboxLease lease, Instant nextAttemptAt) { - return count(delegate.markDeferred(lease, nextAttemptAt)); + public OutboxCompletion markDeferred( + OutboxLease lease, Instant nextAttemptAt, String failureCode) { + return count(delegate.markDeferred(lease, nextAttemptAt, failureCode)); } @Override @@ -317,8 +419,8 @@ public abstract class OutboxLeaseFencingContract { } @Override - public OutboxCompletion markDead(OutboxLease lease) { - return count(delegate.markDead(lease)); + public OutboxCompletion markDead(OutboxLease lease, String failureCode) { + return count(delegate.markDead(lease, failureCode)); } private OutboxCompletion count(OutboxCompletion completion) { @@ -337,5 +439,10 @@ public abstract class OutboxLeaseFencingContract { public Map oldestUnpublishedAgeSecondsByEventType(Instant now) { return delegate.oldestUnpublishedAgeSecondsByEventType(now); } + + @Override + public OutboxLongRetries longRetries(int minDeferredClaims, Instant now) { + return delegate.longRetries(minDeferredClaims, now); + } } } diff --git a/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/OutboxMessagePublishContract.java b/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/OutboxMessagePublishContract.java index 3f7d39cf..0686110a 100644 --- a/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/OutboxMessagePublishContract.java +++ b/src/application-core/src/testFixtures/java/dev/caskeleton/application/outbox/testkit/OutboxMessagePublishContract.java @@ -6,6 +6,7 @@ import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.application.messaging.event.testkit.BrokerAnswer; import dev.caskeleton.application.outbox.ClaimedOutboxEvent; import dev.caskeleton.application.outbox.OutboxMessagePublishPort; +import dev.caskeleton.application.outbox.OutboxPublishAttempt; import dev.caskeleton.application.outbox.OutboxPublishOutcome; import java.time.Instant; import java.util.List; @@ -40,7 +41,7 @@ public abstract class OutboxMessagePublishContract { OutboxMessagePublishPort port = publisherOverABrokerThat(BrokerAnswer.CONFIRMS); // when - OutboxPublishOutcome outcome = port.publishForOutcome(claimedRow()); + OutboxPublishOutcome outcome = port.publishForOutcome(claimedRow()).outcome(); // then assertThat(outcome).isEqualTo(OutboxPublishOutcome.CONFIRMED); @@ -54,7 +55,7 @@ public abstract class OutboxMessagePublishContract { OutboxMessagePublishPort port = publisherOverABrokerThat(BrokerAnswer.NEVER_CONFIRMS); // when - OutboxPublishOutcome outcome = port.publishForOutcome(claimedRow()); + OutboxPublishOutcome outcome = port.publishForOutcome(claimedRow()).outcome(); // then assertThat(outcome).isEqualTo(OutboxPublishOutcome.AMBIGUOUS); @@ -67,12 +68,25 @@ public abstract class OutboxMessagePublishContract { OutboxMessagePublishPort port = publisherOverABrokerThat(BrokerAnswer.REFUSES_AFTER_RECEIVING); // when - OutboxPublishOutcome outcome = port.publishForOutcome(claimedRow()); + OutboxPublishOutcome outcome = port.publishForOutcome(claimedRow()).outcome(); // then assertThat(outcome).isEqualTo(OutboxPublishOutcome.REJECTED_AFTER_BROKER); } + @Test + @DisplayName("broker의 거부는 그 거부에 붙은 원인 코드를 함께 알린다") + void aBrokerRefusalCarriesItsFailureCode() { + // given + OutboxMessagePublishPort port = publisherOverABrokerThat(BrokerAnswer.REFUSES_FOR_NOW); + + // when + OutboxPublishAttempt attempt = port.publishForOutcome(claimedRow()); + + // then + assertThat(attempt.failureCode()).isPresent(); + } + @Test @DisplayName("예외로 알리는 발행은 broker가 확인했을 때만 정상 반환한다") void theThrowingPublishReturnsWhenTheBrokerConfirmed() { diff --git a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/OverlappingWrites.java b/src/application-core/src/testFixtures/java/dev/caskeleton/application/transaction/testkit/OverlappingWrites.java similarity index 89% rename from src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/OverlappingWrites.java rename to src/application-core/src/testFixtures/java/dev/caskeleton/application/transaction/testkit/OverlappingWrites.java index 6ce99191..6d1ab721 100644 --- a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/OverlappingWrites.java +++ b/src/application-core/src/testFixtures/java/dev/caskeleton/application/transaction/testkit/OverlappingWrites.java @@ -1,4 +1,4 @@ -package dev.caskeleton.application.notification.platform.policy.testkit; +package dev.caskeleton.application.transaction.testkit; import java.time.Duration; import java.util.List; @@ -21,8 +21,11 @@ import java.util.function.Supplier; * has no transactions simply runs the two one after the other, which is the answer it must give * anyway. * - *

    Public because every store contract with an insert-or-find has the same race to show: the - * suppression and preference contracts here, and the admin operation claim. + *

    Public, and here beside the transaction port's own fake, because every store contract with an + * insert-or-find has the same race to show: the outbox order counter's first number, and the + * notification suppression, preference, deduplication, request and admin operation stores. It lived + * in application-notification until the outbox contract needed it too; one copy, not two that + * drift. */ public final class OverlappingWrites { diff --git a/src/application-fileserver/build.gradle b/src/application-fileserver/build.gradle index 41282aad..aa3128b5 100644 --- a/src/application-fileserver/build.gradle +++ b/src/application-fileserver/build.gradle @@ -11,10 +11,11 @@ dependencies { // The port owner's test fixtures: one hand-rolled fake per port, and the abstract port // contracts (`*Contract`) that a fake and the real adapter both extend (AGENTS.md test rule 6, - // capability criterion 5). A use-case test that passes against a fake is passing against the - // behaviour the JPA adapters in persistence-fileserver are held to. Adapters reach them with - // `testImplementation testFixtures(...)`. FileserverException's error code is a shared-contract - // type; the contracts assert on it. + // capability criterion 5). Every store port persistence-fileserver implements has one, and so + // does the access policy (docs/testing/TESTING_STRATEGY.md 9.3), so a use-case test that passes + // against a fake is passing against the behaviour the JPA adapters are held to on PostgreSQL. + // Adapters reach them with `testImplementation testFixtures(...)`. FileserverException's error + // code is a shared-contract type; the contracts assert on it. testFixturesImplementation project(':application-core') // DirectTransactions records and injects faults around application-core's shared fake rather // than being a second transaction port of its own. diff --git a/src/application-fileserver/src/main/java/dev/caskeleton/application/fileserver/api/security/FileAccessSubject.java b/src/application-fileserver/src/main/java/dev/caskeleton/application/fileserver/api/security/FileAccessSubject.java index d37e884e..b1e34c2e 100644 --- a/src/application-fileserver/src/main/java/dev/caskeleton/application/fileserver/api/security/FileAccessSubject.java +++ b/src/application-fileserver/src/main/java/dev/caskeleton/application/fileserver/api/security/FileAccessSubject.java @@ -1,5 +1,6 @@ package dev.caskeleton.application.fileserver.api.security; +import dev.caskeleton.application.fileserver.api.metadata.FileDescriptor; import dev.caskeleton.application.fileserver.api.metadata.FileOwner; import java.util.Map; import java.util.Objects; @@ -45,4 +46,21 @@ public record FileAccessSubject( public FileOwner asOwner() { return new FileOwner(principalId, tenant()); } + + /** + * Whether this caller created {@code file}, in this caller's tenant: the one ownership rule every + * {@link FileAccessPolicyPort} applies to an operation on an existing file (owner decision 14). + * + *

    A file with no recorded owner predates ownership and belongs to nobody. A file created + * without a tenant is not owned by the same principal calling with one, and the other way round. + * Policies call this instead of comparing owners themselves, so the fake and every production + * policy cannot drift apart (review 2026-11-03 RR-07-06); {@code FileAccessPolicyPortContract} + * holds each of them to it. + * + * @param file the file the operation targets + * @return true when this caller owns it + */ + public boolean owns(FileDescriptor file) { + return file.owner().filter(asOwner()::equals).isPresent(); + } } diff --git a/src/application-fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/QuotaReclaimPort.java b/src/application-fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/QuotaReclaimPort.java index 0a63a6f5..2829f3a4 100644 --- a/src/application-fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/QuotaReclaimPort.java +++ b/src/application-fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/QuotaReclaimPort.java @@ -6,7 +6,9 @@ import dev.caskeleton.application.fileserver.api.metadata.QuotaScope; * Returns committed capacity after content is physically gone. * *

    Reclaim happens after the delete, never before: giving the quota back first would let a tenant - * over-allocate against space that has not actually been freed. + * over-allocate against space that has not actually been freed. The one other caller is a move to + * another namespace, which reclaims the file's bytes from the source in the same transaction that + * commits them to the target — the bytes stay counted, only their namespace changes. */ public interface QuotaReclaimPort { diff --git a/src/application-fileserver/src/main/java/dev/caskeleton/application/fileserver/lifecycle/MoveFileApplicationUseCase.java b/src/application-fileserver/src/main/java/dev/caskeleton/application/fileserver/lifecycle/MoveFileApplicationUseCase.java index ba23d23f..348faf9a 100644 --- a/src/application-fileserver/src/main/java/dev/caskeleton/application/fileserver/lifecycle/MoveFileApplicationUseCase.java +++ b/src/application-fileserver/src/main/java/dev/caskeleton/application/fileserver/lifecycle/MoveFileApplicationUseCase.java @@ -4,21 +4,44 @@ import dev.caskeleton.application.capability.Idempotency; import dev.caskeleton.application.capability.RepositoryAccess; import dev.caskeleton.application.capability.UseCaseCapability; import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.api.StorageNamespace; import dev.caskeleton.application.fileserver.api.metadata.FileMetadataPort; +import dev.caskeleton.application.fileserver.api.metadata.FileQuotaPort; import dev.caskeleton.application.fileserver.api.metadata.FileRecord; import dev.caskeleton.application.fileserver.api.metadata.FileRecords; +import dev.caskeleton.application.fileserver.api.metadata.QuotaReservation; +import dev.caskeleton.application.fileserver.api.metadata.QuotaScope; import dev.caskeleton.application.fileserver.api.security.FileAccessPolicyPort; import dev.caskeleton.application.fileserver.api.security.FileOperation; +import dev.caskeleton.application.fileserver.cleanup.QuotaReclaimPort; import dev.caskeleton.application.fileserver.port.in.MoveFileCommand; import dev.caskeleton.application.fileserver.port.in.MoveFileUseCase; import dev.caskeleton.application.fileserver.upload.FileView; +import dev.caskeleton.application.fileserver.upload.UploadPolicies; +import dev.caskeleton.application.fileserver.upload.UploadPolicy; import dev.caskeleton.application.security.RequiresPermission; import dev.caskeleton.application.transaction.TransactionMode; import dev.caskeleton.application.transaction.TransactionPort; import java.util.Objects; import java.util.Optional; -/** Moves a readable file to another namespace that {@link TargetNamespacePolicy} opens. */ +/** + * Moves a readable file to another namespace that {@link TargetNamespacePolicy} opens, and its + * bytes' quota with it. + * + *

    Quota usage is booked per namespace, so a move that only relocated the record left the bytes + * counted where the file no longer is and uncounted where it now lives: the target could pass its + * limit through moves, the source carried the departed file forever, and deleting the moved file + * reclaimed from the target what the target never booked (review 2026-11-03 RR-01-05). A move + * across namespaces therefore does, in the one transaction that relocates the record, what an + * upload into the target and a delete from the source would: {@link + * dev.caskeleton.application.fileserver.quota.NamespaceQuota#requireRoom} against the target, + * reserve and commit the file's bytes there, and reclaim the same bytes from the source. A refused + * move changes nothing. A move within the namespace leaves the ledger alone. + * + *

    Copy-then-delete was the other way to close it, and was not taken: a move would stop being + * atomic and would rewrite bytes the content key already addresses. + */ @RequiresPermission("file:move") @UseCaseCapability( transactionMode = TransactionMode.WRITE, @@ -30,17 +53,25 @@ public class MoveFileApplicationUseCase implements MoveFileUseCase { private final FileMetadataPort metadataStore; private final FileAccessPolicyPort accessPolicy; + private final UploadPolicy uploadPolicy; private final TargetNamespacePolicy targetNamespaces; + private final FileQuotaPort quota; + private final QuotaReclaimPort reclaims; private final TransactionPort transactions; public MoveFileApplicationUseCase( FileMetadataPort metadataStore, - FileAccessPolicyPort accessPolicy, + UploadPolicies policies, TargetNamespacePolicy targetNamespaces, + FileQuotaPort quota, + QuotaReclaimPort reclaims, TransactionPort transactions) { this.metadataStore = Objects.requireNonNull(metadataStore, "metadataStore"); - this.accessPolicy = Objects.requireNonNull(accessPolicy, "accessPolicy"); + this.accessPolicy = policies.access(); + this.uploadPolicy = policies.upload(); this.targetNamespaces = Objects.requireNonNull(targetNamespaces, "targetNamespaces"); + this.quota = Objects.requireNonNull(quota, "quota"); + this.reclaims = Objects.requireNonNull(reclaims, "reclaims"); this.transactions = Objects.requireNonNull(transactions, "transactions"); } @@ -57,7 +88,28 @@ public class MoveFileApplicationUseCase implements MoveFileUseCase { return FileView.of( transactions .inWrite( - () -> metadataStore.relocate(fileId, record.version(), command.targetNamespace())) + () -> { + transferQuota(record, command.targetNamespace()); + return metadataStore.relocate( + fileId, record.version(), command.targetNamespace()); + }) .toDescriptor()); } + + /** Books the file's bytes on {@code target} and takes them off the namespace it leaves. */ + private void transferQuota(FileRecord record, StorageNamespace target) { + if (record.namespace().equals(target)) { + return; + } + long bytes = record.actualSize().orElseThrow(MoveFileApplicationUseCase::readyWithoutSize); + QuotaScope targetScope = QuotaScope.ofNamespace(target.value()); + uploadPolicy.namespaceQuota().requireRoom(quota, targetScope, bytes); + QuotaReservation reservation = quota.reserve(targetScope, bytes, uploadPolicy.reservationTtl()); + quota.commit(reservation, bytes); + reclaims.reclaim(QuotaScope.ofNamespace(record.namespace().value()), bytes); + } + + private static IllegalStateException readyWithoutSize() { + return new IllegalStateException("READY record has no size"); + } } diff --git a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java index c0d4b042..dd0bd88d 100644 --- a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java +++ b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java @@ -17,6 +17,7 @@ import dev.caskeleton.application.fileserver.api.transfer.UploadMode; import dev.caskeleton.application.fileserver.observability.FileserverMetricsPort; import dev.caskeleton.application.fileserver.testkit.DirectTransactions; import dev.caskeleton.application.fileserver.testkit.FakeCleanupContentGateway; +import dev.caskeleton.application.fileserver.testkit.FakeFileQuotaService; import dev.caskeleton.application.fileserver.testkit.FileserverFixtures; import dev.caskeleton.application.fileserver.testkit.InMemoryCleanupQueue; import dev.caskeleton.application.fileserver.testkit.InMemoryFileMetadataStore; @@ -40,7 +41,8 @@ class CleanupServiceTest { private final FakeCleanupContentGateway content = new FakeCleanupContentGateway(); private final InMemoryFileMetadataStore metadata = new InMemoryFileMetadataStore(); private final InMemoryUploadSessionStore sessions = new InMemoryUploadSessionStore(); - private final RecordingQuotaReclaimGateway quota = new RecordingQuotaReclaimGateway(); + private final RecordingQuotaReclaimGateway quota = + new RecordingQuotaReclaimGateway(new FakeFileQuotaService()); private final DirectTransactions transactions = new DirectTransactions(); private final CleanupService cleanup = diff --git a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/lifecycle/FileLifecycleUseCasesTest.java b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/lifecycle/FileLifecycleUseCasesTest.java index db33316f..5849351e 100644 --- a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/lifecycle/FileLifecycleUseCasesTest.java +++ b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/lifecycle/FileLifecycleUseCasesTest.java @@ -35,6 +35,7 @@ import dev.caskeleton.application.fileserver.testkit.InMemoryCleanupQueue; import dev.caskeleton.application.fileserver.testkit.InMemoryFileMetadataStore; import dev.caskeleton.application.fileserver.testkit.InMemoryUploadSessionStore; import dev.caskeleton.application.fileserver.testkit.ReadyFileFixture; +import dev.caskeleton.application.fileserver.testkit.RecordingQuotaReclaimGateway; import dev.caskeleton.application.fileserver.testkit.SequentialUploadIdentifierFactory; import dev.caskeleton.application.fileserver.upload.FileView; import dev.caskeleton.application.fileserver.upload.UploadPolicies; @@ -83,8 +84,20 @@ class FileLifecycleUseCasesTest { private final CopyFileApplicationUseCase copy = copyUnder(UploadPolicy.standard(FileserverFixtures.NAMESPACE)); + private final RecordingQuotaReclaimGateway reclaims = new RecordingQuotaReclaimGateway(quota); + private final MoveFileApplicationUseCase move = - new MoveFileApplicationUseCase(metadata, accessPolicy, TARGETS, transactions); + moveUnder(UploadPolicy.standard(FileserverFixtures.NAMESPACE)); + + private MoveFileApplicationUseCase moveUnder(UploadPolicy policy) { + return new MoveFileApplicationUseCase( + metadata, + new UploadPolicies(accessPolicy, OriginalFilenamePolicy.standard(), policy), + TARGETS, + quota, + reclaims, + transactions); + } private CopyFileApplicationUseCase copyUnder(UploadPolicy policy) { return new CopyFileApplicationUseCase( @@ -449,6 +462,56 @@ class FileLifecycleUseCasesTest { .containsExactly(FileserverErrorCode.ACCESS_DENIED, ready.namespace()); } + @Test + @DisplayName("이동은 파일의 바이트를 출발 namespace 사용량에서 대상 namespace 사용량으로 옮긴다") + void moveTransfersTheFilesBytesFromTheSourceNamespaceToTheTarget() { + // given + FileRecord ready = ReadyFileFixture.ready(metadata, SIZE, DIGEST); + QuotaScope source = QuotaScope.ofNamespace(ready.namespace().value()); + QuotaScope target = QuotaScope.ofNamespace(TARGET.value()); + quota.alreadyHolds(source, SIZE); + + // when + move.handle( + new MoveFileCommand( + ready.fileId(), TARGET, Optional.empty(), FileserverFixtures.context())); + + // then + assertThat(List.of(quota.usedBytes(source), quota.usedBytes(target))).containsExactly(0L, SIZE); + } + + @Test + @DisplayName("대상 namespace 쿼터 상한을 넘기는 이동은 QUOTA_EXCEEDED로 거절되고 파일과 사용량은 그대로다") + void moveThatWouldTakeTheTargetNamespacePastItsQuotaIsRefused() { + // given + FileRecord ready = ReadyFileFixture.ready(metadata, SIZE, DIGEST); + QuotaScope source = QuotaScope.ofNamespace(ready.namespace().value()); + QuotaScope target = QuotaScope.ofNamespace(TARGET.value()); + quota.alreadyHolds(source, SIZE); + quota.alreadyHolds(target, 15); + MoveFileApplicationUseCase limited = + moveUnder( + UploadPolicy.standard(FileserverFixtures.NAMESPACE) + .withNamespaceQuota(NamespaceQuota.ofBytes(24))); + + // when + Throwable failure = + catchThrowable( + () -> + limited.handle( + new MoveFileCommand( + ready.fileId(), TARGET, Optional.empty(), FileserverFixtures.context()))); + + // then + assertThat( + List.of( + codeOf(failure), + metadata.find(ready.fileId()).orElseThrow().namespace(), + quota.usedBytes(source), + quota.usedBytes(target))) + .containsExactly(FileserverErrorCode.QUOTA_EXCEEDED, ready.namespace(), SIZE, 15L); + } + private static Object codeOf(Throwable failure) { if (failure == null) { return "no failure"; diff --git a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/FileReconciliationServiceTest.java b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/FileReconciliationServiceTest.java index b352eb9e..f0d836a9 100644 --- a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/FileReconciliationServiceTest.java +++ b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/FileReconciliationServiceTest.java @@ -11,6 +11,7 @@ import dev.caskeleton.application.fileserver.api.metadata.FileRecordMutation; import dev.caskeleton.application.fileserver.api.metadata.UploadSessionDraft; import dev.caskeleton.application.fileserver.api.transfer.UploadMode; import dev.caskeleton.application.fileserver.testkit.DirectTransactions; +import dev.caskeleton.application.fileserver.testkit.FakeFileQuotaService; import dev.caskeleton.application.fileserver.testkit.FakeQuotaCommitGateway; import dev.caskeleton.application.fileserver.testkit.FakeReconciliationContentProbe; import dev.caskeleton.application.fileserver.testkit.FileserverFixtures; @@ -33,7 +34,8 @@ class FileReconciliationServiceTest { private final FakeReconciliationContentProbe probe = new FakeReconciliationContentProbe(); private final InMemoryRecoveryQueue recovery = new InMemoryRecoveryQueue(); private final InMemoryUploadSessionStore sessions = new InMemoryUploadSessionStore(); - private final FakeQuotaCommitGateway quota = new FakeQuotaCommitGateway(); + private final FakeQuotaCommitGateway quota = + new FakeQuotaCommitGateway(new FakeFileQuotaService(), metadata); private final FileReconciliationService service = new DefaultFileReconciliationService( diff --git a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/FakeFileAccessPolicyContractTest.java b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/FakeFileAccessPolicyContractTest.java new file mode 100644 index 00000000..106c3b45 --- /dev/null +++ b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/FakeFileAccessPolicyContractTest.java @@ -0,0 +1,25 @@ +package dev.caskeleton.application.fileserver.testkit; + +import dev.caskeleton.application.fileserver.api.security.FileAccessPolicyPort; +import dev.caskeleton.application.fileserver.api.security.FileAccessSubject; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +class FakeFileAccessPolicyContractTest extends FileAccessPolicyPortContract { + + private final FakeFileAccessPolicy policy = new FakeFileAccessPolicy(); + + @Override + protected FileAccessPolicyPort policy() { + return policy; + } + + @Override + protected FileAccessSubject permittedSubject(String principalId, Optional tenant) { + return new FileAccessSubject( + principalId, + Set.of("uploader"), + tenant.map(value -> Map.of(FileAccessSubject.TENANT_ATTRIBUTE, value)).orElse(Map.of())); + } +} diff --git a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryContentReferenceLedgerContractTest.java b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryContentReferenceLedgerContractTest.java new file mode 100644 index 00000000..08546dc9 --- /dev/null +++ b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryContentReferenceLedgerContractTest.java @@ -0,0 +1,19 @@ +package dev.caskeleton.application.fileserver.testkit; + +import dev.caskeleton.application.fileserver.admin.ContentReferenceLedgerPort; +import dev.caskeleton.application.fileserver.api.metadata.FileMetadataPort; + +class InMemoryContentReferenceLedgerContractTest extends ContentReferenceLedgerPortContract { + + private final InMemoryFileMetadataStore metadata = new InMemoryFileMetadataStore(); + + @Override + protected ContentReferenceLedgerPort ledger() { + return metadata; + } + + @Override + protected FileMetadataPort metadata() { + return metadata; + } +} diff --git a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryQuotaLedgerContractTest.java b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryQuotaLedgerContractTest.java new file mode 100644 index 00000000..2494b443 --- /dev/null +++ b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryQuotaLedgerContractTest.java @@ -0,0 +1,58 @@ +package dev.caskeleton.application.fileserver.testkit; + +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.api.StorageNamespace; +import dev.caskeleton.application.fileserver.api.metadata.FileQuotaPort; +import dev.caskeleton.application.fileserver.api.metadata.FileRecordDraft; +import dev.caskeleton.application.fileserver.api.metadata.UploadSession; +import dev.caskeleton.application.fileserver.cleanup.QuotaReclaimPort; +import dev.caskeleton.application.fileserver.upload.QuotaCommitPort; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.UUID; + +class InMemoryQuotaLedgerContractTest extends FileQuotaLedgerContract { + + private final InMemoryFileMetadataStore metadata = new InMemoryFileMetadataStore(); + private final FakeFileQuotaService ledger = new FakeFileQuotaService(); + private final FakeQuotaCommitGateway commits = new FakeQuotaCommitGateway(ledger, metadata); + private final RecordingQuotaReclaimGateway reclaims = new RecordingQuotaReclaimGateway(ledger); + private Instant now = FileserverFixtures.NOW; + + @Override + protected FileQuotaPort quota() { + return ledger; + } + + @Override + protected QuotaCommitPort commits() { + return commits; + } + + @Override + protected QuotaReclaimPort reclaims() { + return reclaims; + } + + @Override + protected UploadSession uploadIn(StorageNamespace namespace) { + FileId fileId = FileId.of(UUID.randomUUID()); + metadata.insert( + new FileRecordDraft( + fileId, + namespace, + "report.bin", + Optional.empty(), + OptionalLong.of(1_000), + FileserverFixtures.context().subject().asOwner())); + return FileserverFixtures.session(fileId, 1_000, 1_000); + } + + @Override + protected void elapse(Duration duration) { + now = now.plus(duration); + ledger.advanceTo(now); + } +} diff --git a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryRecoveryQueueContractTest.java b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryRecoveryQueueContractTest.java new file mode 100644 index 00000000..0712a18e --- /dev/null +++ b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryRecoveryQueueContractTest.java @@ -0,0 +1,20 @@ +package dev.caskeleton.application.fileserver.testkit; + +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.recovery.RecoveryQueuePort; +import java.util.UUID; + +class InMemoryRecoveryQueueContractTest extends RecoveryQueuePortContract { + + private final InMemoryRecoveryQueue queue = new InMemoryRecoveryQueue(); + + @Override + protected RecoveryQueuePort queue() { + return queue; + } + + @Override + protected FileId recordedFile() { + return FileId.of(UUID.randomUUID()); + } +} diff --git a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryStagingUploadLocatorContractTest.java b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryStagingUploadLocatorContractTest.java new file mode 100644 index 00000000..1cd78eca --- /dev/null +++ b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryStagingUploadLocatorContractTest.java @@ -0,0 +1,40 @@ +package dev.caskeleton.application.fileserver.testkit; + +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.api.metadata.UploadSessionPort; +import dev.caskeleton.application.fileserver.recovery.StagingUploadLocatorPort; +import java.time.Duration; +import java.time.Instant; +import java.util.UUID; + +class InMemoryStagingUploadLocatorContractTest extends StagingUploadLocatorPortContract { + + private final InMemoryUploadSessionStore sessions = new InMemoryUploadSessionStore(); + private Instant now = FileserverFixtures.NOW; + + @Override + protected StagingUploadLocatorPort locator() { + return sessions; + } + + @Override + protected UploadSessionPort sessions() { + return sessions; + } + + @Override + protected FileId recordedFile() { + return FileId.of(UUID.randomUUID()); + } + + @Override + protected Instant now() { + return now; + } + + @Override + protected void elapse(Duration duration) { + now = now.plus(duration); + sessions.advanceTo(now); + } +} diff --git a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryUploadSessionStoreContractTest.java b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryUploadSessionStoreContractTest.java new file mode 100644 index 00000000..2c6d86c6 --- /dev/null +++ b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/testkit/InMemoryUploadSessionStoreContractTest.java @@ -0,0 +1,34 @@ +package dev.caskeleton.application.fileserver.testkit; + +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.api.metadata.UploadSessionPort; +import java.time.Duration; +import java.time.Instant; +import java.util.UUID; + +class InMemoryUploadSessionStoreContractTest extends UploadSessionPortContract { + + private final InMemoryUploadSessionStore sessions = new InMemoryUploadSessionStore(); + private Instant now = FileserverFixtures.NOW; + + @Override + protected UploadSessionPort sessions() { + return sessions; + } + + @Override + protected FileId recordedFile() { + return FileId.of(UUID.randomUUID()); + } + + @Override + protected Instant now() { + return now; + } + + @Override + protected void elapse(Duration duration) { + now = now.plus(duration); + sessions.advanceTo(now); + } +} diff --git a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/upload/FinalizeUploadApplicationUseCaseTest.java b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/upload/FinalizeUploadApplicationUseCaseTest.java index 48fcd04c..b2cb6d51 100644 --- a/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/upload/FinalizeUploadApplicationUseCaseTest.java +++ b/src/application-fileserver/src/test/java/dev/caskeleton/application/fileserver/upload/FinalizeUploadApplicationUseCaseTest.java @@ -21,6 +21,7 @@ import dev.caskeleton.application.fileserver.concurrency.WriterLeaseCoordinator; import dev.caskeleton.application.fileserver.port.in.FinalizeUploadCommand; import dev.caskeleton.application.fileserver.testkit.DirectTransactions; import dev.caskeleton.application.fileserver.testkit.FakeFileAccessPolicy; +import dev.caskeleton.application.fileserver.testkit.FakeFileQuotaService; import dev.caskeleton.application.fileserver.testkit.FakeQuotaCommitGateway; import dev.caskeleton.application.fileserver.testkit.FakeUploadContentGateway; import dev.caskeleton.application.fileserver.testkit.FileserverFixtures; @@ -43,7 +44,8 @@ class FinalizeUploadApplicationUseCaseTest { private final InMemoryFileMetadataStore metadata = new InMemoryFileMetadataStore(); private final InMemoryUploadSessionStore sessions = new InMemoryUploadSessionStore(); private final FakeUploadContentGateway content = new FakeUploadContentGateway(); - private final FakeQuotaCommitGateway quota = new FakeQuotaCommitGateway(); + private final FakeQuotaCommitGateway quota = + new FakeQuotaCommitGateway(new FakeFileQuotaService(), metadata); private final InMemoryCleanupQueue cleanup = new InMemoryCleanupQueue(); private final InMemoryRecoveryQueue recovery = new InMemoryRecoveryQueue(); private final FakeFileAccessPolicy accessPolicy = new FakeFileAccessPolicy(); diff --git a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/ContentReferenceLedgerPortContract.java b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/ContentReferenceLedgerPortContract.java new file mode 100644 index 00000000..bcf1d46b --- /dev/null +++ b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/ContentReferenceLedgerPortContract.java @@ -0,0 +1,123 @@ +package dev.caskeleton.application.fileserver.testkit; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.application.fileserver.admin.ContentReferenceLedgerPort; +import dev.caskeleton.application.fileserver.api.ContentKey; +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.api.FileState; +import dev.caskeleton.application.fileserver.api.metadata.FileMetadataPort; +import dev.caskeleton.application.fileserver.api.metadata.FileRecord; +import dev.caskeleton.application.fileserver.api.metadata.FileRecordMutation; +import java.util.UUID; +import java.util.function.Supplier; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * What every {@link ContentReferenceLedgerPort} promises, run against the in-memory metadata store + * and the JPA ledger on real PostgreSQL (review 2026-11-03 RR-01-04). + * + *

    A key is referenced while any record names it, in any state: an orphan scan that deleted the + * object of a record mid-delete would race the cleanup worker. + */ +public abstract class ContentReferenceLedgerPortContract { + + protected abstract ContentReferenceLedgerPort ledger(); + + /** The metadata store the ledger answers from, to record files with. */ + protected abstract FileMetadataPort metadata(); + + /** Runs one store call inside the caller transaction the adapter requires, and commits it. */ + protected T inTransaction(Supplier work) { + return work.get(); + } + + @Test + @DisplayName("어느 레코드도 가리키지 않는 키는 참조되지 않는다") + void aKeyNoRecordNamesIsNotReferenced() { + // given + ContentKey key = newKey(); + + // when + boolean referenced = inTransaction(() -> ledger().isReferenced(key)); + + // then + assertThat(referenced).isFalse(); + } + + @Test + @DisplayName("공개된 레코드가 가리키는 키는 참조된다") + void aKeyAPublishedRecordNamesIsReferenced() { + // given + ContentKey key = newKey(); + publish(key); + + // when + boolean referenced = inTransaction(() -> ledger().isReferenced(key)); + + // then + assertThat(referenced).isTrue(); + } + + @Test + @DisplayName("지우는 중인 레코드가 가리키는 키도 참조된다") + void aKeyStaysReferencedWhileItsRecordIsBeingDeleted() { + // given + ContentKey key = newKey(); + FileRecord ready = publish(key); + inTransaction(() -> metadata().markDeleting(ready.fileId(), ready.version())); + + // when + boolean referenced = inTransaction(() -> ledger().isReferenced(key)); + + // then + assertThat(referenced).isTrue(); + } + + /** Walks a fresh record to READY under {@code key} through the edges an upload takes. */ + private FileRecord publish(ContentKey key) { + FileId fileId = FileId.of(UUID.randomUUID()); + return inTransaction( + () -> { + FileRecord created = metadata().insert(FileserverFixtures.draft(fileId, 10)); + FileRecord uploading = + metadata() + .transition( + fileId, + created.version(), + FileState.CREATED, + FileState.UPLOADING, + FileRecordMutation.none()); + FileRecord uploaded = + metadata() + .transition( + fileId, + uploading.version(), + FileState.UPLOADING, + FileState.UPLOADED, + FileRecordMutation.uploaded(10, FileserverFixtures.DIGEST)); + FileRecord verifying = + metadata() + .transition( + fileId, + uploaded.version(), + FileState.UPLOADED, + FileState.VERIFYING, + FileRecordMutation.none()); + return metadata() + .transition( + fileId, + verifying.version(), + FileState.VERIFYING, + FileState.READY, + FileRecordMutation.publishAt( + key, 10, FileserverFixtures.DIGEST, FileserverFixtures.NOW)); + }); + } + + private static ContentKey newKey() { + String flat = UUID.randomUUID().toString().replace("-", ""); + return ContentKey.of(flat.substring(0, 2) + '/' + flat.substring(2, 4) + '/' + flat); + } +} diff --git a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeFileAccessPolicy.java b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeFileAccessPolicy.java index 4c00435f..7e4bc000 100644 --- a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeFileAccessPolicy.java +++ b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeFileAccessPolicy.java @@ -18,7 +18,9 @@ import java.util.Set; * *

    It keeps the one rule every production policy owes the port: an operation on an existing file * is refused unless the caller created it, in the same tenant (owner decision 14). Without it a use - * case that stopped passing the file's descriptor would still pass every test. + * case that stopped passing the file's descriptor would still pass every test. The rule is {@link + * FileAccessSubject#owns}, the same call the production policy makes, and {@link + * FileAccessPolicyPortContract} holds both to it. */ public final class FakeFileAccessPolicy implements FileAccessPolicyPort { @@ -42,8 +44,7 @@ public final class FakeFileAccessPolicy implements FileAccessPolicyPort { "operation denied by policy", FileserverFailureContext.of(FileserverErrorCode.ACCESS_DENIED, false)); } - if (descriptor.isPresent() - && !descriptor.get().owner().equals(Optional.of(subject.asOwner()))) { + if (descriptor.isPresent() && !subject.owns(descriptor.get())) { throw new FileserverException( "subject does not own the file", FileserverFailureContext.of(FileserverErrorCode.ACCESS_DENIED, false)); diff --git a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeFileQuotaService.java b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeFileQuotaService.java index 3e3170db..7a5e189b 100644 --- a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeFileQuotaService.java +++ b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeFileQuotaService.java @@ -1,84 +1,115 @@ package dev.caskeleton.application.fileserver.testkit; +import dev.caskeleton.application.fileserver.api.error.FileserverErrorCode; +import dev.caskeleton.application.fileserver.api.error.FileserverException; +import dev.caskeleton.application.fileserver.api.error.FileserverFailureContext; import dev.caskeleton.application.fileserver.api.metadata.FileQuotaPort; import dev.caskeleton.application.fileserver.api.metadata.QuotaReservation; import dev.caskeleton.application.fileserver.api.metadata.QuotaReservationStatus; import dev.caskeleton.application.fileserver.api.metadata.QuotaScope; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Comparator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; /** - * Hand-rolled quota service recording reservations, commits, and releases. + * The in-memory quota ledger: the fake of {@link FileQuotaPort}, and the ledger that {@link + * FakeQuotaCommitGateway} and {@link RecordingQuotaReclaimGateway} write, as the three JPA adapters + * share one reservation table. * - *

    It keeps the ledger the durable adapter keeps: a live reservation counts toward its scope's - * usage until it is committed (then the committed bytes count instead) or released (then nothing - * counts). That is what a configured namespace limit is compared against. + *

    It keeps the rows the durable adapters keep and answers the way they do, which {@link + * FileQuotaLedgerContract} holds it to on the same cases as PostgreSQL: a live reservation counts + * until it expires, is committed (then the committed bytes count) or is released (then nothing + * counts); an expired or released reservation cannot be extended and a released one cannot be + * committed ({@code QUOTA_EXCEEDED}). Time is {@link FileserverFixtures#NOW} until {@link + * #advanceTo} moves it. + * + *

    It also records what it was asked, for use-case tests that check which calls a flow made. */ public final class FakeFileQuotaService implements FileQuotaPort { + private final Map rows = new LinkedHashMap<>(); private final List reserved = new ArrayList<>(); private final List released = new ArrayList<>(); private final List committed = new ArrayList<>(); - private final Map liveBytes = new HashMap<>(); - private final Map scopes = new HashMap<>(); - private final Map committedUsage = new HashMap<>(); + private Instant now = FileserverFixtures.NOW; + private long writes; + + /** Moves the ledger's clock; reservations whose TTL ends by then stop counting. */ + public void advanceTo(Instant instant) { + this.now = instant; + } /** Books usage the scope already holds, as if earlier uploads had committed it. */ public void alreadyHolds(QuotaScope scope, long bytes) { - committedUsage.merge(scope, bytes, Long::sum); + Row row = newRow(scope, bytes, now); + row.commit(bytes, ++writes); } @Override public QuotaReservation reserve(QuotaScope scope, long expectedBytes, Duration ttl) { - QuotaReservation reservation = - new QuotaReservation( - UUID.randomUUID(), - scope, - expectedBytes, - 0, - FileserverFixtures.NOW.plus(ttl), - QuotaReservationStatus.RESERVED, - 0); + requireNotNegative(expectedBytes, "expectedBytes"); + Row row = newRow(scope, expectedBytes, now.plus(ttl)); + QuotaReservation reservation = row.view(); reserved.add(reservation); - liveBytes.put(reservation.reservationId(), expectedBytes); - scopes.put(reservation.reservationId(), scope); return reservation; } @Override public void extend(QuotaReservation reservation, long additionalBytes) { + requireNotNegative(additionalBytes, "additionalBytes"); // The recorded reservation identity stays the same; the live total grows as the durable // adapter's does. reserved.add(reservation); - liveBytes.computeIfPresent(reservation.reservationId(), (id, bytes) -> bytes + additionalBytes); + Row row = rows.get(reservation.reservationId()); + if (row == null || !row.isLiveAt(now)) { + throw quotaConflict("reservation is no longer extendable"); + } + row.reservedBytes += additionalBytes; + row.updatedAt = ++writes; } @Override public void commit(QuotaReservation reservation, long actualBytes) { + requireNotNegative(actualBytes, "actualBytes"); committed.add(actualBytes); - if (liveBytes.remove(reservation.reservationId()) != null) { - committedUsage.merge(scopes.get(reservation.reservationId()), actualBytes, Long::sum); + Row row = rows.get(reservation.reservationId()); + // Expiry does not stop a commit: bytes that were written are durable usage whether or not the + // reservation outlived its TTL. Only a settled reservation is refused. + if (row == null || row.status != QuotaReservationStatus.RESERVED) { + throw quotaConflict("reservation is no longer committable"); } + row.commit(actualBytes, ++writes); } @Override public void release(QuotaReservation reservation) { released.add(reservation); - liveBytes.remove(reservation.reservationId()); + Row row = rows.get(reservation.reservationId()); + if (row != null && row.status == QuotaReservationStatus.RESERVED) { + row.release(++writes); + } } @Override public long usedBytes(QuotaScope scope) { - long live = - liveBytes.entrySet().stream() - .filter(entry -> scope.equals(scopes.get(entry.getKey()))) - .mapToLong(Map.Entry::getValue) - .sum(); - return live + committedUsage.getOrDefault(scope, 0L); + long total = 0; + for (Row row : rows.values()) { + if (!row.scope.equals(scope)) { + continue; + } + if (row.isLiveAt(now)) { + total += row.reservedBytes; + } else if (row.status == QuotaReservationStatus.COMMITTED) { + total += row.committedBytes; + } + } + return total; } public List reservedScopes() { @@ -96,4 +127,113 @@ public final class FakeFileQuotaService implements FileQuotaPort { public List committedBytes() { return List.copyOf(committed); } + + /** + * Settles an upload's share as {@code JpaQuotaCommit} does: the scope's oldest live reservation + * is committed, and with none left the bytes are still booked as committed usage. + */ + void settleOldestLive(QuotaScope scope, long actualBytes) { + requireNotNegative(actualBytes, "actualBytes"); + Optional oldest = oldestLive(scope); + if (oldest.isPresent()) { + oldest.get().commit(actualBytes, ++writes); + } else if (actualBytes > 0) { + newRow(scope, actualBytes, now).commit(actualBytes, ++writes); + } + } + + /** Releases the scope's oldest live reservation, as {@code JpaQuotaCommit.release} does. */ + void releaseOldestLive(QuotaScope scope) { + oldestLive(scope).ifPresent(row -> row.release(++writes)); + } + + /** + * Gives committed bytes back as {@code JpaQuotaReclaim} does: newest committed rows first, never + * below zero, a remainder no row can absorb dropped. Live reservations are not touched. + */ + void reclaim(QuotaScope scope, long bytes) { + requireNotNegative(bytes, "bytes"); + long outstanding = bytes; + List newestFirst = + rows.values().stream() + .filter(row -> row.scope.equals(scope)) + .filter(row -> row.status == QuotaReservationStatus.COMMITTED) + .filter(row -> row.committedBytes > 0) + .sorted(Comparator.comparingLong((Row row) -> row.updatedAt).reversed()) + .toList(); + for (Row row : newestFirst) { + if (outstanding == 0) { + return; + } + long draw = Math.min(outstanding, row.committedBytes); + row.committedBytes -= draw; + row.updatedAt = ++writes; + outstanding -= draw; + } + } + + private Optional oldestLive(QuotaScope scope) { + return rows.values().stream() + .filter(row -> row.scope.equals(scope)) + .filter(row -> row.isLiveAt(now)) + .findFirst(); + } + + private Row newRow(QuotaScope scope, long reservedBytes, Instant expiresAt) { + Row row = new Row(UUID.randomUUID(), scope, reservedBytes, expiresAt, ++writes); + rows.put(row.id, row); + return row; + } + + private static void requireNotNegative(long bytes, String name) { + if (bytes < 0) { + throw new IllegalArgumentException(name + " must not be negative"); + } + } + + private static FileserverException quotaConflict(String message) { + return new FileserverException( + message, FileserverFailureContext.of(FileserverErrorCode.QUOTA_EXCEEDED, false)); + } + + /** One reservation row, in insertion order; {@code updatedAt} is a write counter. */ + private static final class Row { + + private final UUID id; + private final QuotaScope scope; + private final Instant expiresAt; + private long reservedBytes; + private long committedBytes; + private QuotaReservationStatus status = QuotaReservationStatus.RESERVED; + private long updatedAt; + + private Row(UUID id, QuotaScope scope, long reservedBytes, Instant expiresAt, long createdAt) { + this.id = id; + this.scope = scope; + this.reservedBytes = reservedBytes; + this.expiresAt = expiresAt; + this.updatedAt = createdAt; + } + + private boolean isLiveAt(Instant instant) { + return status == QuotaReservationStatus.RESERVED && expiresAt.isAfter(instant); + } + + private void commit(long actualBytes, long write) { + status = QuotaReservationStatus.COMMITTED; + committedBytes = actualBytes; + reservedBytes = 0; + updatedAt = write; + } + + private void release(long write) { + status = QuotaReservationStatus.RELEASED; + reservedBytes = 0; + updatedAt = write; + } + + private QuotaReservation view() { + return new QuotaReservation(id, scope, reservedBytes, committedBytes, expiresAt, status, 0); + } + } } diff --git a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeQuotaCommitGateway.java b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeQuotaCommitGateway.java index cd6f422c..053baecc 100644 --- a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeQuotaCommitGateway.java +++ b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeQuotaCommitGateway.java @@ -1,24 +1,57 @@ package dev.caskeleton.application.fileserver.testkit; +import dev.caskeleton.application.fileserver.api.metadata.FileMetadataPort; +import dev.caskeleton.application.fileserver.api.metadata.FileRecord; +import dev.caskeleton.application.fileserver.api.metadata.QuotaScope; import dev.caskeleton.application.fileserver.api.metadata.UploadSession; import dev.caskeleton.application.fileserver.upload.QuotaCommitPort; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Objects; -/** Hand-rolled quota gateway recording committed and released byte counts. */ +/** + * The fake of {@link QuotaCommitPort}: settles an upload's share of its file's namespace on the + * in-memory ledger, as {@code JpaQuotaCommit} does on the reservation table, and records what it + * was asked. A session whose file has no record settles nothing. {@link FileQuotaLedgerContract} + * holds it to the JPA adapter's cases. + */ public final class FakeQuotaCommitGateway implements QuotaCommitPort { + private final FakeFileQuotaService ledger; + private final FileMetadataPort metadata; private final Map committed = new LinkedHashMap<>(); private final Map released = new LinkedHashMap<>(); + /** + * A gateway over {@code ledger}, resolving each session's namespace from {@code metadata}. + * + * @param ledger the ledger the reservations live in + * @param metadata the records the sessions' files are found in + */ + public FakeQuotaCommitGateway(FakeFileQuotaService ledger, FileMetadataPort metadata) { + this.ledger = Objects.requireNonNull(ledger, "ledger"); + this.metadata = Objects.requireNonNull(metadata, "metadata"); + } + @Override public void commit(UploadSession session, long actualBytes) { + if (actualBytes < 0) { + throw new IllegalArgumentException("actualBytes must not be negative"); + } committed.put(session, actualBytes); + metadata + .find(session.fileId()) + .map(FakeQuotaCommitGateway::scopeOf) + .ifPresent(scope -> ledger.settleOldestLive(scope, actualBytes)); } @Override public void release(UploadSession session) { released.put(session, Boolean.TRUE); + metadata + .find(session.fileId()) + .map(FakeQuotaCommitGateway::scopeOf) + .ifPresent(ledger::releaseOldestLive); } public long committedBytes() { @@ -32,4 +65,8 @@ public final class FakeQuotaCommitGateway implements QuotaCommitPort { public boolean anyReleased() { return !released.isEmpty(); } + + private static QuotaScope scopeOf(FileRecord record) { + return QuotaScope.ofNamespace(record.namespace().value()); + } } diff --git a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FileAccessPolicyPortContract.java b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FileAccessPolicyPortContract.java new file mode 100644 index 00000000..5288a309 --- /dev/null +++ b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FileAccessPolicyPortContract.java @@ -0,0 +1,175 @@ +package dev.caskeleton.application.fileserver.testkit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.api.FileState; +import dev.caskeleton.application.fileserver.api.error.FileserverErrorCode; +import dev.caskeleton.application.fileserver.api.error.FileserverException; +import dev.caskeleton.application.fileserver.api.metadata.FileDescriptor; +import dev.caskeleton.application.fileserver.api.metadata.FileOwner; +import dev.caskeleton.application.fileserver.api.security.FileAccessPolicyPort; +import dev.caskeleton.application.fileserver.api.security.FileAccessSubject; +import dev.caskeleton.application.fileserver.api.security.FileOperation; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The ownership rule every {@link FileAccessPolicyPort} owes the port (owner decision 14), run + * against the fake and every production policy alike. + * + *

    The rule itself is written once, {@link FileAccessSubject#owns(FileDescriptor)}; this contract + * is what keeps a policy from dropping or loosening its use of it. Use-case tests that prove a + * stranger cannot reach a file run on the fake, so they only mean something while the fake refuses + * exactly what the production policy refuses (review 2026-11-03 RR-07-06). + * + *

    Roles are each policy's own business; a subclass hands the contract a subject its role rules + * let through every data-plane operation, and the cases vary only who owns the file. + */ +public abstract class FileAccessPolicyPortContract { + + /** The policy under test. */ + protected abstract FileAccessPolicyPort policy(); + + /** + * A subject the policy's role rules permit for every data-plane operation. + * + * @param principalId the caller + * @param tenant the caller's tenant, empty when the request was not tenant scoped + * @return that caller + */ + protected abstract FileAccessSubject permittedSubject( + String principalId, Optional tenant); + + @Test + @DisplayName("소유자는 자기 파일을 내려받고 지울 수 있다") + void theOwnerMayDownloadAndDeleteTheirFile() { + // given + FileAccessSubject owner = permittedSubject("user-1", Optional.of("tenant-1")); + Optional file = fileOwnedBy("user-1", Optional.of("tenant-1")); + + // when + Throwable refused = + catchThrowable( + () -> { + policy().authorize(FileOperation.DOWNLOAD, owner, file); + policy().authorize(FileOperation.DELETE, owner, file); + }); + + // then + assertThat(refused).isNull(); + } + + @Test + @DisplayName("같은 역할을 가진 다른 사용자의 파일은 ACCESS_DENIED다") + void anotherPrincipalsFileIsRefused() { + // given + FileAccessSubject stranger = permittedSubject("user-2", Optional.of("tenant-1")); + + // when + Throwable refused = + catchThrowable( + () -> + policy() + .authorize( + FileOperation.DOWNLOAD, + stranger, + fileOwnedBy("user-1", Optional.of("tenant-1")))); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.ACCESS_DENIED); + } + + @Test + @DisplayName("같은 사용자라도 다른 tenant에서 만든 파일은 ACCESS_DENIED다") + void theSamePrincipalInAnotherTenantIsRefused() { + // given + FileAccessSubject elsewhere = permittedSubject("user-1", Optional.of("tenant-2")); + + // when + Throwable refused = + catchThrowable( + () -> + policy() + .authorize( + FileOperation.DOWNLOAD, + elsewhere, + fileOwnedBy("user-1", Optional.of("tenant-1")))); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.ACCESS_DENIED); + } + + @Test + @DisplayName("tenant 없이 만든 파일은 tenant가 있는 같은 사용자에게도 ACCESS_DENIED다") + void aFileCreatedWithoutATenantIsRefusedToTheSamePrincipalWithOne() { + // given + FileAccessSubject scoped = permittedSubject("user-1", Optional.of("tenant-1")); + + // when + Throwable refused = + catchThrowable( + () -> + policy() + .authorize( + FileOperation.DELETE, scoped, fileOwnedBy("user-1", Optional.empty()))); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.ACCESS_DENIED); + } + + @Test + @DisplayName("소유자가 기록되지 않은 파일은 누구에게도 ACCESS_DENIED다") + void aFileWithNoRecordedOwnerIsRefused() { + // given + FileAccessSubject caller = permittedSubject("user-1", Optional.empty()); + + // when + Throwable refused = + catchThrowable( + () -> policy().authorize(FileOperation.DOWNLOAD, caller, descriptor(Optional.empty()))); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.ACCESS_DENIED); + } + + @Test + @DisplayName("파일이 생기기 전의 연산은 소유자 없이 역할로만 판단한다") + void anOperationBeforeTheFileExistsIsDecidedWithoutAnOwner() { + // given + FileAccessSubject caller = permittedSubject("user-1", Optional.of("tenant-1")); + + // when + Throwable refused = + catchThrowable(() -> policy().authorize(FileOperation.CREATE, caller, Optional.empty())); + + // then + assertThat(refused).isNull(); + } + + private static Optional fileOwnedBy(String principalId, Optional tenant) { + return descriptor(Optional.of(new FileOwner(principalId, tenant))); + } + + private static Optional descriptor(Optional owner) { + return Optional.of( + new FileDescriptor( + FileId.of(UUID.randomUUID()), + FileserverFixtures.NAMESPACE, + FileState.READY, + "report.bin", + "application/octet-stream", + 10, + FileserverFixtures.DIGEST, + FileserverFixtures.NOW, + owner, + 3)); + } + + private static Object codeOf(Throwable failure) { + return failure instanceof FileserverException refused ? refused.code() : failure; + } +} diff --git a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FileQuotaLedgerContract.java b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FileQuotaLedgerContract.java new file mode 100644 index 00000000..b217dad1 --- /dev/null +++ b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FileQuotaLedgerContract.java @@ -0,0 +1,346 @@ +package dev.caskeleton.application.fileserver.testkit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.fileserver.api.StorageNamespace; +import dev.caskeleton.application.fileserver.api.error.FileserverErrorCode; +import dev.caskeleton.application.fileserver.api.error.FileserverException; +import dev.caskeleton.application.fileserver.api.metadata.FileQuotaPort; +import dev.caskeleton.application.fileserver.api.metadata.QuotaReservation; +import dev.caskeleton.application.fileserver.api.metadata.QuotaScope; +import dev.caskeleton.application.fileserver.api.metadata.UploadSession; +import dev.caskeleton.application.fileserver.cleanup.QuotaReclaimPort; +import dev.caskeleton.application.fileserver.upload.QuotaCommitPort; +import java.time.Duration; +import java.util.List; +import java.util.UUID; +import java.util.function.Supplier; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The quota ledger, through all three ports that write it: {@link FileQuotaPort} reserves and + * settles reservations, {@link QuotaCommitPort} settles an upload's share, {@link QuotaReclaimPort} + * gives committed bytes back. They are one contract because they are one ledger — what a commit or + * a reclaim did is only visible as {@link FileQuotaPort#usedBytes}, the number a configured + * namespace limit is compared against (review 2026-11-03 RR-01-04). + * + *

    Run against the in-memory ledger and the JPA adapters on real PostgreSQL. Every test works in + * namespaces of its own, so a subclass may share one store across tests. + */ +public abstract class FileQuotaLedgerContract { + + private static final Duration TTL = Duration.ofHours(1); + + protected abstract FileQuotaPort quota(); + + protected abstract QuotaCommitPort commits(); + + protected abstract QuotaReclaimPort reclaims(); + + /** + * An upload of a file recorded in {@code namespace}, as the finalize path holds one: the commit + * port settles against the namespace of the file the session belongs to. + */ + protected abstract UploadSession uploadIn(StorageNamespace namespace); + + /** Lets {@code duration} pass on the clock the ledger reads. */ + protected abstract void elapse(Duration duration); + + /** Runs one ledger call inside the caller transaction the adapters require, and commits it. */ + protected T inTransaction(Supplier work) { + return work.get(); + } + + @Test + @DisplayName("살아 있는 예약은 그 scope의 사용량에 들어간다") + void aLiveReservationCountsTowardItsScope() { + // given + QuotaScope scope = newScope(); + + // when + reserve(scope, 100); + + // then + assertThat(usedBytes(scope)).isEqualTo(100); + } + + @Test + @DisplayName("예약을 늘리면 늘린 만큼 사용량이 커진다") + void extendingAReservationGrowsTheUsage() { + // given + QuotaScope scope = newScope(); + QuotaReservation reservation = reserve(scope, 100); + + // when + run(() -> quota().extend(reservation, 50)); + + // then + assertThat(usedBytes(scope)).isEqualTo(150); + } + + @Test + @DisplayName("확정하면 예약한 바이트 대신 실제 바이트가 사용량이 된다") + void committingCountsTheActualBytesInsteadOfTheReserved() { + // given + QuotaScope scope = newScope(); + QuotaReservation reservation = reserve(scope, 100); + + // when + run(() -> quota().commit(reservation, 60)); + + // then + assertThat(usedBytes(scope)).isEqualTo(60); + } + + @Test + @DisplayName("해제한 예약은 사용량에서 빠진다") + void releasingAReservationGivesItBack() { + // given + QuotaScope scope = newScope(); + QuotaReservation reservation = reserve(scope, 100); + + // when + run(() -> quota().release(reservation)); + + // then + assertThat(usedBytes(scope)).isZero(); + } + + @Test + @DisplayName("만료된 예약은 사용량에서 빠진다") + void anExpiredReservationNoLongerCounts() { + // given + QuotaScope scope = newScope(); + reserve(scope, 100); + + // when + elapse(TTL.plusMinutes(1)); + + // then + assertThat(usedBytes(scope)).isZero(); + } + + @Test + @DisplayName("만료된 예약은 늘릴 수 없고 QUOTA_EXCEEDED로 거절된다") + void anExpiredReservationCannotBeExtended() { + // given + QuotaReservation reservation = reserve(newScope(), 100); + elapse(TTL.plusMinutes(1)); + + // when + Throwable refused = catchThrowable(() -> run(() -> quota().extend(reservation, 10))); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.QUOTA_EXCEEDED); + } + + @Test + @DisplayName("해제한 예약은 늘릴 수 없고 QUOTA_EXCEEDED로 거절된다") + void aReleasedReservationCannotBeExtended() { + // given + QuotaReservation reservation = reserve(newScope(), 100); + run(() -> quota().release(reservation)); + + // when + Throwable refused = catchThrowable(() -> run(() -> quota().extend(reservation, 10))); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.QUOTA_EXCEEDED); + } + + @Test + @DisplayName("해제한 예약은 확정할 수 없고 QUOTA_EXCEEDED로 거절된다") + void aReleasedReservationCannotBeCommitted() { + // given + QuotaReservation reservation = reserve(newScope(), 100); + run(() -> quota().release(reservation)); + + // when + Throwable refused = catchThrowable(() -> run(() -> quota().commit(reservation, 60))); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.QUOTA_EXCEEDED); + } + + /** + * Bytes that were written are durable usage whether or not the reservation outlived its TTL; + * dropping them because the reservation expired is how a ledger drifts below the truth. + */ + @Test + @DisplayName("만료됐지만 해제되지 않은 예약을 확정하면 실제 바이트가 사용량이 된다") + void committingAnExpiredReservationStillRecordsTheActualBytes() { + // given + QuotaScope scope = newScope(); + QuotaReservation reservation = reserve(scope, 100); + elapse(TTL.plusMinutes(1)); + + // when + run(() -> quota().commit(reservation, 60)); + + // then + assertThat(usedBytes(scope)).isEqualTo(60); + } + + @Test + @DisplayName("scope마다 사용량을 따로 센다") + void eachScopeIsCountedOnItsOwn() { + // given + QuotaScope scope = newScope(); + reserve(scope, 100); + + // when + reserve(newScope(), 30); + + // then + assertThat(usedBytes(scope)).isEqualTo(100); + } + + @Test + @DisplayName("음수 바이트 예약은 IllegalArgumentException으로 거절된다") + void aNegativeReservationIsRefused() { + // when + Throwable refused = catchThrowable(() -> reserve(newScope(), -1)); + + // then + assertThat(refused).isInstanceOf(IllegalArgumentException.class); + } + + @Test + @DisplayName("업로드 확정은 그 파일 namespace의 살아 있는 예약을 실제 바이트로 확정한다") + void settlingAnUploadCommitsItsNamespacesLiveReservation() { + // given + StorageNamespace namespace = newNamespace(); + UploadSession upload = uploadIn(namespace); + reserve(scopeOf(namespace), 1_000); + + // when + run(() -> commits().commit(upload, 600)); + + // then + assertThat(usedBytes(scopeOf(namespace))).isEqualTo(600); + } + + @Test + @DisplayName("살아 있는 예약이 없어도 업로드 확정은 실제 바이트를 사용량으로 남긴다") + void settlingAnUploadWithNoLiveReservationStillRecordsTheUsage() { + // given + StorageNamespace namespace = newNamespace(); + UploadSession upload = uploadIn(namespace); + + // when + run(() -> commits().commit(upload, 450)); + + // then + assertThat(usedBytes(scopeOf(namespace))).isEqualTo(450); + } + + @Test + @DisplayName("업로드 해제는 그 파일 namespace의 살아 있는 예약을 돌려준다") + void releasingAnUploadGivesItsNamespacesReservationBack() { + // given + StorageNamespace namespace = newNamespace(); + UploadSession upload = uploadIn(namespace); + reserve(scopeOf(namespace), 2_000); + + // when + run(() -> commits().release(upload)); + + // then + assertThat(usedBytes(scopeOf(namespace))).isZero(); + } + + @Test + @DisplayName("회수는 여러 확정분에 걸쳐 확정 사용량을 줄인다") + void reclaimingDrawsCommittedUsageDownAcrossCommits() { + // given + QuotaScope scope = newScope(); + commit(scope, 300); + commit(scope, 700); + + // when + run(() -> reclaims().reclaim(scope, 800)); + + // then + assertThat(usedBytes(scope)).isEqualTo(200); + } + + @Test + @DisplayName("기록보다 많이 회수하면 사용량은 0에서 멈춘다") + void reclaimingMoreThanIsRecordedStopsAtZero() { + // given + QuotaScope scope = newScope(); + commit(scope, 100); + + // when + run(() -> reclaims().reclaim(scope, 5_000)); + + // then + assertThat(usedBytes(scope)).isZero(); + } + + @Test + @DisplayName("회수는 살아 있는 예약과 다른 scope를 건드리지 않는다") + void reclaimingLeavesLiveReservationsAndOtherScopesAlone() { + // given + QuotaScope scope = newScope(); + QuotaScope other = newScope(); + reserve(scope, 100); + commit(scope, 50); + commit(other, 70); + + // when + run(() -> reclaims().reclaim(scope, 500)); + + // then + assertThat(List.of(usedBytes(scope), usedBytes(other))).containsExactly(100L, 70L); + } + + @Test + @DisplayName("음수 바이트 회수는 IllegalArgumentException으로 거절된다") + void aNegativeReclaimIsRefused() { + // when + Throwable refused = catchThrowable(() -> run(() -> reclaims().reclaim(newScope(), -1))); + + // then + assertThat(refused).isInstanceOf(IllegalArgumentException.class); + } + + private QuotaReservation reserve(QuotaScope scope, long bytes) { + return inTransaction(() -> quota().reserve(scope, bytes, TTL)); + } + + private void commit(QuotaScope scope, long bytes) { + QuotaReservation reservation = reserve(scope, bytes); + run(() -> quota().commit(reservation, bytes)); + } + + private long usedBytes(QuotaScope scope) { + return inTransaction(() -> quota().usedBytes(scope)); + } + + private void run(Runnable work) { + inTransaction( + () -> { + work.run(); + return null; + }); + } + + private static StorageNamespace newNamespace() { + return StorageNamespace.of("q-" + UUID.randomUUID().toString().substring(0, 12)); + } + + private static QuotaScope newScope() { + return scopeOf(newNamespace()); + } + + private static QuotaScope scopeOf(StorageNamespace namespace) { + return QuotaScope.ofNamespace(namespace.value()); + } + + private static Object codeOf(Throwable failure) { + return failure instanceof FileserverException refused ? refused.code() : failure; + } +} diff --git a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryFileMetadataStore.java b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryFileMetadataStore.java index f55836d9..dd00e088 100644 --- a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryFileMetadataStore.java +++ b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryFileMetadataStore.java @@ -1,5 +1,7 @@ package dev.caskeleton.application.fileserver.testkit; +import dev.caskeleton.application.fileserver.admin.ContentReferenceLedgerPort; +import dev.caskeleton.application.fileserver.api.ContentKey; import dev.caskeleton.application.fileserver.api.FileId; import dev.caskeleton.application.fileserver.api.FileState; import dev.caskeleton.application.fileserver.api.StorageNamespace; @@ -25,8 +27,13 @@ import java.util.concurrent.atomic.AtomicInteger; *

    Application tests must observe identical conflict behaviour, so this fake enforces the * expected state and version rather than accepting any write. It is held to {@link * FileMetadataPortContract}, the contract the JPA adapter passes against PostgreSQL. + * + *

    Also answers {@link ContentReferenceLedgerPort} from the same records, as the JPA ledger reads + * the file table: a key is referenced while a record in any state names it ({@link + * ContentReferenceLedgerPortContract}). */ -public final class InMemoryFileMetadataStore implements FileMetadataPort { +public final class InMemoryFileMetadataStore + implements FileMetadataPort, ContentReferenceLedgerPort { private final Map records = new LinkedHashMap<>(); private final AtomicInteger transitionFailuresToInject = new AtomicInteger(); @@ -70,6 +77,12 @@ public final class InMemoryFileMetadataStore implements FileMetadataPort { return record; } + @Override + public boolean isReferenced(ContentKey key) { + return records.values().stream() + .anyMatch(record -> record.contentKey().filter(key::equals).isPresent()); + } + @Override public Optional find(FileId fileId) { return Optional.ofNullable(records.get(fileId)); diff --git a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryRecoveryQueue.java b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryRecoveryQueue.java index 61a526bb..8de863df 100644 --- a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryRecoveryQueue.java +++ b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryRecoveryQueue.java @@ -8,7 +8,12 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -/** Hand-rolled recovery queue that records every escalation reason. */ +/** + * Hand-rolled recovery queue that records every escalation reason. + * + *

    Answers as the JPA queue does ({@link RecoveryQueuePortContract}): one open entry per file, + * oldest first, resolved entries out of the pending list and raisable again. + */ public final class InMemoryRecoveryQueue implements RecoveryQueuePort { private final Map pending = new LinkedHashMap<>(); @@ -21,6 +26,9 @@ public final class InMemoryRecoveryQueue implements RecoveryQueuePort { @Override public List pending(int limit) { + if (limit < 1) { + throw new IllegalArgumentException("limit must be positive"); + } return List.copyOf( new ArrayList<>(pending.keySet()).subList(0, Math.min(limit, pending.size()))); } diff --git a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryUploadSessionStore.java b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryUploadSessionStore.java index 3a1be230..abc28e98 100644 --- a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryUploadSessionStore.java +++ b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryUploadSessionStore.java @@ -12,7 +12,7 @@ import dev.caskeleton.application.fileserver.api.metadata.WriterLease; import dev.caskeleton.application.fileserver.recovery.StagingUploadLocatorPort; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -28,6 +28,11 @@ import java.util.UUID; * *

    Also answers {@link StagingUploadLocatorPort} from the same sessions, as the JPA locator reads * the session table: the newest session recorded for a file. + * + *

    {@link UploadSessionPortContract} and {@link StagingUploadLocatorPortContract} hold it to the + * JPA store's cases on PostgreSQL, including the two that read time: no lease on an upload past its + * expiry, and no offset commit once the writer's lease has lapsed ({@link #advanceTo} moves the + * store's clock). */ public final class InMemoryUploadSessionStore implements UploadSessionPort, StagingUploadLocatorPort { @@ -107,7 +112,8 @@ public final class InMemoryUploadSessionStore UploadSession current = require(uploadId); boolean leaseFree = current.leaseUntil().isEmpty() || !current.leaseUntil().get().isAfter(clockNow); - if (current.terminal() || current.version() != expectedVersion || !leaseFree) { + boolean uploadOpen = current.expiresAt().isAfter(clockNow); + if (current.terminal() || current.version() != expectedVersion || !leaseFree || !uploadOpen) { throw new FileserverException( "writer lease is held by another owner", FileserverFailureContext.forUpload( @@ -156,7 +162,13 @@ public final class InMemoryUploadSessionStore UploadId uploadId, WriterLease lease, long expectedOffset, long committedOffset) { UploadSession current = require(uploadId); boolean tokenMatches = current.leaseToken().map(lease.token()::equals).orElse(false); - if (current.terminal() || !tokenMatches || current.committedOffset() != expectedOffset) { + // The JPA statement reads its own clock here, not the caller's: a writer whose lease lapsed + // cannot advance the offset even when nobody has taken the lease over yet. + boolean stillHeld = current.leaseUntil().map(now::isBefore).orElse(false); + if (current.terminal() + || !tokenMatches + || !stillHeld + || current.committedOffset() != expectedOffset) { throw new FileserverException( "offset commit rejected", FileserverFailureContext.forOffset( @@ -222,13 +234,13 @@ public final class InMemoryUploadSessionStore @Override public List findExpired(Instant cutoff, int limit) { - List expired = new ArrayList<>(); - for (UploadSession session : sessions.values()) { - if (!session.terminal() && session.isExpiredAt(cutoff) && expired.size() < limit) { - expired.add(session); - } - } - return List.copyOf(expired); + // Strictly before the cutoff and earliest expiry first, as the JPA query reads them. + return sessions.values().stream() + .filter(session -> !session.terminal()) + .filter(session -> session.expiresAt().isBefore(cutoff)) + .sorted(Comparator.comparing(UploadSession::expiresAt)) + .limit(limit) + .toList(); } private UploadSession require(UploadId uploadId) { diff --git a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/RecordingQuotaReclaimGateway.java b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/RecordingQuotaReclaimGateway.java index b7369c7a..25001840 100644 --- a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/RecordingQuotaReclaimGateway.java +++ b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/RecordingQuotaReclaimGateway.java @@ -4,12 +4,28 @@ import dev.caskeleton.application.fileserver.api.metadata.QuotaScope; import dev.caskeleton.application.fileserver.cleanup.QuotaReclaimPort; import java.util.ArrayList; import java.util.List; +import java.util.Objects; -/** Records every reclaim so a test can prove capacity is returned exactly once. */ +/** + * The fake of {@link QuotaReclaimPort}: gives committed bytes back on the in-memory ledger as + * {@code JpaQuotaReclaim} does (newest first, floor zero), and records every reclaim so a test can + * prove capacity is returned exactly once. {@link FileQuotaLedgerContract} holds it to the JPA + * adapter's cases. + */ public final class RecordingQuotaReclaimGateway implements QuotaReclaimPort { + private final FakeFileQuotaService ledger; private final List reclaimed = new ArrayList<>(); + /** + * A gateway over {@code ledger}. + * + * @param ledger the ledger the committed usage lives in + */ + public RecordingQuotaReclaimGateway(FakeFileQuotaService ledger) { + this.ledger = Objects.requireNonNull(ledger, "ledger"); + } + public List reclaimed() { return List.copyOf(reclaimed); } @@ -20,6 +36,7 @@ public final class RecordingQuotaReclaimGateway implements QuotaReclaimPort { @Override public void reclaim(QuotaScope scope, long bytes) { + ledger.reclaim(scope, bytes); reclaimed.add(bytes); } } diff --git a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/RecoveryQueuePortContract.java b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/RecoveryQueuePortContract.java new file mode 100644 index 00000000..a10b339b --- /dev/null +++ b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/RecoveryQueuePortContract.java @@ -0,0 +1,137 @@ +package dev.caskeleton.application.fileserver.testkit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.recovery.ReconciliationStatus; +import dev.caskeleton.application.fileserver.recovery.RecoveryQueuePort; +import java.time.Duration; +import java.util.List; +import java.util.function.Supplier; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * What every {@link RecoveryQueuePort} promises, run against the in-memory queue and the JPA queue + * on real PostgreSQL (review 2026-11-03 RR-01-04). + * + *

    An entry is a request for an operator's decision: one open entry per file however often it is + * reported, gone from the pending list once resolved, and raisable again afterwards. + */ +public abstract class RecoveryQueuePortContract { + + /** The queue under test, empty at the start of each test. */ + protected abstract RecoveryQueuePort queue(); + + /** A file that has a record, which an entry may name. */ + protected abstract FileId recordedFile(); + + /** + * Lets {@code duration} pass on the clock the queue stamps entries with. The pending list is + * oldest first; a store that orders by insertion keeps the default. + */ + protected void elapse(Duration duration) {} + + /** Runs one queue call inside the caller transaction the adapter requires, and commits it. */ + protected T inTransaction(Supplier work) { + return work.get(); + } + + @Test + @DisplayName("올린 파일은 대기 목록에 있다") + void anEnqueuedFileIsPending() { + // given + FileId file = recordedFile(); + + // when + run(() -> queue().enqueue(file, "READY_DIGEST_MISMATCH")); + + // then + assertThat(pending(10)).containsExactly(file); + } + + @Test + @DisplayName("같은 파일을 두 번 올려도 대기 항목은 하나다") + void theSameFileReportedTwiceIsPendingOnce() { + // given + FileId file = recordedFile(); + run(() -> queue().enqueue(file, "READY_DIGEST_MISMATCH")); + + // when + run(() -> queue().enqueue(file, "READY_SIZE_MISMATCH")); + + // then + assertThat(pending(10)).containsExactly(file); + } + + @Test + @DisplayName("처리한 파일은 대기 목록에서 빠진다") + void aResolvedFileLeavesThePendingList() { + // given + FileId file = recordedFile(); + run(() -> queue().enqueue(file, "PUBLISH_EVIDENCE_INCOMPLETE")); + + // when + run(() -> queue().resolve(file, ReconciliationStatus.QUARANTINE_REQUIRED)); + + // then + assertThat(pending(10)).isEmpty(); + } + + @Test + @DisplayName("처리한 파일도 나중에 다시 올릴 수 있다") + void aResolvedFileCanBeRaisedAgain() { + // given + FileId file = recordedFile(); + run(() -> queue().enqueue(file, "FIRST")); + run(() -> queue().resolve(file, ReconciliationStatus.UNRESOLVED)); + + // when + run(() -> queue().enqueue(file, "SECOND")); + + // then + assertThat(pending(10)).containsExactly(file); + } + + @Test + @DisplayName("대기 목록은 먼저 올린 파일부터 한도만큼 준다") + void thePendingListIsOldestFirstUpToTheLimit() { + // given + FileId first = recordedFile(); + FileId second = recordedFile(); + FileId third = recordedFile(); + for (FileId file : List.of(first, second, third)) { + run(() -> queue().enqueue(file, "UNSETTLED")); + elapse(Duration.ofSeconds(1)); + } + + // when + List pending = pending(2); + + // then + assertThat(pending).containsExactly(first, second); + } + + @Test + @DisplayName("한도가 1보다 작으면 IllegalArgumentException으로 거절된다") + void aLimitBelowOneIsRefused() { + // when + Throwable refused = catchThrowable(() -> pending(0)); + + // then + assertThat(refused).isInstanceOf(IllegalArgumentException.class); + } + + private List pending(int limit) { + return inTransaction(() -> queue().pending(limit)); + } + + private void run(Runnable work) { + inTransaction( + () -> { + work.run(); + return null; + }); + } +} diff --git a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/StagingUploadLocatorPortContract.java b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/StagingUploadLocatorPortContract.java new file mode 100644 index 00000000..401043b6 --- /dev/null +++ b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/StagingUploadLocatorPortContract.java @@ -0,0 +1,92 @@ +package dev.caskeleton.application.fileserver.testkit; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.api.UploadId; +import dev.caskeleton.application.fileserver.api.metadata.UploadSessionDraft; +import dev.caskeleton.application.fileserver.api.metadata.UploadSessionPort; +import dev.caskeleton.application.fileserver.api.transfer.UploadMode; +import dev.caskeleton.application.fileserver.recovery.StagingUploadLocatorPort; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.UUID; +import java.util.function.Supplier; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * What every {@link StagingUploadLocatorPort} promises, run against the in-memory session store and + * the JPA locator on real PostgreSQL (review 2026-11-03 RR-01-04). + * + *

    Only the newest upload of a file is returned: an older one names a staging object that has + * since been reclaimed, and treating it as live evidence would tell reconciliation the upload is + * resumable when it is not. + */ +public abstract class StagingUploadLocatorPortContract { + + protected abstract StagingUploadLocatorPort locator(); + + /** The session store the locator reads, to record uploads with. */ + protected abstract UploadSessionPort sessions(); + + /** A file that has a record, which an upload may belong to. */ + protected abstract FileId recordedFile(); + + /** The instant the store reads as now. */ + protected abstract Instant now(); + + /** Lets {@code duration} pass on the clock the store stamps sessions with. */ + protected abstract void elapse(Duration duration); + + /** Runs one store call inside the caller transaction the adapter requires, and commits it. */ + protected T inTransaction(Supplier work) { + return work.get(); + } + + @Test + @DisplayName("업로드가 없던 파일은 아무것도 찾지 않는다") + void aFileThatWasNeverStagedLocatesNothing() { + // given + FileId file = recordedFile(); + + // when + Optional located = inTransaction(() -> locator().locate(file)); + + // then + assertThat(located).isEmpty(); + } + + @Test + @DisplayName("업로드가 여럿인 파일은 가장 최근 업로드를 찾는다") + void aFileStagedTwiceLocatesItsNewestUpload() { + // given + FileId file = recordedFile(); + stage(file); + elapse(Duration.ofSeconds(1)); + UploadId newest = stage(file); + + // when + Optional located = inTransaction(() -> locator().locate(file)); + + // then + assertThat(located).contains(newest); + } + + private UploadId stage(FileId file) { + UploadId uploadId = UploadId.of(UUID.randomUUID()); + inTransaction( + () -> + sessions() + .create( + new UploadSessionDraft( + uploadId, + file, + UploadMode.SINGLE_SHOT, + OptionalLong.of(100), + now().plus(Duration.ofHours(1))))); + return uploadId; + } +} diff --git a/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/UploadSessionPortContract.java b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/UploadSessionPortContract.java new file mode 100644 index 00000000..ae49d7a7 --- /dev/null +++ b/src/application-fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/UploadSessionPortContract.java @@ -0,0 +1,367 @@ +package dev.caskeleton.application.fileserver.testkit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.fileserver.api.UploadId; +import dev.caskeleton.application.fileserver.api.error.FileserverErrorCode; +import dev.caskeleton.application.fileserver.api.error.FileserverException; +import dev.caskeleton.application.fileserver.api.metadata.UploadSession; +import dev.caskeleton.application.fileserver.api.metadata.UploadSessionDraft; +import dev.caskeleton.application.fileserver.api.metadata.UploadSessionPort; +import dev.caskeleton.application.fileserver.api.metadata.WriterLease; +import dev.caskeleton.application.fileserver.api.transfer.UploadMode; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.OptionalLong; +import java.util.UUID; +import java.util.function.Supplier; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * What every {@link UploadSessionPort} promises, run against the in-memory store and the JPA store + * on real PostgreSQL (review 2026-11-03 RR-01-04). + * + *

    The writer lease is the only thing that keeps two instances from appending to one upload, so + * the cases are its rules: who may take a lease, who may renew it, who may advance the offset, and + * when an upload stops being writable at all. + */ +public abstract class UploadSessionPortContract { + + private static final Duration LEASE = Duration.ofMinutes(1); + private static final Duration UPLOAD_LIFETIME = Duration.ofHours(1); + + /** The store under test, holding no session at the start of each test. */ + protected abstract UploadSessionPort sessions(); + + /** A file that has a record, which an upload may belong to. */ + protected abstract FileId recordedFile(); + + /** The instant the store reads as now. */ + protected abstract Instant now(); + + /** Lets {@code duration} pass on the clock the store reads. */ + protected abstract void elapse(Duration duration); + + /** Runs one store call inside the caller transaction the adapter requires, and commits it. */ + protected T inTransaction(Supplier work) { + return work.get(); + } + + @Test + @DisplayName("새 업로드는 offset 0, lease 없음, 열린 상태로 다시 읽힌다") + void aCreatedUploadIsReadBackOpenAtOffsetZeroWithNoLease() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + + // when + UploadSession found = find(uploadId); + + // then + assertThat(List.of(found.committedOffset(), found.leaseToken().isPresent(), found.terminal())) + .containsExactly(0L, false, false); + } + + @Test + @DisplayName("다른 writer가 lease를 쥐고 있으면 lease를 주지 않는다") + void aLeaseIsRefusedWhileAnotherWriterHoldsIt() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + acquire(uploadId, "node-a"); + + // when + Throwable refused = catchThrowable(() -> acquire(uploadId, "node-b")); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.CONCURRENT_MODIFICATION); + } + + @Test + @DisplayName("만료된 lease는 다른 writer가 넘겨받는다") + void anExpiredLeaseIsTakenOverByAnotherWriter() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + acquire(uploadId, "node-a"); + elapse(LEASE.plusSeconds(1)); + + // when + WriterLease taken = acquire(uploadId, "node-b"); + + // then + assertThat(find(uploadId).leaseToken()).contains(taken.token()); + } + + @Test + @DisplayName("기대한 버전이 지난 것이면 lease를 주지 않는다") + void aLeaseIsRefusedOnAStaleVersion() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + long stale = find(uploadId).version(); + WriterLease first = acquire(uploadId, "node-a"); + inTransaction( + () -> { + sessions().releaseLease(uploadId, first); + return null; + }); + + // when + Throwable refused = + catchThrowable( + () -> + inTransaction( + () -> sessions().acquireLease(uploadId, "node-b", now(), LEASE, stale))); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.CONCURRENT_MODIFICATION); + } + + @Test + @DisplayName("업로드 기한이 지나면 lease를 주지 않는다") + void aLeaseIsRefusedOnceTheUploadHasExpired() { + // given + UploadId uploadId = create(Duration.ofMinutes(5)); + elapse(Duration.ofMinutes(6)); + + // when + Throwable refused = catchThrowable(() -> acquire(uploadId, "node-a")); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.CONCURRENT_MODIFICATION); + } + + @Test + @DisplayName("갱신한 lease는 처음 만료 시각이 지나도 다른 writer에게 넘어가지 않는다") + void aRenewedLeaseOutlivesItsFirstExpiry() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + WriterLease lease = acquire(uploadId, "node-a"); + elapse(LEASE.dividedBy(2)); + inTransaction(() -> sessions().renewLease(lease, now(), LEASE)); + elapse(LEASE.dividedBy(2).plusSeconds(1)); + + // when + Throwable refused = catchThrowable(() -> acquire(uploadId, "node-b")); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.CONCURRENT_MODIFICATION); + } + + @Test + @DisplayName("이미 만료된 lease는 갱신하지 않는다") + void aLapsedLeaseCannotBeRenewed() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + WriterLease lease = acquire(uploadId, "node-a"); + elapse(LEASE.plusSeconds(1)); + + // when + Throwable refused = + catchThrowable(() -> inTransaction(() -> sessions().renewLease(lease, now(), LEASE))); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.CONCURRENT_MODIFICATION); + } + + @Test + @DisplayName("lease를 쥔 writer는 기대한 offset에서 offset을 올린다") + void theLeaseHolderAdvancesTheOffsetFromTheExpectedOne() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + WriterLease lease = acquire(uploadId, "node-a"); + + // when + UploadSession advanced = inTransaction(() -> sessions().commitOffset(uploadId, lease, 0, 40)); + + // then + assertThat(advanced.committedOffset()).isEqualTo(40); + } + + @Test + @DisplayName("기대한 offset이 다르면 offset을 올리지 않는다") + void anOffsetCommitFromTheWrongOffsetIsRefused() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + WriterLease lease = acquire(uploadId, "node-a"); + + // when + Throwable refused = + catchThrowable(() -> inTransaction(() -> sessions().commitOffset(uploadId, lease, 10, 40))); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.CONCURRENT_MODIFICATION); + } + + @Test + @DisplayName("lease를 빼앗긴 writer는 offset을 올리지 못한다") + void aWriterWhoseLeaseWasTakenOverCannotAdvanceTheOffset() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + WriterLease stale = acquire(uploadId, "node-a"); + elapse(LEASE.plusSeconds(1)); + acquire(uploadId, "node-b"); + + // when + Throwable refused = + catchThrowable(() -> inTransaction(() -> sessions().commitOffset(uploadId, stale, 0, 40))); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.CONCURRENT_MODIFICATION); + } + + @Test + @DisplayName("lease가 만료된 writer는 아무도 넘겨받지 않았어도 offset을 올리지 못한다") + void aWriterWhoseLeaseLapsedCannotAdvanceTheOffset() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + WriterLease lease = acquire(uploadId, "node-a"); + elapse(LEASE.plusSeconds(1)); + + // when + Throwable refused = + catchThrowable(() -> inTransaction(() -> sessions().commitOffset(uploadId, lease, 0, 40))); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.CONCURRENT_MODIFICATION); + } + + @Test + @DisplayName("다른 writer의 토큰으로 해제해도 쥐고 있는 lease는 남는다") + void releasingWithAnotherWritersTokenLeavesTheLeaseHeld() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + WriterLease held = acquire(uploadId, "node-a"); + WriterLease other = new WriterLease(uploadId, "node-b", UUID.randomUUID(), now(), 0); + + // when + inTransaction( + () -> { + sessions().releaseLease(uploadId, other); + return null; + }); + + // then + assertThat(find(uploadId).leaseToken()).contains(held.token()); + } + + @Test + @DisplayName("업로드를 끝내는 호출은 처음 한 번만 끝냈다고 답한다") + void terminatingAnUploadAnswersTrueOnlyTheFirstTime() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + boolean first = inTransaction(() -> sessions().terminate(uploadId)); + + // when + boolean second = inTransaction(() -> sessions().terminate(uploadId)); + + // then + assertThat(List.of(first, second)).containsExactly(true, false); + } + + @Test + @DisplayName("끝난 업로드에는 lease를 주지 않는다") + void aTerminatedUploadRefusesANewLease() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + inTransaction(() -> sessions().terminate(uploadId)); + + // when + Throwable refused = catchThrowable(() -> acquire(uploadId, "node-a")); + + // then + assertThat(codeOf(refused)).isEqualTo(FileserverErrorCode.CONCURRENT_MODIFICATION); + } + + @Test + @DisplayName("끝났지만 writer가 아직 lease를 쥔 업로드는 정리 대상으로 잡지 않는다") + void aTerminatedUploadWhoseWriterStillHoldsTheLeaseIsNotClaimedForCleanup() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + acquire(uploadId, "node-a"); + inTransaction(() -> sessions().terminate(uploadId)); + + // when + boolean claimed = inTransaction(() -> sessions().claimForCleanup(uploadId, now())); + + // then + assertThat(claimed).isFalse(); + } + + @Test + @DisplayName("끝났고 lease가 만료된 업로드는 정리 대상으로 잡힌다") + void aTerminatedUploadWithALapsedLeaseIsClaimedForCleanup() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + acquire(uploadId, "node-a"); + inTransaction(() -> sessions().terminate(uploadId)); + elapse(LEASE.plusSeconds(1)); + + // when + boolean claimed = inTransaction(() -> sessions().claimForCleanup(uploadId, now())); + + // then + assertThat(claimed).isTrue(); + } + + @Test + @DisplayName("열린 업로드는 정리 대상으로 잡지 않는다") + void anOpenUploadIsNotClaimedForCleanup() { + // given + UploadId uploadId = create(UPLOAD_LIFETIME); + + // when + boolean claimed = inTransaction(() -> sessions().claimForCleanup(uploadId, now())); + + // then + assertThat(claimed).isFalse(); + } + + @Test + @DisplayName("기한이 지난 열린 업로드만 기한이 이른 것부터 찾는다") + void expiredOpenUploadsAreFoundEarliestExpiryFirst() { + // given + UploadId later = create(Duration.ofMinutes(50)); + UploadId earlier = create(Duration.ofMinutes(20)); + create(Duration.ofHours(3)); + UploadId ended = create(Duration.ofMinutes(10)); + inTransaction(() -> sessions().terminate(ended)); + Instant cutoff = now().plus(Duration.ofHours(1)); + + // when + List expired = inTransaction(() -> sessions().findExpired(cutoff, 10)); + + // then + assertThat(expired).extracting(UploadSession::uploadId).containsExactly(earlier, later); + } + + private UploadId create(Duration lifetime) { + UploadId uploadId = UploadId.of(UUID.randomUUID()); + FileId file = recordedFile(); + inTransaction( + () -> + sessions() + .create( + new UploadSessionDraft( + uploadId, + file, + UploadMode.RESUMABLE, + OptionalLong.of(100), + now().plus(lifetime)))); + return uploadId; + } + + private UploadSession find(UploadId uploadId) { + return inTransaction(() -> sessions().find(uploadId)).orElseThrow(); + } + + private WriterLease acquire(UploadId uploadId, String owner) { + long version = find(uploadId).version(); + return inTransaction(() -> sessions().acquireLease(uploadId, owner, now(), LEASE, version)); + } + + private static Object codeOf(Throwable failure) { + return failure instanceof FileserverException refused ? refused.code() : failure; + } +} diff --git a/src/application-notification/build.gradle b/src/application-notification/build.gradle index 71aaa736..d0e46d03 100644 --- a/src/application-notification/build.gradle +++ b/src/application-notification/build.gradle @@ -16,6 +16,9 @@ dependencies { // so a provider adapter in any leaf extends the same contract (capability criterion 5). testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api' testFixturesImplementation 'org.assertj:assertj-core' + // OverlappingWrites (two transactions of one key, overlapped on purpose) lives beside + // application-core's transaction fake: the outbox counter contract needs it too. + testFixturesImplementation testFixtures(project(':application-core')) // Use-case tests run over application-core's shared transaction fake (DirectTransactionPort), // the one held to the transaction port contract, not a hand-written port per test class. diff --git a/src/application-notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJobStorePort.java b/src/application-notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJobStorePort.java index 5c3929bd..acfee9c9 100644 --- a/src/application-notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJobStorePort.java +++ b/src/application-notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJobStorePort.java @@ -35,7 +35,9 @@ public interface ReconciliationJobStorePort { *

    The claim is ownership that outlives the statement that took it: until {@code now + lease} * no other caller is handed the same job, whether or not a transaction is open. That is what lets * the caller ask a provider about each job with no database transaction held across the call. A - * job whose lease has passed is claimable again, so the lease must cover the whole pass. + * job whose lease has passed is claimable again, so the lease must cover the whole pass. Taking + * over such a job counts the abandoned claim as one attempt, so a job its worker dies on every + * time still reaches the attempt limit. * * @param limit how many to take * @param now the current instant diff --git a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/admin/testkit/AdminOperationStoreContract.java b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/admin/testkit/AdminOperationStoreContract.java index 86409ca8..f186787e 100644 --- a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/admin/testkit/AdminOperationStoreContract.java +++ b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/admin/testkit/AdminOperationStoreContract.java @@ -12,7 +12,7 @@ import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; import dev.caskeleton.application.notification.platform.api.NotificationId; import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; import dev.caskeleton.application.notification.platform.api.TenantId; -import dev.caskeleton.application.notification.platform.policy.testkit.OverlappingWrites; +import dev.caskeleton.application.transaction.testkit.OverlappingWrites; import java.util.List; import java.util.Optional; import java.util.Set; diff --git a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryReconciliationJobStore.java b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryReconciliationJobStore.java index f937cffb..18e877ea 100644 --- a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryReconciliationJobStore.java +++ b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryReconciliationJobStore.java @@ -81,6 +81,10 @@ public final class InMemoryReconciliationJobStore implements ReconciliationJobSt .toList(); List claimed = new ArrayList<>(); for (Row row : due) { + if (row.claimedUntil != null) { + // The previous claim's lease ran out with no reschedule or completion: it was an attempt. + row.attempts++; + } row.claimedUntil = now.plus(lease); row.claimFence++; claimed.add(row.copy()); diff --git a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/NotificationRequestStoreContract.java b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/NotificationRequestStoreContract.java index 96238b7f..fbf2a2d6 100644 --- a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/NotificationRequestStoreContract.java +++ b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/NotificationRequestStoreContract.java @@ -12,7 +12,7 @@ import dev.caskeleton.application.notification.platform.dispatch.NotificationReq import dev.caskeleton.application.notification.platform.dispatch.NotificationRequestRecord; import dev.caskeleton.application.notification.platform.dispatch.NotificationRequestStorePort; import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryRecord; -import dev.caskeleton.application.notification.platform.policy.testkit.OverlappingWrites; +import dev.caskeleton.application.transaction.testkit.OverlappingWrites; import java.time.Instant; import java.util.List; import java.util.Optional; diff --git a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/ReconciliationJobStoreContract.java b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/ReconciliationJobStoreContract.java index d50ab68e..a27d5cae 100644 --- a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/ReconciliationJobStoreContract.java +++ b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/ReconciliationJobStoreContract.java @@ -118,6 +118,25 @@ public abstract class ReconciliationJobStoreContract { .containsExactly(id); } + /** + * A worker that dies on the same job every time never reschedules it, so the lease expiry is the + * only trace its attempt leaves. Counting it is what lets the attempt limit end the loop, as the + * file server's reclaim does (review RR-03-08). + */ + @Test + @DisplayName("lease가 지나 다시 가져간 job은 끝내지 못한 이전 claim을 시도 한 번으로 센다") + void reclaimingAJobWhoseLeasePassedCountsTheAbandonedClaimAsAnAttempt() { + // given + scheduleDueJob(); + store().claimDue(10, NOW, LEASE); + + // when + List reclaimed = store().claimDue(10, NOW.plus(LEASE), LEASE); + + // then + assertThat(reclaimed).extracting(ReconciliationJob::attempts).containsExactly(1); + } + @Test @DisplayName("lease가 1마이크로초라도 남은 job은 다른 인스턴스가 가져가지 못한다") void aJobWhoseLeaseHasNotPassedIsNotClaimed() { @@ -161,11 +180,12 @@ public abstract class ReconciliationJobStoreContract { // when store().reschedule(stale, "STILL_UNKNOWN", NOW.plus(Duration.ofHours(1))); - // then + // then: two abandoned leases count, the stale reschedule does not (it would also have moved + // the job an hour out of this claim) assertThat(store().claimDue(10, NOW.plus(LEASE.multipliedBy(2)), LEASE)) .as("two holders counting one question would reach the attempt limit early") .extracting(ReconciliationJob::attempts) - .containsExactly(0); + .containsExactly(2); } @Test diff --git a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/DeduplicationStoreContract.java b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/DeduplicationStoreContract.java index 73ddfe82..5c6cbef3 100644 --- a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/DeduplicationStoreContract.java +++ b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/DeduplicationStoreContract.java @@ -7,6 +7,7 @@ import dev.caskeleton.application.notification.platform.api.TenantId; import dev.caskeleton.application.notification.platform.policy.DeduplicationResult; import dev.caskeleton.application.notification.platform.policy.DeduplicationStorePort; import dev.caskeleton.application.notification.platform.policy.RecipientIdentity; +import dev.caskeleton.application.transaction.testkit.OverlappingWrites; import java.util.List; import java.util.UUID; import java.util.function.Supplier; diff --git a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/PreferenceStoreContract.java b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/PreferenceStoreContract.java index 995732d5..c7e5691b 100644 --- a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/PreferenceStoreContract.java +++ b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/PreferenceStoreContract.java @@ -6,6 +6,7 @@ import dev.caskeleton.application.notification.platform.api.TenantId; import dev.caskeleton.application.notification.platform.api.routing.Channel; import dev.caskeleton.application.notification.platform.policy.PreferenceRecord; import dev.caskeleton.application.notification.platform.policy.PreferenceStorePort; +import dev.caskeleton.application.transaction.testkit.OverlappingWrites; import java.time.Instant; import java.util.List; import java.util.Optional; diff --git a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/SuppressionStoreContract.java b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/SuppressionStoreContract.java index e48b15f2..58cfd4a6 100644 --- a/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/SuppressionStoreContract.java +++ b/src/application-notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/policy/testkit/SuppressionStoreContract.java @@ -9,6 +9,7 @@ import dev.caskeleton.application.notification.platform.policy.SuppressionReason import dev.caskeleton.application.notification.platform.policy.SuppressionScope; import dev.caskeleton.application.notification.platform.policy.SuppressionSource; import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort; +import dev.caskeleton.application.transaction.testkit.OverlappingWrites; import java.time.Duration; import java.time.Instant; import java.util.List; diff --git a/src/build-logic/README.md b/src/build-logic/README.md index e4e20312..97afbfdb 100644 --- a/src/build-logic/README.md +++ b/src/build-logic/README.md @@ -16,7 +16,7 @@ capability, and the files it reads are declared as that test task's inputs. | `ca.java-library` / `ca.spring-library` / `ca.spring-config` | leaf-level composition of the Java/quality conventions for plain and Spring libraries | | `ca.platform-module` / `ca.grpc-platform-module` | what a vendored platform leaf (`messaging:*`, `grpc:*`, `grpc-advanced:*`) is: `java-library`, plus the gRPC BOM for the leaves that had it | | `ca.strict-test-lane` | the one "named tests ran, none skipped" mechanism: tagged / named / own-source-set `Test` lanes that fail when they execute nothing, miss a `requires` class, or (with `rejectSkipped`) skip anything. `mainStage()` moves a tagged lane out of `test` into the leaf's `mainCheck`; `explicitOnly(reason)` keeps a lane that needs an outside input out of every aggregate, and the aggregate prints the reason. Extra test source sets are Gradle `jvm-test-suite` suites declared in the leaf (`testing { suites { … } }`); a lane named after its suite makes the suite's own `Test` task strict, and `ca.java-conventions` gives every suite the BOM's JUnit | -| `ca.strict-lane-stages` (root) | the other half of the lane mechanism: `verifyStrictLaneStages --dry-run` fails when a strict lane that is not `explicitOnly` is in the task plan of no declared stage that a workflow executes, or when a declared stage is executed by no workflow (a `--dry-run` or `-p ` command does not count). The plan is Gradle's own; the workflows are read for their `run:` steps with YAML folding applied (review 2026-11-01 08-01) | +| `ca.strict-lane-stages` (root of each build with strict lanes: the main build and optional-platforms) | the other half of the lane mechanism: `verifyStrictLaneStages --dry-run` fails when a strict lane of that build that is not `explicitOnly` is in the task plan of no declared stage that a workflow executes, or when a declared stage is executed by no workflow. A `--dry-run`, a command for another build (`workflowProjectDirectory` names the `-p` value that starts this one), and a command under `continue-on-error` or an `if:` other than `always()`/`success()`/`!cancelled()` do not count, and a lane the command removes with `-x` is not covered. The plan is Gradle's own; the workflows are read for their `run:` steps with YAML folding applied (review 2026-11-01 08-01, 2026-11-03 RR-06-03/04) | | `ca.redis-topology-lane` | the Redis topology lane (mode selection, `-P` property validation, per-mode required classes) the redis-sdk-topology workflow drives | | `ca.api-surface` | read-only public API surface verification with a separate update task, wired into the leaf's `check` (owner decision 20); optionally holds other leaves' imports to a reviewed adopter subset (`adopterBaseline`) | | `ca.config-contract` | app-bootstrap's `verifyEnvKeys` (moved from the deleted build-tools included build; owner decision 4 keeps the task in app-bootstrap) | @@ -61,7 +61,8 @@ cd src The lane and registry conventions are covered by Gradle TestKit against real builds rather than by reading the plugin source, because the properties that matter — a lane that discovers nothing fails, -a lane never reports up-to-date, a malformed registry is refused before any project is included — are +a lane reuses a result only when its classes and its lane policy (`rejectSkipped`, `requires`, tag, +category) are unchanged and it starts no server, a malformed registry is refused before any project is included — are runtime behaviour rather than text in a script. Two of those tests exist because the obvious reading of the Gradle documentation was wrong: `failOnNoDiscoveredTests` does not fire when a tag filter matches nothing, and `failOnNoMatchingTests` does not fire when only *some* of the named tests are diff --git a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/ConfigContractPlugin.java b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/ConfigContractPlugin.java index 8b36b456..19aa949a 100644 --- a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/ConfigContractPlugin.java +++ b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/ConfigContractPlugin.java @@ -21,7 +21,6 @@ public final class ConfigContractPlugin implements Plugin { task.setGroup("verification"); task.setDescription( "Verifies shipped configuration APP_ references, src/.env.example, and env-keys.yaml stay registered."); - task.dependsOn(":adapter:outbound:redis-runtime:compileJava"); task.getEnvFile().fileValue(root.file(".env.example")); task.getConfigurationFiles() .from( @@ -43,8 +42,11 @@ public final class ConfigContractPlugin implements Plugin { tree.exclude("app-bootstrap/**"); tree.exclude("**/build/**"); })); - // What an operator edits or runs: the root Compose files and the Logback - // configuration. Their APP_ names must be registered and read (07-10). + // What an operator edits or runs: the root Compose files, the Logback + // configuration, and the Compose lane contract with the smoke wrapper that drives it + // (each lane's activationEnv, the wrapper's CAPABILITIES_OFF). Their APP_ names must + // be registered and read (07-10; RR-06-07: an unread CASKELETON_GRPC_ENABLED-style + // switch in a lane was found by hand, not by this task). task.getOperatorFiles() .from( root.fileTree( @@ -55,13 +57,13 @@ public final class ConfigContractPlugin implements Plugin { // clients and other services, not to this application. tree.exclude("docker-compose.infra.yml"); }), - root.file("app-bootstrap/src/main/resources/logback-spring.xml")); + root.file("app-bootstrap/src/main/resources/logback-spring.xml"), + root.file("config/runtime/compose-profile-contracts.json"), + root.file("../scripts/run-compose-runtime-smoke.sh")); task.getRegistryFile().fileValue(root.file("../docs/registries/env-keys.yaml")); - task.getRedisSdkMetadata() - .fileValue( - root.file( - "adapter/outbound/redis-runtime/build/classes/java/main/" - + "META-INF/spring-configuration-metadata.json")); + // No module path here: the typed properties come from the configuration metadata on + // the classpath the composition root wires (metadataClasspath), so this task knows no + // leaf by name (review 2026-11-03 RR-06-10). task.getProductionSources() .from( root.fileTree( diff --git a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/ConfigurationMetadataClasspath.java b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/ConfigurationMetadataClasspath.java new file mode 100644 index 00000000..624be5be --- /dev/null +++ b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/ConfigurationMetadataClasspath.java @@ -0,0 +1,87 @@ +package dev.caskeleton.buildlogic.config; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import tools.jackson.databind.DeserializationFeature; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; + +/** + * The typed configuration properties a classpath declares, read from each entry's {@code + * META-INF/spring-configuration-metadata.json} (the file the configuration processor writes into a + * module's classes and jar). + * + *

    The environment contract used to read one leaf's metadata by its build path and depend on that + * leaf's {@code compileJava} — the env contract task knew the Redis runtime module by name + * (review 2026-11-03 RR-06-10, capability criterion 7). Reading the composition's classpath instead, + * a module that ships metadata is checked because it is selected, and one that is removed takes its + * metadata with it. No Gradle API. + */ +public final class ConfigurationMetadataClasspath { + static final String METADATA = "META-INF/spring-configuration-metadata.json"; + + private static final ObjectMapper JSON = + JsonMapper.builder().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).build(); + + private ConfigurationMetadataClasspath() {} + + /** + * Every property name starting with {@code prefix} that some classpath entry's metadata declares. + * + * @param classpath jars and class directories; entries without metadata are skipped + * @param prefix the property prefix, for example {@code app.redis.} + * @return the names, sorted + */ + public static Set propertiesUnder(Collection classpath, String prefix) { + Set properties = new TreeSet<>(); + for (File entry : classpath) { + for (MetadataProperty property : read(entry)) { + if (property.name() != null && property.name().startsWith(prefix)) { + properties.add(property.name()); + } + } + } + return properties; + } + + private static List read(File entry) { + try { + if (entry.isDirectory()) { + File metadata = new File(entry, METADATA); + return metadata.isFile() ? parse(Files.readAllBytes(metadata.toPath())) : List.of(); + } + if (entry.isFile() && entry.getName().endsWith(".jar")) { + try (ZipFile jar = new ZipFile(entry)) { + ZipEntry metadata = jar.getEntry(METADATA); + if (metadata == null) { + return List.of(); + } + try (InputStream in = jar.getInputStream(metadata)) { + return parse(in.readAllBytes()); + } + } + } + return List.of(); + } catch (IOException failure) { + throw new UncheckedIOException("failed to read configuration metadata of " + entry, failure); + } + } + + private static List parse(byte[] json) { + Metadata metadata = JSON.readValue(json, Metadata.class); + return metadata.properties() == null ? List.of() : metadata.properties(); + } + + private record Metadata(List properties) {} + + private record MetadataProperty(String name) {} +} diff --git a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/EnvContractVerifier.java b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/EnvContractVerifier.java index 07b645ec..c6cd6192 100644 --- a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/EnvContractVerifier.java +++ b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/EnvContractVerifier.java @@ -36,6 +36,18 @@ public final class EnvContractVerifier { /** A variable Logback reads directly: {@code }. */ private static final Pattern LOGBACK_SOURCE = Pattern.compile("source=\"(APP_[A-Z][A-Z0-9_]*)\""); + /** + * A variable a JSON object sets: the {@code activationEnv} maps of the Compose lane contract, and + * the smoke wrapper's {@code CAPABILITIES_OFF} literal. An {@code "APP_X=value"} string (an + * argument the wrapper passes on) is not a key and is not matched. + */ + private static final Pattern JSON_KEY = Pattern.compile("\"(APP_[A-Z][A-Z0-9_]*)\"\\s*:"); + /** + * A list element ({@code APP_HTTPCLIENT_CLIENTS_0_NAME}). Its field template is registered by the + * owning module's field registry ({@code docs/httpclient/env-fields.yaml}, held by that module's + * manifest test and strict binder), not as a row here. + */ + private static final Pattern INDEXED_KEY = Pattern.compile(".*_[0-9]+(_.*)?"); private static final ObjectMapper JSON = JsonMapper.builder().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).build(); @@ -62,8 +74,10 @@ public final class EnvContractVerifier { * Verifies the contract, and also every {@code APP_*} name the operator-facing files set. * *

    Operator files are what an operator edits or runs to supply values — the root Compose files - * that run the application and {@code logback-spring.xml}. A Compose file contributes the keys of - * its {@code environment:} maps, Logback its {@code springProperty} sources. A name there must be registered, and something must read it: + * that run the application, {@code logback-spring.xml}, the Compose lane contract and the smoke + * wrapper that drives it. A Compose file contributes the keys of its {@code environment:} maps, + * Logback its {@code springProperty} sources, a JSON file or a shell script the keys of its JSON + * objects (review 2026-11-03 RR-06-07). A name there must be registered, and something must read it: * a shipped-configuration placeholder, a production source, or a typed property. A variable set * in a Compose file that nothing reads is a setting the operator believes is in force and is not * ({@code APP_LOG_JSON_ENABLED} in the prod-smoke file, 07-10). @@ -197,17 +211,17 @@ public final class EnvContractVerifier { Set operatorReferences = new TreeSet<>(); for (File operatorFile : operatorFiles) { requireFile(operatorFile, "missing operator file " + operatorFile); - Matcher matcher = - (operatorFile.getName().endsWith(".xml") ? LOGBACK_SOURCE : COMPOSE_ENVIRONMENT_KEY) - .matcher(read(operatorFile)); + Matcher matcher = operatorKeyPattern(operatorFile.getName()).matcher(read(operatorFile)); while (matcher.find()) { - operatorReferences.add(matcher.group(1)); + if (!INDEXED_KEY.matcher(matcher.group(1)).matches()) { + operatorReferences.add(matcher.group(1)); + } } } failUnlessEmpty( difference(operatorReferences, registryAppKeys), - "operator files (Compose, logback-spring.xml) set APP_ keys absent from " - + "docs/registries/env-keys.yaml: "); + "operator files (Compose, logback-spring.xml, the Compose lane contract, the smoke wrapper)" + + " set APP_ keys absent from docs/registries/env-keys.yaml: "); Set typedRows = new TreeSet<>(); registryRows.stream() .filter(row -> row.property().isPresent()) @@ -220,8 +234,9 @@ public final class EnvContractVerifier { } failUnlessEmpty( unreadOperatorReferences, - "operator files (Compose, logback-spring.xml) set APP_ keys nothing reads — no shipped " - + "configuration placeholder, production source or typed property: "); + "operator files (Compose, logback-spring.xml, the Compose lane contract, the smoke" + + " wrapper) set APP_ keys nothing reads — no shipped configuration placeholder," + + " production source or typed property: "); Set unconsumed = new TreeSet<>(); for (EnvRegistryRow row : registryRows) { @@ -263,6 +278,16 @@ public final class EnvContractVerifier { warnings); } + private static Pattern operatorKeyPattern(String fileName) { + if (fileName.endsWith(".xml")) { + return LOGBACK_SOURCE; + } + if (fileName.endsWith(".json") || fileName.endsWith(".sh")) { + return JSON_KEY; + } + return COMPOSE_ENVIRONMENT_KEY; + } + private static SpringConfigurationMetadata parseMetadata(File file) { SpringConfigurationMetadata metadata = JSON.readValue(file, SpringConfigurationMetadata.class); diff --git a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/VerifyEnvKeysTask.java b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/VerifyEnvKeysTask.java index ce79b83a..9ff45763 100644 --- a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/VerifyEnvKeysTask.java +++ b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/config/VerifyEnvKeysTask.java @@ -1,13 +1,20 @@ package dev.caskeleton.buildlogic.config; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; import java.util.List; +import java.util.Set; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.InputFile; import org.gradle.api.tasks.InputFiles; -import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.PathSensitive; import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; @@ -24,6 +31,12 @@ public abstract class VerifyEnvKeysTask extends DefaultTask { "APP_LEASE_REDIS_", "APP_SESSION_REDIS_"); + /** + * The property trees whose every typed property needs a registry row. A policy of the registry, + * not a module location: the metadata is found on {@link #getMetadataClasspath()}. + */ + private static final List TYPED_PREFIXES = List.of("app.redis."); + @InputFile public abstract RegularFileProperty getEnvFile(); @@ -34,9 +47,12 @@ public abstract class VerifyEnvKeysTask extends DefaultTask { @InputFile public abstract RegularFileProperty getRegistryFile(); - @Optional - @InputFile - public abstract RegularFileProperty getRedisSdkMetadata(); + /** + * The composition's classpath with every capability a service can select; the typed properties + * under {@link #TYPED_PREFIXES} are read from its configuration metadata. + */ + @Classpath + public abstract ConfigurableFileCollection getMetadataClasspath(); @InputFiles @PathSensitive(PathSensitivity.RELATIVE) @@ -48,17 +64,24 @@ public abstract class VerifyEnvKeysTask extends DefaultTask { @TaskAction public void verifyContract() { + if (getMetadataClasspath().isEmpty()) { + throw new GradleException( + "verifyEnvKeys: no metadata classpath. Give it the composition's every-capability" + + " classpath, or the typed-property check silently covers nothing."); + } EnvContractResult result; try { + List scopes = new ArrayList<>(); + for (String prefix : TYPED_PREFIXES) { + scopes.add(new MetadataScope(prefix, metadataOf(prefix))); + } result = EnvContractVerifier.verify( getEnvFile().get().getAsFile(), getConfigurationFiles().getFiles(), getRegistryFile().get().getAsFile(), getProductionSources().getFiles(), - List.of( - new MetadataScope( - "app.redis.", getRedisSdkMetadata().get().getAsFile())), + scopes, ENFORCED_PREFIXES, getOperatorFiles().getFiles()); } catch (IllegalStateException invalidContract) { @@ -75,4 +98,24 @@ public abstract class VerifyEnvKeysTask extends DefaultTask { result.typedPropertyCount(), result.consumedOrDeprecatedRegistryRowCount()); } + + /** The typed properties under {@code prefix}, as one metadata file the verifier reads. */ + private File metadataOf(String prefix) { + Set properties = + ConfigurationMetadataClasspath.propertiesUnder(getMetadataClasspath().getFiles(), prefix); + StringBuilder json = new StringBuilder("{\"properties\":["); + String separator = ""; + for (String property : properties) { + json.append(separator).append("{\"name\":\"").append(property).append("\"}"); + separator = ","; + } + json.append("]}"); + File file = new File(getTemporaryDir(), prefix + "metadata.json"); + try { + Files.writeString(file.toPath(), json, StandardCharsets.UTF_8); + } catch (IOException failure) { + throw new UncheckedIOException("failed to write " + file, failure); + } + return file; + } } diff --git a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/StrictLaneStagesExtension.java b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/StrictLaneStagesExtension.java index 771f8d07..ac09e803 100644 --- a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/StrictLaneStagesExtension.java +++ b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/StrictLaneStagesExtension.java @@ -2,6 +2,7 @@ package dev.caskeleton.buildlogic.strictlane; import org.gradle.api.file.DirectoryProperty; import org.gradle.api.provider.ListProperty; +import org.gradle.api.provider.Property; /** * The stages a strict lane may belong to, and where the workflows that run them live. @@ -15,6 +16,14 @@ public abstract class StrictLaneStagesExtension { public abstract DirectoryProperty getWorkflowDirectory(); + /** + * The {@code -p} directory a workflow step passes to start this build, relative to the step's + * working directory — {@code optional-platforms} for the optional build, which steps run as + * {@code ./gradlew -p optional-platforms } from {@code src}. Unset for the build a step + * starts in its working directory. Only commands that start this build execute its stages. + */ + public abstract Property getWorkflowProjectDirectory(); + public void stage(String... taskPaths) { getStages().addAll(taskPaths); } diff --git a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/StrictLaneStagesPlugin.java b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/StrictLaneStagesPlugin.java index fea9b6a4..64bcef82 100644 --- a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/StrictLaneStagesPlugin.java +++ b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/StrictLaneStagesPlugin.java @@ -29,11 +29,18 @@ import org.gradle.api.execution.TaskExecutionGraph; * stated once, here, instead of as one workflow line per lane: * *

      - *
    1. every strict lane that is not {@code explicitOnly} is in the task plan of a declared stage; - *
    2. every declared stage is executed by a workflow — a {@code --dry-run} or a {@code -p - * other-build} command does not count. + *
    3. every strict lane that is not {@code explicitOnly} is in the task plan of a declared stage + * as a workflow runs it — a lane the command removes with {@code -x} is not covered; + *
    4. every declared stage is executed by a workflow — a {@code --dry-run}, a command for another + * build, and a command under {@code continue-on-error} or an {@code if:} other than a status + * function do not count (review 2026-11-03 RR-06-04). *
    * + *

    Each build that holds strict lanes applies it at its own root: the main build, and the + * optional-platforms build, whose stages workflows start with {@code -p optional-platforms} + * ({@link StrictLaneStagesExtension#getWorkflowProjectDirectory()}). Applied to the main build only, + * it said "every strict lane" while the optional build's performance lane ran nowhere (RR-06-03).

    + * *

    The task plan is Gradle's own: {@code verifyStrictLaneStages} depends on the stages, so a * {@code --dry-run} of it builds the real plan, and the check reads it when the graph is ready. It * refuses to run without {@code --dry-run}, which would execute every stage. @@ -83,7 +90,8 @@ public final class StrictLaneStagesPlugin implements Plugin { project, graph, extension.getStages().get(), - extension.getWorkflowDirectory().get().getAsFile()); + extension.getWorkflowDirectory().get().getAsFile(), + extension.getWorkflowProjectDirectory().getOrNull()); if (!problems.isEmpty()) { throw new GradleException( TASK_NAME @@ -102,28 +110,39 @@ public final class StrictLaneStagesPlugin implements Plugin { } private static List problems( - Project root, TaskExecutionGraph graph, List stages, File workflowDirectory) { + Project root, + TaskExecutionGraph graph, + List stages, + File workflowDirectory, + String buildDirectory) { List problems = new ArrayList<>(foldedCommandBreaks(workflowDirectory)); List invocations = invocations(workflowDirectory); - List executedStages = new ArrayList<>(); + // Only an executed stage covers a lane — a stage no workflow runs is the same gap one level + // up — and only as the command runs it: what its -x removes is not covered. + Map> plans = new LinkedHashMap<>(); for (String stage : stages) { - if (invocations.stream().anyMatch(invocation -> invocation.executesInStartBuild(stage))) { - executedStages.add(stage); - } else { + List executions = + invocations.stream() + .filter(invocation -> invocation.executes(stage, buildDirectory)) + .toList(); + if (executions.isEmpty()) { problems.add( "stage " + stage + " is executed by no workflow in " + workflowDirectory.getName() - + " (a --dry-run or -p command does not count)"); + + " (a --dry-run, a command for another build, and a command under" + + " continue-on-error or an if: other than always()/success()/!cancelled() do not" + + " count)"); + continue; } - } - - // Only an executed stage covers a lane: a stage no workflow runs is the same gap one level up. - Map> plans = new LinkedHashMap<>(); - for (String stage : executedStages) { Task stageTask = root.getTasks().findByPath(stage); - plans.put(stage, stageTask == null ? Set.of() : plan(graph, stageTask)); + int run = 0; + for (WorkflowGradleInvocations.Invocation execution : executions) { + plans.put( + stage + "#" + run++, + stageTask == null ? Set.of() : plan(graph, stageTask, execution)); + } } List lanes = new ArrayList<>(); root.getAllprojects().stream() @@ -149,12 +168,14 @@ public final class StrictLaneStagesPlugin implements Plugin { return problems; } - private static Set plan(TaskExecutionGraph graph, Task stage) { + /** The tasks {@code stage} reaches as {@code execution} runs it: what its -x removes is cut off. */ + private static Set plan( + TaskExecutionGraph graph, Task stage, WorkflowGradleInvocations.Invocation execution) { Set seen = new HashSet<>(); Deque pending = new ArrayDeque<>(List.of(stage)); while (!pending.isEmpty()) { Task task = pending.pop(); - if (seen.add(task.getPath())) { + if (!execution.excludes(task.getPath()) && seen.add(task.getPath())) { pending.addAll(graph.getDependencies(task)); } } diff --git a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/StrictTestLanePlugin.java b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/StrictTestLanePlugin.java index 6ef01c97..104671df 100644 --- a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/StrictTestLanePlugin.java +++ b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/StrictTestLanePlugin.java @@ -224,6 +224,20 @@ public final class StrictTestLanePlugin implements Plugin { // build cache: the result it reuses was produced by a run that passed the checks below (no test // ran, a skip, a required test missing), with the same classes, filter and properties. A lane // whose result depends on what Gradle cannot see runs every time (review 2026-11-01 08-02). + // The verdict below reads the lane's policy, not only its test results, so the policy is an + // input: tightening it (rejectSkipped = true, another required test, another category) must + // judge the lane again rather than reuse a result produced under the looser policy. Without + // this a lane that skipped a test stayed green, up to date and from the build cache, after + // rejectSkipped was switched on (review 2026-11-03 RR-06-02). + test.getInputs().property("strictLane.rejectSkipped", project.provider(lane::isRejectSkipped)); + test.getInputs() + .property("strictLane.requiredTests", project.provider(lane::getRequiredTests)); + test.getInputs() + .property( + "strictLane.tag", project.provider(() -> lane.getTag() == null ? "" : lane.getTag())); + test.getInputs() + .property("strictLane.category", project.provider(() -> lane.getCategory().name())); + test.getInputs().property("strictLane.mainStage", project.provider(lane::isMainStage)); String rerunReason = rerunReason(lane); if (rerunReason != null) { test.getOutputs().upToDateWhen(ignored -> false); diff --git a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/WorkflowGradleInvocations.java b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/WorkflowGradleInvocations.java index 61edc631..22dc2291 100644 --- a/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/WorkflowGradleInvocations.java +++ b/src/build-logic/src/main/java/dev/caskeleton/buildlogic/strictlane/WorkflowGradleInvocations.java @@ -2,6 +2,7 @@ package dev.caskeleton.buildlogic.strictlane; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -14,10 +15,21 @@ import java.util.regex.Pattern; * {@code |} or {@code >} block — and applies YAML's folding rule to the second, because a blank line * inside a folded block ends the command (the rest becomes a separate shell command). Comments are * dropped before anything is matched, so a task named only in a comment is not "run". + * + *

    A command is read with what decides whether it gates anything (review 2026-11-03 RR-06-04): + * its {@code -x} exclusions, and the {@code continue-on-error} and {@code if:} of its step and its + * job. A command whose failure cannot fail the run, or that runs only under a condition other than + * a status function, is still returned, with the reason it does not count. */ public final class WorkflowGradleInvocations { private static final Pattern RUN_KEY = Pattern.compile("^(\\s*)(?:-\\s+)?run:(.*)$"); private static final Pattern BLOCK_HEADER = Pattern.compile("^([|>])[-+]?\\d*\\s*(#.*)?$"); + private static final Pattern MAPPING_KEY = + Pattern.compile("^(\\s*)(?:-\\s+)?([A-Za-z0-9_-]+):(?:\\s+(.*?))?\\s*$"); + private static final Pattern JOB_KEY = Pattern.compile("^(\\s*)([A-Za-z0-9_-]+):\\s*(#.*)?$"); + /** Conditions under which a step still runs on every run whose earlier steps passed. */ + private static final Set STATUS_CONDITIONS = + Set.of("always()", "success()", "!cancelled()", "true"); private static final Set OPTIONS_WITH_VALUE = Set.of( "-p", @@ -46,10 +58,19 @@ public final class WorkflowGradleInvocations { * directory * @param tasks the task selectors it requests, in order * @param dryRun whether it only plans ({@code --dry-run} or {@code -m}) instead of executing + * @param excludedTasks the {@code -x} / {@code --exclude-task} selectors, in order + * @param uncountedReason why the command gates nothing although it runs — {@code + * continue-on-error}, or a condition other than a status function — or {@code null} */ - public record Invocation(String projectDir, List tasks, boolean dryRun) { + public record Invocation( + String projectDir, + List tasks, + boolean dryRun, + List excludedTasks, + String uncountedReason) { public Invocation { tasks = List.copyOf(tasks); + excludedTasks = List.copyOf(excludedTasks); } /** @@ -57,24 +78,63 @@ public final class WorkflowGradleInvocations { * build its step starts in. {@code ci} and {@code :ci} name the same root task. */ public boolean executesInStartBuild(String taskPath) { + return executes(taskPath, null); + } + + /** + * Whether this command executes, and gates on, the task at {@code taskPath} of the build a + * workflow starts with {@code -p buildDirectory} ({@code null}: the build in the step's working + * directory). + */ + public boolean executes(String taskPath, String buildDirectory) { String wanted = absolute(taskPath); - return projectDir == null + return sameDirectory(projectDir, buildDirectory) && !dryRun + && uncountedReason == null && tasks.stream().map(Invocation::absolute).anyMatch(wanted::equals); } + /** + * Whether the task at {@code taskPath} is one this command's {@code -x} removes. An absolute + * or project-qualified selector names one task; a bare name removes that task in every project, + * as Gradle does for a command started at the root. + */ + public boolean excludes(String taskPath) { + String name = taskPath.substring(taskPath.lastIndexOf(':') + 1); + for (String selector : excludedTasks) { + if (selector.contains(":") ? absolute(selector).equals(taskPath) : selector.equals(name)) { + return true; + } + } + return false; + } + + private static boolean sameDirectory(String left, String right) { + return left == null || right == null + ? left == null && right == null + : normalized(left).equals(normalized(right)); + } + + private static String normalized(String directory) { + String plain = directory.startsWith("./") ? directory.substring(2) : directory; + return plain.endsWith("/") ? plain.substring(0, plain.length() - 1) : plain; + } + private static String absolute(String selector) { return selector.startsWith(":") ? selector : ":" + selector; } } + /** The script of one {@code run:} step, and why its commands gate nothing, if they do not. */ + private record RunStep(String script, String uncountedReason) {} + private WorkflowGradleInvocations() {} public static List read(String workflow) { List invocations = new ArrayList<>(); - for (String script : runScripts(workflow)) { - for (String command : shellCommands(script)) { - Invocation invocation = gradleInvocation(command); + for (RunStep step : runSteps(workflow)) { + for (String command : shellCommands(step.script())) { + Invocation invocation = gradleInvocation(command, step.uncountedReason()); if (invocation != null) { invocations.add(invocation); } @@ -131,8 +191,12 @@ public final class WorkflowGradleInvocations { } static List runScripts(String workflow) { + return runSteps(workflow).stream().map(RunStep::script).toList(); + } + + private static List runSteps(String workflow) { List lines = workflow.lines().toList(); - List scripts = new ArrayList<>(); + List scripts = new ArrayList<>(); for (int index = 0; index < lines.size(); index++) { Matcher run = RUN_KEY.matcher(lines.get(index)); if (!run.matches()) { @@ -140,9 +204,10 @@ public final class WorkflowGradleInvocations { } int keyIndent = run.group(1).length(); String value = run.group(2).trim(); + String uncounted = uncountedReason(lines, index); Matcher header = BLOCK_HEADER.matcher(value); if (!header.matches()) { - scripts.add(stripScalarComment(value)); + scripts.add(new RunStep(stripScalarComment(value), uncounted)); continue; } List block = new ArrayList<>(); @@ -157,11 +222,139 @@ public final class WorkflowGradleInvocations { block.stream().filter(line -> !line.isBlank()).mapToInt(WorkflowGradleInvocations::indentOf).min().orElse(0); List content = block.stream().map(line -> line.isBlank() ? "" : line.substring(blockIndent)).toList(); - scripts.add(header.group(1).equals("|") ? String.join("\n", content) : fold(content)); + scripts.add( + new RunStep( + header.group(1).equals("|") ? String.join("\n", content) : fold(content), uncounted)); } return scripts; } + /** + * Why the {@code run:} key at {@code runIndex} gates nothing: a {@code continue-on-error} other + * than {@code false}, or an {@code if:} other than a status function, on its step or its job. + */ + private static String uncountedReason(List lines, int runIndex) { + Matcher run = RUN_KEY.matcher(lines.get(runIndex)); + if (!run.matches()) { + return null; + } + boolean itemLine = lines.get(runIndex).trim().startsWith("-"); + int stepKeyIndent = + itemLine + ? lines.get(runIndex).indexOf("run:") + : run.group(1).length(); + int stepStart = runIndex; + if (!itemLine) { + while (stepStart > 0) { + stepStart--; + String line = lines.get(stepStart); + if (line.isBlank() || line.trim().startsWith("#")) { + continue; + } + if (indentOf(line) < stepKeyIndent) { + break; + } + } + } + String step = gatingProblem(lines, stepStart, stepKeyIndent, "step"); + if (step != null) { + return step; + } + int jobHeader = jobHeader(lines, runIndex); + if (jobHeader < 0) { + return null; + } + int jobKeyIndent = -1; + for (int next = jobHeader + 1; next < lines.size(); next++) { + String line = lines.get(next); + if (!line.isBlank() && !line.trim().startsWith("#")) { + jobKeyIndent = indentOf(line); + break; + } + } + return jobKeyIndent <= indentOf(lines.get(jobHeader)) + ? null + : gatingProblem(lines, jobHeader + 1, jobKeyIndent, "job"); + } + + /** The first {@code continue-on-error} or {@code if:} at {@code keyIndent} in the mapping. */ + private static String gatingProblem(List lines, int start, int keyIndent, String scope) { + for (int index = start; index < lines.size(); index++) { + String line = lines.get(index); + if (line.isBlank() || line.trim().startsWith("#")) { + continue; + } + if (index > start && indentOf(line) < keyIndent) { + break; + } + Matcher key = MAPPING_KEY.matcher(line); + if (!key.matches() || keyColumn(line) != keyIndent) { + continue; + } + String value = stripScalarComment(key.group(3) == null ? "" : key.group(3)); + if (key.group(2).equals("continue-on-error") && !value.equals("false")) { + return "its " + scope + " has continue-on-error: " + value + ", so its failure fails nothing"; + } + if (key.group(2).equals("if") && !STATUS_CONDITIONS.contains(expression(value))) { + return "its " + + scope + + " runs only if: " + + value + + ", not on every run (only always(), success() and !cancelled() count)"; + } + } + return null; + } + + /** The column a mapping key starts at, after a list item's {@code - }. */ + private static int keyColumn(String line) { + int column = indentOf(line); + if (line.startsWith("-", column)) { + column++; + while (column < line.length() && line.charAt(column) == ' ') { + column++; + } + } + return column; + } + + private static String expression(String condition) { + String plain = condition.trim(); + if (plain.startsWith("${{") && plain.endsWith("}}")) { + plain = plain.substring(3, plain.length() - 2).trim(); + } + return plain.replace(" ", "").toLowerCase(Locale.ROOT); + } + + /** The line of the job containing {@code index}: the last key at the indentation under {@code jobs:}. */ + private static int jobHeader(List lines, int index) { + int jobs = -1; + for (int line = 0; line < index; line++) { + if (lines.get(line).matches("^jobs:\\s*(#.*)?$")) { + jobs = line; + } + } + if (jobs < 0) { + return -1; + } + int jobIndent = -1; + for (int line = jobs + 1; line < index; line++) { + String text = lines.get(line); + if (!text.isBlank() && !text.trim().startsWith("#")) { + jobIndent = indentOf(text); + break; + } + } + int header = -1; + for (int line = jobs + 1; line < index; line++) { + Matcher job = JOB_KEY.matcher(lines.get(line)); + if (job.matches() && job.group(1).length() == jobIndent) { + header = line; + } + } + return header; + } + /** YAML folding: adjacent lines join with a space, and each blank line is a line break. */ private static String fold(List content) { StringBuilder folded = new StringBuilder(); @@ -204,7 +397,7 @@ public final class WorkflowGradleInvocations { return commands; } - private static Invocation gradleInvocation(String command) { + private static Invocation gradleInvocation(String command, String uncountedReason) { List tokens = new ArrayList<>(); for (String token : command.trim().split("\\s+")) { if (token.startsWith("#")) { @@ -227,6 +420,7 @@ public final class WorkflowGradleInvocations { String projectDir = null; boolean dryRun = false; List tasks = new ArrayList<>(); + List excluded = new ArrayList<>(); for (int index = start + 1; index < tokens.size(); index++) { String token = tokens.get(index); if (token.equals("--dry-run") || token.equals("-m")) { @@ -236,13 +430,20 @@ public final class WorkflowGradleInvocations { index++; } else if (token.startsWith("--project-dir=")) { projectDir = token.substring("--project-dir=".length()); + } else if (token.equals("-x") || token.equals("--exclude-task")) { + if (index + 1 < tokens.size()) { + excluded.add(tokens.get(index + 1)); + } + index++; + } else if (token.startsWith("--exclude-task=")) { + excluded.add(token.substring("--exclude-task=".length())); } else if (OPTIONS_WITH_VALUE.contains(token)) { index++; } else if (!token.startsWith("-")) { tasks.add(token); } } - return new Invocation(projectDir, tasks, dryRun); + return new Invocation(projectDir, tasks, dryRun, excluded, uncountedReason); } private static String unquote(String token) { diff --git a/src/build-logic/src/test/java/dev/caskeleton/buildlogic/config/ConfigurationMetadataClasspathTest.java b/src/build-logic/src/test/java/dev/caskeleton/buildlogic/config/ConfigurationMetadataClasspathTest.java new file mode 100644 index 00000000..c77ba3d0 --- /dev/null +++ b/src/build-logic/src/test/java/dev/caskeleton/buildlogic/config/ConfigurationMetadataClasspathTest.java @@ -0,0 +1,48 @@ +package dev.caskeleton.buildlogic.config; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ConfigurationMetadataClasspathTest { + + @Test + @DisplayName("classpath의 jar와 클래스 디렉터리에서 prefix 아래 typed property를 모두 읽는다") + void typedPropertiesUnderAPrefixAreReadFromEveryJarAndDirectoryOnTheClasspath( + @TempDir Path temp) throws Exception { + // given + Path classes = temp.resolve("classes"); + Files.createDirectories(classes.resolve("META-INF")); + Files.writeString( + classes.resolve("META-INF/spring-configuration-metadata.json"), + "{\"properties\":[{\"name\":\"app.redis.host\"},{\"name\":\"app.cache.ttl\"}]}"); + Path jar = temp.resolve("module.jar"); + try (OutputStream out = Files.newOutputStream(jar); + ZipOutputStream zip = new ZipOutputStream(out)) { + zip.putNextEntry(new ZipEntry("META-INF/spring-configuration-metadata.json")); + zip.write( + "{\"properties\":[{\"name\":\"app.redis.port\"}]}".getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + Path withoutMetadata = temp.resolve("plain"); + Files.createDirectories(withoutMetadata); + + // when + Set properties = + ConfigurationMetadataClasspath.propertiesUnder( + List.of(classes.toFile(), jar.toFile(), withoutMetadata.toFile()), "app.redis."); + + // then + assertThat(properties).containsExactly("app.redis.host", "app.redis.port"); + } +} diff --git a/src/build-logic/src/test/java/dev/caskeleton/buildlogic/config/EnvContractVerifierTest.java b/src/build-logic/src/test/java/dev/caskeleton/buildlogic/config/EnvContractVerifierTest.java index bc3410be..a14483c5 100644 --- a/src/build-logic/src/test/java/dev/caskeleton/buildlogic/config/EnvContractVerifierTest.java +++ b/src/build-logic/src/test/java/dev/caskeleton/buildlogic/config/EnvContractVerifierTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -230,6 +231,59 @@ class EnvContractVerifierTest { assertEquals(2, result.envKeyCount()); } + @Test + @DisplayName("compose lane 계약 json의 activationEnv 키가 레지스트리에 없으면 실패한다") + void aLaneActivationKeyMissingFromTheRegistryFails(@TempDir Path temp) throws Exception { + // given + Fixture fixture = fixture(temp); + Path contracts = fixture.repositoryRoot().resolve("compose-profile-contracts.json"); + Files.writeString( + contracts, + "{\"lanes\":[{\"id\":\"grpc\",\"activationEnv\":{\"CASKELETON_GRPC_ENABLED\":\"true\"," + + "\"APP_GRPC_UNREGISTERED\":\"true\",\"APP_HTTPCLIENT_CLIENTS_0_NAME\":\"a\"}}]}"); + + // when + IllegalStateException failure = + assertThrows( + IllegalStateException.class, + () -> verifyWithOperatorFiles(fixture, contracts)); + + // then + assertTrue(failure.getMessage().contains("APP_GRPC_UNREGISTERED"), failure.getMessage()); + assertTrue(!failure.getMessage().contains("APP_HTTPCLIENT_CLIENTS_0_NAME"), failure.getMessage()); + } + + @Test + @DisplayName("smoke 래퍼가 끄는 capability 키가 레지스트리에 없으면 실패한다") + void aSmokeWrapperSwitchMissingFromTheRegistryFails(@TempDir Path temp) throws Exception { + // given + Fixture fixture = fixture(temp); + Path smoke = fixture.repositoryRoot().resolve("run-compose-runtime-smoke.sh"); + Files.writeString( + smoke, + "CAPABILITIES_OFF='{\n \"APP_REQUIRED\": \"false\",\n \"APP_OBJECTSTORAGE_OFF\": \"false\"\n}'\n" + + "args+=(\"APP_NAME=ca-skeleton-${id}\")\n"); + + // when + IllegalStateException failure = + assertThrows(IllegalStateException.class, () -> verifyWithOperatorFiles(fixture, smoke)); + + // then + assertTrue(failure.getMessage().contains("APP_OBJECTSTORAGE_OFF"), failure.getMessage()); + assertTrue(!failure.getMessage().contains("APP_NAME"), failure.getMessage()); + } + + private static EnvContractResult verifyWithOperatorFiles(Fixture fixture, Path operatorFile) { + return EnvContractVerifier.verify( + fixture.envFile().toFile(), + List.of(fixture.applicationYaml().toFile()), + fixture.registry().toFile(), + List.of(), + List.of(new MetadataScope("app.redis.", fixture.redisMetadata().toFile())), + List.of("APP_REDIS_"), + List.of(operatorFile.toFile())); + } + private static Fixture fixture(Path temp) throws Exception { Path repositoryRoot = temp.resolve("repository/src"); Path docsRoot = temp.resolve("repository/docs/registries"); diff --git a/src/build-logic/src/test/java/dev/caskeleton/buildlogic/strictlane/StrictLaneStagesFunctionalTest.java b/src/build-logic/src/test/java/dev/caskeleton/buildlogic/strictlane/StrictLaneStagesFunctionalTest.java index c8397e41..9c038036 100644 --- a/src/build-logic/src/test/java/dev/caskeleton/buildlogic/strictlane/StrictLaneStagesFunctionalTest.java +++ b/src/build-logic/src/test/java/dev/caskeleton/buildlogic/strictlane/StrictLaneStagesFunctionalTest.java @@ -186,6 +186,135 @@ class StrictLaneStagesFunctionalTest { assertThat(result.getOutput()).contains("main.yml line 13: '--stacktrace' follows a blank line"); } + @Test + @DisplayName("단계를 실행해도 -x로 뺀 lane은 덮이지 않으므로 그 lane 이름으로 실패한다") + void aLaneExcludedWithDashXIsNotCoveredByTheStageThatRuns() throws Exception { + // given + leafLanes( + """ + lane('databaseLane') { integration(); tag = 'db'; description = 'Real database.' } + lane('cacheLane') { integration(); tag = 'cache'; description = 'Real cache.' } + """); + workflow( + """ + jobs: + main: + steps: + - run: ./gradlew :ci --stacktrace + - run: ./gradlew :integrationCheck -x :leaf:databaseLane --continue + """); + + // when + BuildResult result = runner(":verifyStrictLaneStages", "--dry-run").buildAndFail(); + + // then + assertThat(result.getOutput()) + .contains("strict lane :leaf:databaseLane is in no executed stage") + .doesNotContain("strict lane :leaf:cacheLane"); + } + + @Test + @DisplayName("continue-on-error job이나 단계가 부르는 단계는 실패해도 막지 못하므로 실행으로 세지 않는다") + void aStageRunUnderContinueOnErrorIsNotAnExecution() throws Exception { + // given + leafLanes( + """ + lane('databaseLane') { integration(); tag = 'db'; description = 'Real database.' } + """); + workflow( + """ + jobs: + gate: + steps: + - run: ./gradlew :ci --stacktrace + advisory: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - run: ./gradlew :integrationCheck + other: + steps: + - name: Advisory step + continue-on-error: true + run: ./gradlew :integrationCheck + """); + + // when + BuildResult result = runner(":verifyStrictLaneStages", "--dry-run").buildAndFail(); + + // then + assertThat(result.getOutput()) + .contains("stage :integrationCheck is executed by no workflow") + .contains("strict lane :leaf:databaseLane is in no executed stage"); + } + + @Test + @DisplayName("상태 함수가 아닌 if 조건 아래의 단계 실행은 매번 돈다고 볼 수 없으므로 세지 않는다") + void aStageRunUnderAConditionIsNotAnExecution() throws Exception { + // given + leafLanes( + """ + lane('databaseLane') { integration(); tag = 'db'; description = 'Real database.' } + """); + workflow( + """ + jobs: + gate: + steps: + - run: ./gradlew :ci --stacktrace + disabled: + if: ${{ false }} + steps: + - run: ./gradlew :integrationCheck + manual: + steps: + - if: github.event_name == 'workflow_dispatch' + run: ./gradlew :integrationCheck + reported: + steps: + - if: always() + run: ./gradlew :integrationCheck --dry-run + """); + + // when + BuildResult result = runner(":verifyStrictLaneStages", "--dry-run").buildAndFail(); + + // then + assertThat(result.getOutput()).contains("stage :integrationCheck is executed by no workflow"); + } + + @Test + @DisplayName("다른 빌드로 실행되는 빌드는 workflow가 -p 그 디렉터리로 부른 단계만 실행으로 센다") + void aBuildStartedWithDashPCountsOnlyCommandsThatNameItsDirectory() throws Exception { + // given + leafLanes( + """ + lane('databaseLane') { integration(); tag = 'db'; description = 'Real database.' } + """); + Files.writeString( + gradleRoot.resolve("build.gradle"), + Files.readString(gradleRoot.resolve("build.gradle")) + + "strictLaneStages { workflowProjectDirectory = 'optional' }\n"); + workflow( + """ + jobs: + main: + steps: + - working-directory: src + run: ./gradlew -p optional ci --stacktrace + - working-directory: src + run: ./gradlew :integrationCheck + """); + + // when + BuildResult result = runner(":verifyStrictLaneStages", "--dry-run").buildAndFail(); + + // then + assertThat(result.getOutput()) + .contains("stage :integrationCheck is executed by no workflow") + .doesNotContain("stage :ci is executed by no workflow"); + } + private void leafLanes(String lanes) throws Exception { Files.writeString( gradleRoot.resolve("leaf/build.gradle"), diff --git a/src/build-logic/src/test/java/dev/caskeleton/buildlogic/strictlane/StrictTestLaneConventionTest.java b/src/build-logic/src/test/java/dev/caskeleton/buildlogic/strictlane/StrictTestLaneConventionTest.java index 5a9aca60..5cda0950 100644 --- a/src/build-logic/src/test/java/dev/caskeleton/buildlogic/strictlane/StrictTestLaneConventionTest.java +++ b/src/build-logic/src/test/java/dev/caskeleton/buildlogic/strictlane/StrictTestLaneConventionTest.java @@ -546,6 +546,42 @@ class StrictTestLaneConventionTest { TaskOutcome.FROM_CACHE, TaskOutcome.FROM_CACHE, TaskOutcome.SUCCESS, TaskOutcome.SUCCESS); } + @Test + @DisplayName("lane 정책만 바꾼 다음 실행은 이전 결과를 다시 쓰지 않고 새 정책으로 판정한다") + void aLanePolicyChangeIsJudgedAgainInsteadOfReusingTheEarlierResult() throws Exception { + // given + String lane = + """ + strictTestLanes { + lane('carriedLane') { + tag = 'carried' + description = 'Selected tests.' + rejectSkipped = %s + } + } + """; + buildFile(junitDependencies() + lane.formatted("false")); + writeTest( + "CarriedTest.java", + """ + import org.junit.jupiter.api.Disabled; + import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; + class CarriedTest { + @Test @Tag("carried") void ran() {} + @Test @Tag("carried") @Disabled("skipped while the policy allowed it") void skipped() {} + } + """); + runner("carriedLane", "--build-cache").build(); + buildFile(junitDependencies() + lane.formatted("true")); + + // when + BuildResult stricter = runner("carriedLane", "--build-cache").buildAndFail(); + + // then + assertThat(stricter.getOutput()).contains("forbids skipped tests"); + } + @Test @DisplayName("서버를 띄우지 않는 TEST·ARCHITECTURE lane은 실서버 lane의 단일 슬롯을 잡지 않는다") void onlyServerLanesTakeTheExclusiveRealServerSlot() throws Exception { diff --git a/src/build-logic/src/test/java/dev/caskeleton/buildlogic/strictlane/WorkflowGradleInvocationsTest.java b/src/build-logic/src/test/java/dev/caskeleton/buildlogic/strictlane/WorkflowGradleInvocationsTest.java index 155c53e6..3bd5f940 100644 --- a/src/build-logic/src/test/java/dev/caskeleton/buildlogic/strictlane/WorkflowGradleInvocationsTest.java +++ b/src/build-logic/src/test/java/dev/caskeleton/buildlogic/strictlane/WorkflowGradleInvocationsTest.java @@ -197,4 +197,56 @@ class WorkflowGradleInvocationsTest { // then assertThat(problems).isEmpty(); } + + @Test + @DisplayName("-x와 --exclude-task로 뺀 태스크는 경로나 이름이 맞는 태스크만 제외로 읽는다") + void excludedTasksAreReadFromBothSpellings() { + // given + String workflow = + """ + - run: ./gradlew :integrationCheck -x :leaf:databaseLane --exclude-task=test + """; + + // when + Invocation invocation = WorkflowGradleInvocations.read(workflow).get(0); + + // then + assertThat(invocation.excludes(":leaf:databaseLane")).isTrue(); + assertThat(invocation.excludes(":other:test")).isTrue(); + assertThat(invocation.excludes(":leaf:cacheLane")).isFalse(); + assertThat(invocation.executesInStartBuild(":integrationCheck")).isTrue(); + } + + @Test + @DisplayName("job의 continue-on-error와 단계의 if 조건은 명령을 실행으로 세지 않는 이유로 남는다") + void jobAndStepGatingIsKeptAsTheReasonACommandDoesNotCount() { + // given + String workflow = + """ + jobs: + advisory: + continue-on-error: true + steps: + - run: ./gradlew :ci + gated: + steps: + - name: Manual only + if: github.event_name == 'workflow_dispatch' + run: ./gradlew :ci + plain: + steps: + - if: ${{ always() }} + run: ./gradlew :ci + """; + + // when + List invocations = WorkflowGradleInvocations.read(workflow); + + // then + assertThat(invocations).extracting(Invocation::uncountedReason) + .satisfiesExactly( + reason -> assertThat(reason).contains("job has continue-on-error: true"), + reason -> assertThat(reason).contains("step runs only if: github.event_name"), + reason -> assertThat(reason).isNull()); + } } diff --git a/src/composition/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverFileUseCaseConfiguration.java b/src/composition/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverFileUseCaseConfiguration.java index 163f2181..b9e80d5b 100644 --- a/src/composition/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverFileUseCaseConfiguration.java +++ b/src/composition/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverFileUseCaseConfiguration.java @@ -6,6 +6,7 @@ import dev.caskeleton.application.fileserver.api.metadata.FileQuotaPort; import dev.caskeleton.application.fileserver.api.metadata.UploadSessionPort; import dev.caskeleton.application.fileserver.api.security.FileAccessPolicyPort; import dev.caskeleton.application.fileserver.cleanup.CleanupQueuePort; +import dev.caskeleton.application.fileserver.cleanup.QuotaReclaimPort; import dev.caskeleton.application.fileserver.download.DescribeFileApplicationUseCase; import dev.caskeleton.application.fileserver.download.DownloadContentPort; import dev.caskeleton.application.fileserver.download.OpenDownloadContentApplicationUseCase; @@ -120,10 +121,12 @@ class FileserverFileUseCaseConfiguration { @ConditionalOnMissingBean MoveFileUseCase fileserverMoveFile( FileMetadataPort metadataStore, - FileAccessPolicyPort accessPolicy, + UploadPolicies policies, TargetNamespacePolicy targetNamespaces, + FileQuotaPort quotaService, + QuotaReclaimPort quotaReclaim, TransactionPort transactions) { return new MoveFileApplicationUseCase( - metadataStore, accessPolicy, targetNamespaces, transactions); + metadataStore, policies, targetNamespaces, quotaService, quotaReclaim, transactions); } } diff --git a/src/composition/idempotency/src/main/java/dev/caskeleton/composition/idempotency/IdempotencySettings.java b/src/composition/idempotency/src/main/java/dev/caskeleton/composition/idempotency/IdempotencySettings.java index b43ec421..e56a0ccc 100644 --- a/src/composition/idempotency/src/main/java/dev/caskeleton/composition/idempotency/IdempotencySettings.java +++ b/src/composition/idempotency/src/main/java/dev/caskeleton/composition/idempotency/IdempotencySettings.java @@ -2,6 +2,7 @@ package dev.caskeleton.composition.idempotency; import dev.caskeleton.adapter.outbound.support.StrictSettingsBinder; import dev.caskeleton.adapter.outbound.support.startup.StartupFailures; +import dev.caskeleton.application.idempotency.IdempotencyExecutor; import java.time.Duration; import java.util.Set; import org.springframework.core.env.Environment; @@ -17,7 +18,9 @@ import org.springframework.core.env.Environment; * instead of leaving its default silently in force. * * @param provider which store is composed: {@code disabled}, {@code redis} or {@code postgresql} - * @param processingLease how long one attempt holds a claim before another may take it over + * @param processingLease how long one attempt holds a claim before another may take it over; longer + * than the in-flight 409's {@code Retry-After} ({@link + * IdempotencyExecutor#IN_FLIGHT_RETRY_AFTER}) * @param replayTtl how long a completed response is replayable (at most 72h, D6) * @param failureRetention how long a failed attempt's evidence is kept * @param responseCodecId identifier of the codec the stored payload is written with @@ -49,6 +52,16 @@ public record IdempotencySettings( "processing-lease", orDefault(processingLease, Duration.ofSeconds(30)), MAX_PROCESSING_LEASE); + // A duplicate is told to come back after IN_FLIGHT_RETRY_AFTER. With a lease no longer than + // that, the resend always arrives after the attempt it was waiting for has lost its claim. + if (processingLease.compareTo(IdempotencyExecutor.IN_FLIGHT_RETRY_AFTER) <= 0) { + throw StartupFailures.envValidation( + PREFIX + + ".processing-lease must be longer than " + + IdempotencyExecutor.IN_FLIGHT_RETRY_AFTER + + " (the in-flight Retry-After), was " + + processingLease); + } replayTtl = bounded("replay-ttl", orDefault(replayTtl, Duration.ofHours(24)), MAX_REPLAY_TTL); failureRetention = bounded( diff --git a/src/composition/idempotency/src/test/java/dev/caskeleton/composition/idempotency/IdempotencySettingsTest.java b/src/composition/idempotency/src/test/java/dev/caskeleton/composition/idempotency/IdempotencySettingsTest.java index 37903827..1c573f2d 100644 --- a/src/composition/idempotency/src/test/java/dev/caskeleton/composition/idempotency/IdempotencySettingsTest.java +++ b/src/composition/idempotency/src/test/java/dev/caskeleton/composition/idempotency/IdempotencySettingsTest.java @@ -151,6 +151,31 @@ class IdempotencySettingsTest { .hasMessageContaining("app.capabilities.idempotency." + message); } + /** + * The in-flight 409 tells the client to come back after {@code Retry-After} (2 seconds). A lease + * no longer than that sends every resend past the attempt it was meant to wait for, so the rule + * fails startup rather than living in prose (re-review 2026-11-03, RR-02-05). + */ + @ParameterizedTest + @CsvSource({"PT1S", "PT2S"}) + @DisplayName("processing-lease 가 처리 중 409 의 Retry-After 이하면 기동을 거부한다") + void aProcessingLeaseNoLongerThanTheInFlightRetryAfterIsRefused(Duration lease) { + // when + Throwable thrown = + catchThrowable( + () -> + new IdempotencySettings( + IdempotencySettings.Provider.REDIS, lease, null, null, null, null, null)); + + // then + assertThat(thrown) + .isInstanceOfSatisfying( + StartupFailureException.class, + e -> assertThat(e.errorCode()).isEqualTo(StartupErrorCode.STARTUP_VALIDATION_FAILED)) + .hasMessageContaining( + "app.capabilities.idempotency.processing-lease must be longer than PT2S"); + } + @ParameterizedTest @CsvSource({"'', response-codec-id must contain 1-64 characters", "' ', response-codec-id"}) void blankCodecIdIsRefused(String codecId, String message) { diff --git a/src/composition/outbox-relay/CLAUDE.md b/src/composition/outbox-relay/CLAUDE.md new file mode 100644 index 00000000..f63f9fa9 --- /dev/null +++ b/src/composition/outbox-relay/CLAUDE.md @@ -0,0 +1,21 @@ +# composition:outbox-relay + +- Module ID `composition-outbox-relay`, Gradle path `:composition:outbox-relay`. Registry SSOT: + `src/config/architecture/modules.json`. +- Responsibility: assemble and drive the application's outbox relay. It builds + `PublishPendingOutboxEventsUseCase` by hand from `app.outbox.*` (`OutboxSettings`, strict binding), + runs it from `OutboxRelayScheduler` (fixed-delay ticks, a `SmartLifecycle` that stops above the + messaging platform's admission phase), and owns the outbox metrics (`OutboxMetrics`: gauges on a + thread of their own, the cycle-outcome counter from the relay). +- Boundaries: it names ports only — `OutboxStorePort`, `OutboxMessagePublishPort`, + `OutboxRelayFailureReportPort`, `TransactionPort` — never a module type, so removing the + persistence or messaging line removes the relay. No business rule, no SQL, no broker type. The + relay use case is never a bean (`@SystemInvoked`, no permission gate). Allowed edges come only from + the registry entry. +- Startup refusals live here because only here do the settings meet the ports: a batch whose worst + case (`batch-size` × the publish port's `longestAttempt`) reaches `in-flight-timeout` fails startup. +- Tests: `./gradlew :composition:outbox-relay:check`. The shutdown order is pinned against the + platform's own phase constant (`OutboxRelayShutdownOrderTest`); the end-to-end relay → bridge → real + Kafka path is `:adapter:outbound:messaging-kafka:messagingKafkaRelayIntegrationTest`. +- Design and rejected alternatives: `README.md`. Operations: `docs/runbooks/outbox-publish-failed.md`, + `docs/runbooks/outbox-dead-letter.md`. diff --git a/src/composition/outbox-relay/README.md b/src/composition/outbox-relay/README.md index dc861755..87cbc7b8 100644 --- a/src/composition/outbox-relay/README.md +++ b/src/composition/outbox-relay/README.md @@ -18,12 +18,15 @@ composition:outbox-relay │ PublishPendingOutboxEventsUseCase 를 fixed-delay 로 구동, │ 발행이 이어지는 동안 한 tick 안에서 바로 다시 claim(최대 50회차), │ 종료 때 platform보다 먼저 멈추고 남은 claim을 놓음(SmartLifecycle) - ├─ OutboxMetrics outbox.pending.size / outbox.publisher.lag(relay와 무관하게 - │ poll 주기마다 갱신) / published.total(relay 회차 결과) - ├─ OutboxRelayOffWarning 발행 포트가 있는데 relay가 꺼져 있으면 기동 시 WARN 한 줄 - └─ OutboxLeaderElectionToken SKIP LOCKED 가 리더 선출이라는 표시(로직 없음) + ├─ OutboxMetrics outbox.pending.size / outbox.publisher.lag / outbox.retry.long.* + │ (자기 스레드에서 poll 주기마다 갱신) / published.total(relay 회차 결과) + └─ OutboxRelayOffWarning 발행 포트가 있는데 relay가 꺼져 있으면 기동 시 WARN 한 줄 ``` +여러 인스턴스가 함께 relay를 돌려도 조정자는 없다. claim SQL의 `FOR UPDATE SKIP LOCKED`가 인스턴스마다 +겹치지 않는 행을 주고, lease token이 늦은 완료를 막는다(`PostgreSqlOutboxClaimRepository`, V17). 리더를 +뽑지 않는다. + - 조립 순서: `PersistenceJpaAutoConfiguration`(store)과 `MessagingBridgeRootAutoConfiguration`(발행 포트) 뒤에 온다(`afterName`, 문자열이라 컴파일 의존이 없다). 둘 다 포트라서 이 모듈은 어떤 모듈 타입도 이름으로 부르지 않는다. @@ -73,7 +76,8 @@ poll 주기·batch 크기·in-flight timeout·보존 기간은 env 키 없이 `a relay 생성자에 전달한다. - **릴레이 use case 를 독립 컨텍스트 빈으로 등록하지 않는다.** relay는 스케줄러만 부르는 `@SystemInvoked` 유스케이스라 권한 선언이 없다. 빈으로 올리면 전송 계층이 권한 검사 없이 닿을 - 수 있다. 스케줄러 스레드에는 `Authentication`이 없으므로 권한 게이트를 달아도 매 틱 거부될 뿐이다. + 수 있으므로, 빈으로 등록된 `@SystemInvoked` 유스케이스는 `composition:service-runtime`의 + `PermissionEnforcementGuard`가 기동을 실패시킨다. 스케줄러 스레드에는 `Authentication`이 없으므로 권한 게이트를 달아도 매 틱 거부될 뿐이다. - **`outboxRelayScheduler` 빈은 `app.outbox.relay-enabled=true`일 때만 만든다(기본 false).** 키가 없으면 꺼진 것으로 본다. 모듈 기본값(`persistence-jpa.yml`의 `${APP_OUTBOX_RELAY_ENABLED:false}`), `@ConditionalOnProperty`, `OutboxSettings`의 null 기본값이 @@ -90,28 +94,31 @@ poll 주기·batch 크기·in-flight timeout·보존 기간은 env 키 없이 `a 코드다. driven adapter인 persistence-jpa가 use case를 만들거나 호출하면 `PERSISTENCE_DOES_NOT_DEPEND_ON_APPLICATION_SERVICES` 규칙 위반이다. 모듈 타입은 하나도 이름으로 부르지 않고 포트만 쓰므로, persistence-jpa 한 줄을 지우면 릴레이도 함께 꺼진다. -- **`outboxLeaderElection` 토큰 빈은 outbox capability 안에서 조건 없이 등록한다.** SKIP LOCKED 는 - 멀티 인스턴스 전역 스위치와 무관하게 outbox relay 자체의 리더십/claim 메커니즘이라, outbox가 켜진 - 상태에서 별도 하위 토글로 끌 수 있게 만들지 않는다. 이 토큰은 outbox 조립의 자기 설명용 marker이지 - `APP_MULTI_INSTANCE_ENABLED`의 전역 필수 빈이 아니다. +- **`outboxLeaderElection` 토큰 빈은 지웠다(리뷰 2026-11-01 06-13).** 로직 없는 marker였고, Javadoc은 + "있기만 하면 멀티 인스턴스 조정 요구를 채운다"고 했지만 그 빈을 요구하는 검사도, 리더 선출도 없었다 + (claim은 행 단위다). 버린 대안: 이름을 실제 동작(`OutboxClaimStrategy`)으로 바꾸기 — 여전히 아무도 + 읽지 않는 marker다. 동작 설명은 위 트리 아래 문단과 `PostgreSqlOutboxClaimRepository`에 있다. +- **relay 조립은 배치가 lease 안에 다 나가는지 확인한다(06-12).** 한 회차는 batch 전체를 lease 하나로 + 잡고 차례로 발행한다. `batch-size` × 발행 포트의 `longestAttempt()`(bridge: platform 발행 기한의 두 배, + 기본 10초)가 `in-flight-timeout` 이상이면 기동을 거부한다. lease가 먼저 끝나면 다른 인스턴스가 같은 행을 + 다시 보내기 때문이다(fencing은 상태 덮어쓰기만 막는다). 기본값은 20 × 10초 = 200초 < 5분이다. 포트가 + 최악 시간을 모르면(`Optional.empty()`) 검사하지 않는다. 버린 대안: relay가 회차 중 남은 lease를 보고 + 남은 행을 놓기 — 설정 실수를 운영 중 중복 대신 기동 실패로 드러내는 쪽을 골랐다. - **`outboxMetrics` 빈은 `MeterRegistry`가 없을 때 no-op 이다.** Actuator 가 클래스패스에 없는 환경에서도 와이어링이 깨지지 않도록 `ObjectProvider`로 레지스트리를 선택적으로 주입한다. -### OutboxLeaderElectionToken -- **로직이 전혀 없는 마커(documentation artifact) 빈이다.** 이 타입은 동작을 갖지 않고, Spring - 컨텍스트에 존재한다는 사실만으로 멀티 인스턴스 조정 빈 요구사항을 충족시킨다. -- **리더 선출을 외부 코디네이터 없이 PostgreSQL `FOR UPDATE SKIP LOCKED`로 구현한다는 것을 - 표현한다.** claim 쿼리에 SKIP LOCKED 를 걸면 각 릴레이 인스턴스가 서로 겹치지 않는(disjoint) 행 - 집합을 가져가므로, 별도 코디네이터 없이도 "각 인스턴스가 자기 파티션의 리더" 형태로 리더 선출 - 의미가 성립한다. - ### OutboxMetrics - **`MeterRegistry`가 없으면 모든 메트릭 연산이 no-op 이다.** Actuator 가 없는 환경에서도 릴레이가 정상 동작해야 하므로, 생성자에서 `ObjectProvider`로 레지스트리를 조회해 없으면 게이지도 만들지 않고 기록도 건너뛴다. -- **카운터는 PUBLISHED/FAILED/DEAD 만 집계하고 IN_FLIGHT 는 제외한다.** IN_FLIGHT 는 릴레이 사이클의 - 종료 결과(terminal outcome)가 아니라 처리 중을 나타내는 일시적 상태라 카운터 outcome 으로 의미가 - 없다. +- **카운터 outcome은 relay 회차 결과(`OutboxRelayResult.Outcome`) 다섯 값이다: PUBLISHED, FAILED, DEAD, + STALE, RELEASED.** IN_FLIGHT는 행 상태이지 회차 결과가 아니라 세지 않는다. `docs/registries/metrics.yaml`의 + 허용 값과 enum이 같은지 `OutboxStatusRegistryContractTest`가 비교한다(재리뷰 2026-11-03 RR-05-06). +- **오래 재시도 중인 행을 따로 센다(RR-05-03).** 응답 없음·지금만 거부는 시도 예산을 쓰지 않아 끝이 없다. + `outbox.retry.long.size`는 미룬 claim이 `LONG_RETRY_DEFERRED_CLAIMS`(10)번 이상인 FAILED 행 수, + `outbox.retry.long.age`는 그중 가장 오래된 행의 나이다. 10번은 backoff 상한 5분 기준으로 적어도 약 35분 + 동안 실패했다는 뜻이라 broker 재시작·credential 교체는 넘긴다. publish-failed runbook §4.1의 조회가 같은 + 기준을 쓴다. - **MultiGauge 갱신 시 `overwrite=true`로 등록한다.** 매 스케줄러 틱마다 store 를 다시 조회해 게이지를 갱신하는데, 이전 틱에 있었지만 지금은 사라진 event type 같은 오래된 time-series 태그를 덮어써 제거하기 위함이다. @@ -151,13 +158,18 @@ poll 주기·batch 크기·in-flight timeout·보존 기간은 env 키 없이 `a claim이 비면 tick을 끝낸다. 한 tick의 상한은 `MAX_CYCLES_PER_TICK` = 50회차다. - 처리량 상한(계산, 실측 아님): 한 aggregate는 poll 주기가 아니라 발행 왕복 시간에 묶인다(정상 broker에서 회차당 claim·완료 DB 왕복 + Kafka ack). 인스턴스 하나는 tick당 최대 50 × batch 20 = - 1,000행을 보내고 다음 tick까지 5초 쉰다. 넘는 처리량이 필요하면 batch-size(NN-12의 lease 관계 확인)나 + 1,000행을 보내고 다음 tick까지 5초 쉰다. 넘는 처리량이 필요하면 batch-size(06-12: lease보다 길면 기동이 거부된다)나 인스턴스 수로 늘린다. - 버린 대안: poll 주기만 줄이기 — 빈 tick마다 claim·지표 질의가 는다. -- **지표 게이지는 relay와 따로 갱신한다(06-05).** `outbox.pending.size`·`outbox.publisher.lag`는 - `OutboxMetrics.refreshGauges`가 poll 주기마다 갱신한다(`@Scheduled`, relay가 꺼져 있어도, 회차가 매번 - 예외로 끝나도). 예전에는 relay 회차 안에서만 갱신되어, relay가 꺼진 배포(기본값)나 매 회차 실패하는 - 장애에서 게이지가 멈췄고 `outbox_publisher_lag` 경보가 울리지 않았다. 카운터 +- **지표 게이지는 relay와 따로, 자기 스레드에서 갱신한다(06-05, RR-05-05).** `outbox.pending.size`· + `outbox.publisher.lag`·`outbox.retry.long.*`는 `OutboxMetrics`가 poll 주기마다 갱신한다(relay가 꺼져 + 있어도, 회차가 매번 예외로 끝나도). 예전에는 relay 회차 안에서만 갱신되어, relay가 꺼진 배포(기본값)나 + 매 회차 실패하는 장애에서 게이지가 멈췄다. 그다음에는 `@Scheduled`로 떼었지만 Spring의 fixed-delay 작업은 + 모두 스레드 하나(virtual thread면 `SimpleAsyncTaskScheduler`의 fixedDelayExecutor, 아니면 pool 1)에서 + 줄을 서서, relay tick이 50회차를 도는 동안(느린 broker면 몇 분) 게이지가 기다렸다 — 적체를 봐야 할 바로 + 그때다. 지금은 `OutboxMetrics`가 `SmartLifecycle`로 자기 단일 daemon 스레드(`outbox-gauges`)를 시작하고 + 멈춘다. 버린 대안: tick 상한을 회차 수 대신 시간으로 — 게이지가 여전히 relay 뒤에 줄을 선다. Micrometer + 지연 조회 gauge — scrape마다 DB 질의가 나가고, event type별 series를 덮어쓸 수 없다. 카운터 `outbox.publisher.published.total`은 회차 결과로 relay가 기록한다. 발행 포트가 있는데 relay가 꺼져 있으면 `OutboxRelayOffWarning`이 기동 시 WARN 한 줄을 남긴다(별도 worker 배포가 relay를 돌리는 구성도 있으므로 기동 실패로 만들지 않았다). diff --git a/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxLeaderElectionToken.java b/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxLeaderElectionToken.java deleted file mode 100644 index 4e4e9887..00000000 --- a/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxLeaderElectionToken.java +++ /dev/null @@ -1,21 +0,0 @@ -package dev.caskeleton.composition.outbox; - -/** - * Marker bean: presence in the context satisfies the multi-instance coordination requirement (no - * logic; SKIP LOCKED is the leadership mechanism). See README for the design rationale. - */ -public final class OutboxLeaderElectionToken { - - private static final String STRATEGY_DESCRIPTION = - "SKIP LOCKED claim — each relay instance claims a disjoint row partition; " - + "no external coordinator required (PostgreSQL FOR UPDATE SKIP LOCKED, I3/D8)"; - - public String strategyDescription() { - return STRATEGY_DESCRIPTION; - } - - @Override - public String toString() { - return "OutboxLeaderElectionToken{strategy=SKIP_LOCKED}"; - } -} diff --git a/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxMetrics.java b/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxMetrics.java index 315ba7a0..64cb3cfe 100644 --- a/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxMetrics.java +++ b/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxMetrics.java @@ -1,49 +1,91 @@ package dev.caskeleton.composition.outbox; import dev.caskeleton.adapter.outbound.support.startup.RuntimeWarnings; +import dev.caskeleton.application.outbox.OutboxBackoffPolicy; import dev.caskeleton.application.outbox.OutboxEventStatus; +import dev.caskeleton.application.outbox.OutboxLongRetries; import dev.caskeleton.application.outbox.OutboxRelayResult; import dev.caskeleton.application.outbox.OutboxStorePort; import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.MultiGauge; import io.micrometer.core.instrument.Tags; import java.time.Clock; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import org.springframework.beans.factory.ObjectProvider; -import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.context.SmartLifecycle; /** * Outbox metric collector. No-op when no {@link MeterRegistry} is present (resolved via {@link * ObjectProvider}). See README for the design rationale. * - *

    The gauges ({@code outbox.pending.size}, {@code outbox.publisher.lag}) refresh on their own - * schedule, not from the relay. They are how an operator sees rows piling up, and the two moments - * rows pile up are exactly the ones a relay-driven refresh missed: the relay is off — it is by - * default, and a deployment that forgot {@code APP_OUTBOX_RELAY_ENABLED} looked idle — or every - * cycle throws, which skipped the refresh and froze the gauges at their last value. The counter - * ({@code outbox.publisher.published.total}) is still recorded by the relay, from each cycle's - * outcomes. + *

    The gauges ({@code outbox.pending.size}, {@code outbox.publisher.lag}, {@code + * outbox.retry.long.size}, {@code outbox.retry.long.age}) refresh every poll interval on a thread + * of their own, not from the relay and not on the scheduler the relay runs on. They are how an + * operator sees rows piling up, and rows pile up exactly when the relay is off — it is by default — + * when every cycle throws, or when the broker is slow: a relay tick runs up to {@link + * OutboxRelayScheduler#MAX_CYCLES_PER_TICK} cycles back to back, and every fixed-delay task of the + * application scheduler waits behind it on one thread (review 06-05, re-review 2026-11-03 + * RR-05-05). The counter ({@code outbox.publisher.published.total}) is still recorded by the relay, + * from each cycle's outcomes. */ -public final class OutboxMetrics { +public final class OutboxMetrics implements SmartLifecycle { + + /** + * How many deferred claims make a FAILED row long-retrying. A deferred claim does not spend the + * attempt budget, so such a row is retried until someone looks; ten of them, with the backoff + * capped at {@link OutboxBackoffPolicy#MAX_DELAY}, is at least about 35 minutes of failing — past + * any broker restart or credential rotation. The publish-failed runbook's query uses the same + * number. + */ + static final int LONG_RETRY_DEFERRED_CLAIMS = 10; private static final String COUNTER_NAME = "outbox.publisher.published.total"; private static final String SIZE_GAUGE = "outbox.pending.size"; private static final String LAG_GAUGE = "outbox.publisher.lag"; + private static final String LONG_RETRY_SIZE_GAUGE = "outbox.retry.long.size"; + private static final String LONG_RETRY_AGE_GAUGE = "outbox.retry.long.age"; private final OutboxStorePort store; private final Clock clock; + private final Duration refreshInterval; private final MeterRegistry registry; // null when no Actuator on classpath private final MultiGauge pendingSizeGauge; private final MultiGauge publisherLagGauge; + private final AtomicLong longRetryRows = new AtomicLong(); + private final AtomicLong longRetryOldestAge = new AtomicLong(); + private final Object monitor = new Object(); + private ScheduledExecutorService refresher; + private ScheduledFuture refreshing; + + /** + * Creates the collector. + * + * @param store the outbox store the gauges read + * @param meterRegistryProvider the registry, when Actuator is present + * @param clock wall clock for the age gauges + * @param refreshInterval how often the gauges refresh once started; the relay's poll interval + */ public OutboxMetrics( - OutboxStorePort store, ObjectProvider meterRegistryProvider, Clock clock) { + OutboxStorePort store, + ObjectProvider meterRegistryProvider, + Clock clock, + Duration refreshInterval) { this.store = store; this.clock = clock; + this.refreshInterval = Objects.requireNonNull(refreshInterval, "refreshInterval"); this.registry = meterRegistryProvider.getIfAvailable(); if (this.registry != null) { @@ -56,13 +98,26 @@ public final class OutboxMetrics { .description("Age in seconds of the oldest unpublished row per event type") .baseUnit("seconds") .register(this.registry); + Gauge.builder(LONG_RETRY_SIZE_GAUGE, longRetryRows, AtomicLong::get) + .description( + "FAILED outbox rows deferred at least " + + LONG_RETRY_DEFERRED_CLAIMS + + " times: retried without end until someone looks") + .register(this.registry); + Gauge.builder(LONG_RETRY_AGE_GAUGE, longRetryOldestAge, AtomicLong::get) + .description("Age in seconds of the oldest long-retrying outbox row; 0 when none") + .baseUnit("seconds") + .register(this.registry); } else { this.pendingSizeGauge = null; this.publisherLagGauge = null; } } - /** Counts each per-event outcome (PUBLISHED, FAILED, DEAD, STALE; never IN_FLIGHT). */ + /** + * Counts each per-event outcome of a relay cycle: PUBLISHED, FAILED, DEAD, STALE and RELEASED. + * IN_FLIGHT is a row status, never a cycle outcome. + */ public void recordRelayResult(OutboxRelayResult result) { if (registry == null || result == null) { return; @@ -84,11 +139,9 @@ public final class OutboxMetrics { } /** - * Refreshes the gauges every poll interval, whether or not the relay runs. Store failures are - * contained per gauge (a WARN through {@link RuntimeWarnings}), so a database outage leaves the - * scheduler thread alive for the next refresh. + * Refreshes the gauges once. Store failures are contained per gauge (a WARN through {@link + * RuntimeWarnings}), so a database outage leaves the refresh thread alive for the next one. */ - @Scheduled(fixedDelayString = "${app.outbox.poll-interval:PT5S}") public void refreshGauges() { refresh(clock.instant()); } @@ -97,7 +150,7 @@ public final class OutboxMetrics { * Uses {@code overwrite=true} so stale time-series tags from removed event types are replaced * rather than left dangling. * - * @param now current wall-clock instant (for the lag gauge age computation) + * @param now current wall-clock instant (for the age gauges) */ public void refresh(Instant now) { if (registry == null) { @@ -105,6 +158,57 @@ public final class OutboxMetrics { } refreshPendingSize(); refreshPublisherLag(now); + refreshLongRetries(now); + } + + /** Starts the gauges' own refresh thread; nothing to start without a registry. */ + @Override + public void start() { + synchronized (monitor) { + if (registry == null || refresher != null) { + return; + } + refresher = + Executors.newSingleThreadScheduledExecutor( + task -> { + Thread thread = new Thread(task, "outbox-gauges"); + thread.setDaemon(true); + return thread; + }); + // A periodic task that throws is never run again, silently; each gauge already contains its + // own store failure, and anything else is contained here so the next refresh still happens. + refreshing = + refresher.scheduleWithFixedDelay( + () -> { + try { + refreshGauges(); + } catch (RuntimeException unexpected) { + RuntimeWarnings.meterUnavailable("outbox", "gauge refresh", unexpected); + } + }, + 0, + refreshInterval.toMillis(), + TimeUnit.MILLISECONDS); + } + } + + @Override + public void stop() { + synchronized (monitor) { + if (refresher != null) { + refreshing.cancel(true); + refresher.shutdownNow(); + refresher = null; + refreshing = null; + } + } + } + + @Override + public boolean isRunning() { + synchronized (monitor) { + return refresher != null; + } } private void refreshPendingSize() { @@ -138,4 +242,14 @@ public final class OutboxMetrics { RuntimeWarnings.meterUnavailable("outbox", "publisher-lag gauge", ex); } } + + private void refreshLongRetries(Instant now) { + try { + OutboxLongRetries longRetries = store.longRetries(LONG_RETRY_DEFERRED_CLAIMS, now); + longRetryRows.set(longRetries.rows()); + longRetryOldestAge.set(longRetries.oldestAgeSeconds()); + } catch (Exception ex) { + RuntimeWarnings.meterUnavailable("outbox", "long-retry gauges", ex); + } + } } diff --git a/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxRelayAutoConfiguration.java b/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxRelayAutoConfiguration.java index 1571a67b..923d6b32 100644 --- a/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxRelayAutoConfiguration.java +++ b/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxRelayAutoConfiguration.java @@ -20,8 +20,8 @@ import org.springframework.core.env.Environment; import org.springframework.scheduling.annotation.EnableScheduling; /** - * The transactional outbox relay: assembles the relay use case manually and registers the leader - * election token, metrics, and scheduler. + * The transactional outbox relay: assembles the relay use case manually and registers its metrics + * and scheduler. * *

    Application composition: it wires application-core ports into a use case and drives it on a * timer, which is why it is a composition module rather than part of the persistence module — a @@ -71,6 +71,9 @@ public class OutboxRelayAutoConfiguration { RandomGenerator outboxRandomGenerator, OutboxSettings properties, OutboxMetrics metrics) { + // A batch that cannot be sent within its lease is sent twice; refused here, where the port that + // knows how long a publish can take meets the settings (review 06-12). + publishPort.longestAttempt().ifPresent(properties::requireLeaseOutlastsBatch); // One lease owner per relay instance. Its claims record it, and a completion applies only while // this owner's lease still holds the row (V17 fencing). PublishPendingOutboxEventsUseCase relayUseCase = @@ -93,22 +96,22 @@ public class OutboxRelayAutoConfiguration { return new SplittableRandom(); } - /** Registered unconditionally so the SKIP LOCKED leadership mechanism is always present. */ - @Bean(name = "outboxLeaderElection") - public OutboxLeaderElectionToken outboxLeaderElection() { - return new OutboxLeaderElectionToken(); - } - /** * Outbox metrics collector; no-op when {@link MeterRegistry} is absent. Present whenever the - * store is, relay or not: its gauges refresh on their own schedule (see {@link OutboxMetrics}). + * store is, relay or not: its gauges refresh every poll interval on a thread of their own (see + * {@link OutboxMetrics}). */ @Bean public OutboxMetrics outboxMetrics( OutboxStorePort store, ObjectProvider meterRegistryProvider, - ObjectProvider clock) { - return new OutboxMetrics(store, meterRegistryProvider, clock.getIfAvailable(Clock::systemUTC)); + ObjectProvider clock, + OutboxSettings settings) { + return new OutboxMetrics( + store, + meterRegistryProvider, + clock.getIfAvailable(Clock::systemUTC), + settings.pollInterval()); } /** diff --git a/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxSettings.java b/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxSettings.java index 5b4ff293..2bbf5e06 100644 --- a/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxSettings.java +++ b/src/composition/outbox-relay/src/main/java/dev/caskeleton/composition/outbox/OutboxSettings.java @@ -62,6 +62,41 @@ public record OutboxSettings( return StrictSettingsBinder.bind(environment, PREFIX, OutboxSettings.class); } + /** + * Refuses a batch whose worst case does not fit in its lease (review 2026-11-01 06-12). + * + *

    A cycle claims {@link #batchSize()} rows under one lease of {@link #inFlightTimeout()} and + * publishes them one after another. If every publish takes its longest, the last one starts + * {@code batchSize × longestAttempt} after the claim; past the lease, another instance has + * already reclaimed the row and sent it, and this one sends it again. The lease fence keeps the + * row's status right, not the broker from seeing the event twice. Each value is fine on its own, + * so only the pair can be checked, and it is checked when the relay is assembled — the publish + * port is what knows how long an attempt can take. + * + * @param longestAttempt the longest one publish can take before its port answers + * @throws dev.caskeleton.adapter.outbound.support.startup.StartupFailureException when the + * batch's worst case reaches the lease + */ + public void requireLeaseOutlastsBatch(Duration longestAttempt) { + Duration worstBatch = longestAttempt.multipliedBy(batchSize); + if (worstBatch.compareTo(inFlightTimeout) >= 0) { + throw StartupFailures.envValidation( + PREFIX + + ".in-flight-timeout (" + + inFlightTimeout + + ") must be longer than " + + PREFIX + + ".batch-size (" + + batchSize + + ") times the longest publish (" + + longestAttempt + + "), " + + worstBatch + + ": a lease that ends before its batch is sent lets another instance send the same" + + " rows again — lower batch-size or raise in-flight-timeout"); + } + } + private static Duration positive(String key, Duration value, Duration fallback) { if (value == null) { return fallback; diff --git a/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxMetricsTest.java b/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxMetricsTest.java index 49109dce..74e65e25 100644 --- a/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxMetricsTest.java +++ b/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxMetricsTest.java @@ -3,10 +3,14 @@ package dev.caskeleton.composition.outbox; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import dev.caskeleton.application.outbox.OutboxLongRetries; import dev.caskeleton.application.outbox.OutboxMessagePublishPort; +import dev.caskeleton.application.outbox.OutboxRelayFailureReportPort; import dev.caskeleton.application.outbox.OutboxStorePort; import dev.caskeleton.application.outbox.testkit.InMemoryOutboxStore; import dev.caskeleton.application.outbox.testkit.OutboxEvents; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.search.Search; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; @@ -14,6 +18,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; +import java.util.List; import java.util.UUID; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -64,6 +69,83 @@ class OutboxMetricsTest { assertThat(pending(registry)).isEqualTo(2.0); } + /** + * A relay tick can run fifty cycles back to back, and against a slow broker that takes minutes — + * exactly when the backlog gauges matter. Refreshed on the scheduler's single fixed-delay thread, + * they waited for the tick to end (re-review 2026-11-03 RR-05-05). + */ + @Test + @DisplayName("relay tick이 발행 중에 멈춰 있어도 게이지는 그동안 쌓인 행을 보여 준다") + void theGaugesRefreshWhileARelayTickIsStuckInAPublish() throws Exception { + // given: a relay whose broker call blocks until the test lets it go + InMemoryOutboxStore store = new InMemoryOutboxStore(); + store.append(OutboxEvents.event(UUID.randomUUID().toString(), Instant.now().minusSeconds(5))); + java.util.concurrent.CountDownLatch publishing = new java.util.concurrent.CountDownLatch(1); + java.util.concurrent.CountDownLatch release = new java.util.concurrent.CountDownLatch(1); + OutboxMessagePublishPort stalled = + event -> { + publishing.countDown(); + try { + release.await(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + }; + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + ApplicationContextRunner runner = + new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(OutboxRelayAutoConfiguration.class)) + .withBean(OutboxStorePort.class, () -> store) + .withBean(MeterRegistry.class, () -> registry) + .withBean(OutboxMessagePublishPort.class, () -> stalled) + .withBean(OutboxRelayFailureReportPort.class, () -> report -> {}) + .withBean(TransactionPort.class, DirectTransactionPort::new) + .withBean(Clock.class, Clock::systemUTC) + .withPropertyValues( + "app.outbox.enabled=true", + "app.outbox.relay-enabled=true", + "app.outbox.poll-interval=PT0.05S"); + + // when: the tick is inside the broker call, and two more rows arrive + runner.run( + context -> { + try { + publishing.await(10, java.util.concurrent.TimeUnit.SECONDS); + store.append(OutboxEvents.event(UUID.randomUUID().toString(), Instant.now())); + store.append(OutboxEvents.event(UUID.randomUUID().toString(), Instant.now())); + await() + .atMost(Duration.ofSeconds(5)) + .until(() -> pending(registry) != null && pending(registry) == 2.0); + } finally { + release.countDown(); + } + }); + + // then + assertThat(pending(registry)).isEqualTo(2.0); + } + + @Test + @DisplayName("오래 재시도 중인 행 게이지는 저장소가 센 행 수와 가장 오래된 행의 나이를 보여 준다") + void theLongRetryGaugesShowTheStoresCountAndOldestAge() { + // given + OutboxStorePort store = Mockito.mock(OutboxStorePort.class); + Mockito.when(store.longRetries(OutboxMetrics.LONG_RETRY_DEFERRED_CLAIMS, NOW)) + .thenReturn(new OutboxLongRetries(3, 7200)); + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + OutboxMetrics metrics = metrics(store, registry); + + // when + metrics.refreshGauges(); + + // then + assertThat( + List.of( + registry.get("outbox.retry.long.size").gauge().value(), + registry.get("outbox.retry.long.age").gauge().value())) + .containsExactly(3.0, 7200.0); + } + @Test @DisplayName("게이지 갱신은 PUBLISHED를 세지 않고 지연 게이지는 가장 오래된 미발행 행의 나이다") void aRefreshCountsUnpublishedRowsAndAgesTheOldest() { @@ -97,6 +179,8 @@ class OutboxMetricsTest { Mockito.when(broken.countByStatus()).thenThrow(new IllegalStateException("database down")); Mockito.when(broken.oldestUnpublishedAgeSecondsByEventType(Mockito.any())) .thenThrow(new IllegalStateException("database down")); + Mockito.when(broken.longRetries(Mockito.anyInt(), Mockito.any())) + .thenThrow(new IllegalStateException("database down")); OutboxMetrics metrics = metrics(broken, new SimpleMeterRegistry()); // when @@ -160,6 +244,9 @@ class OutboxMetricsTest { StaticListableBeanFactory registries = new StaticListableBeanFactory(); registries.addBean("meterRegistry", registry); return new OutboxMetrics( - store, registries.getBeanProvider(MeterRegistry.class), Clock.fixed(NOW, ZoneOffset.UTC)); + store, + registries.getBeanProvider(MeterRegistry.class), + Clock.fixed(NOW, ZoneOffset.UTC), + Duration.ofHours(1)); } } diff --git a/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxRelayAutoConfigurationTest.java b/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxRelayAutoConfigurationTest.java index 9ad94203..ca82b543 100644 --- a/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxRelayAutoConfigurationTest.java +++ b/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxRelayAutoConfigurationTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.composition.outbox; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import dev.caskeleton.application.outbox.OutboxMessagePublishPort; import dev.caskeleton.application.outbox.OutboxRelayFailureReportPort; @@ -9,7 +10,10 @@ import dev.caskeleton.application.outbox.OutboxStorePort; import dev.caskeleton.application.transaction.TransactionPort; import java.lang.reflect.Proxy; import java.time.Clock; +import java.time.Duration; +import java.util.Optional; import java.util.random.RandomGenerator; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.AutoConfigurations; @@ -84,6 +88,34 @@ class OutboxRelayAutoConfigurationTest { .run(context -> assertThat(context).hasSingleBean(OutboxRelayScheduler.class)); } + @Test + @DisplayName("발행 포트가 말한 최악 발행 시간으로 한 배치가 lease를 넘으면 relay 조립이 기동을 거부한다") + void aBatchThePublishPortCannotSendWithinTheLeaseFailsStartup() { + // given: a port whose publish can take ten seconds, forty rows a batch, a five-minute lease + OutboxMessagePublishPort tenSecondPublisher = mock(OutboxMessagePublishPort.class); + when(tenSecondPublisher.longestAttempt()).thenReturn(Optional.of(Duration.ofSeconds(10))); + ApplicationContextRunner runner = + new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(OutboxRelayAutoConfiguration.class)) + .withBean(OutboxStorePort.class, OutboxRelayAutoConfigurationTest::unusedStore) + .withBean(OutboxRelayFailureReportPort.class, () -> report -> {}) + .withBean(TransactionPort.class, () -> mock(TransactionPort.class)) + .withBean(Clock.class, Clock::systemUTC) + .withBean(OutboxMessagePublishPort.class, () -> tenSecondPublisher) + .withPropertyValues( + "app.outbox.enabled=true", + "app.outbox.relay-enabled=true", + "app.outbox.batch-size=40", + "app.outbox.in-flight-timeout=PT5M"); + + // when & then + runner.run( + context -> + assertThat(context) + .getFailure() + .hasStackTraceContaining("app.outbox.in-flight-timeout")); + } + @Test void withoutTheRelayKeyTheRelaySchedulerIsNotAssembled() { new ApplicationContextRunner() diff --git a/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxRelaySchedulerDrainTest.java b/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxRelaySchedulerDrainTest.java index 7e77915e..20094488 100644 --- a/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxRelaySchedulerDrainTest.java +++ b/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxRelaySchedulerDrainTest.java @@ -156,7 +156,8 @@ class OutboxRelaySchedulerDrainTest { new OutboxMetrics( store, registries.getBeanProvider(io.micrometer.core.instrument.MeterRegistry.class), - clock)); + clock, + Duration.ofHours(1))); } /** The fake store with the claim SQL's per-aggregate head gate. */ diff --git a/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxRelayShutdownOrderTest.java b/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxRelayShutdownOrderTest.java index 1d038f0f..2be9653f 100644 --- a/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxRelayShutdownOrderTest.java +++ b/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxRelayShutdownOrderTest.java @@ -166,15 +166,16 @@ class OutboxRelayShutdownOrderTest { } @Override - public synchronized OutboxCompletion markFailed(OutboxLease lease, Instant nextAttemptAt) { - OutboxCompletion completion = super.markFailed(lease, nextAttemptAt); + public synchronized OutboxCompletion markFailed( + OutboxLease lease, Instant nextAttemptAt, String failureCode) { + OutboxCompletion completion = super.markFailed(lease, nextAttemptAt, failureCode); completed.set(true); return completion; } @Override - public synchronized OutboxCompletion markDead(OutboxLease lease) { - OutboxCompletion completion = super.markDead(lease); + public synchronized OutboxCompletion markDead(OutboxLease lease, String failureCode) { + OutboxCompletion completion = super.markDead(lease, failureCode); completed.set(true); return completion; } diff --git a/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxSettingsTest.java b/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxSettingsTest.java index 2a90ed07..371f48b1 100644 --- a/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxSettingsTest.java +++ b/src/composition/outbox-relay/src/test/java/dev/caskeleton/composition/outbox/OutboxSettingsTest.java @@ -6,6 +6,7 @@ import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.support.startup.StartupErrorCode; import dev.caskeleton.adapter.outbound.support.startup.StartupFailureException; import java.time.Duration; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -99,4 +100,48 @@ class OutboxSettingsTest { e -> assertThat(e.errorCode()).isEqualTo(StartupErrorCode.STARTUP_VALIDATION_FAILED)) .hasMessageContaining("app.outbox." + message); } + + /** + * A cycle holds every row of its batch under one lease while it publishes them one after another, + * so the batch's worst case has to fit in the lease. Past it, another instance reclaims rows this + * one is still going to send, and the event goes out twice (review 2026-11-01 06-12). + */ + @ParameterizedTest(name = "batch {0} x {1}s against a {2}s lease") + @CsvSource({"40, 10, 300", "30, 10, 300", "20, 10, 120"}) + @DisplayName("한 배치를 다 보내는 최악 시간이 lease 이상인 설정은 기동을 거부한다") + void aBatchWhoseWorstCaseReachesTheLeaseIsRefusedAtStartup( + int batchSize, long longestAttemptSeconds, long leaseSeconds) { + // given + OutboxSettings settings = + new OutboxSettings( + true, true, null, batchSize, Duration.ofSeconds(leaseSeconds), null, null); + + // when + Throwable thrown = + catchThrowable( + () -> settings.requireLeaseOutlastsBatch(Duration.ofSeconds(longestAttemptSeconds))); + + // then + assertThat(thrown) + .isInstanceOfSatisfying( + StartupFailureException.class, + failure -> + assertThat(failure.errorCode()) + .isEqualTo(StartupErrorCode.STARTUP_VALIDATION_FAILED)) + .hasMessageContaining("app.outbox.in-flight-timeout"); + } + + @Test + @DisplayName("기본값(배치 20, 발행 최악 10초, lease 5분)은 기동을 막지 않는다") + void theDefaultsLeaveTheLeaseLongerThanABatch() { + // given + OutboxSettings settings = new OutboxSettings(true, true, null, null, null, null, null); + + // when + Throwable thrown = + catchThrowable(() -> settings.requireLeaseOutlastsBatch(Duration.ofSeconds(10))); + + // then + assertThat(thrown).isNull(); + } } diff --git a/src/composition/service-runtime/README.md b/src/composition/service-runtime/README.md index e2153a24..46112736 100644 --- a/src/composition/service-runtime/README.md +++ b/src/composition/service-runtime/README.md @@ -28,7 +28,9 @@ composition:service-runtime 선언만 남은 채 정상 기동했다. 이제 `PermissionEnforcementGuard`가 선언 빈마다 pointcut이 맞는 advisor가 있는지 보고, 없으면 `REQUIRED_ADAPTER_DISABLED`(exit 72)로 기동을 멈춘다. 한계: "어떤 advisor든 pointcut이 맞으면 통과"라 인가가 아닌 advisor를 구분하지 못한다. `new`로 만든 - 유스케이스는 검사 대상이 아니다(시스템 호출용은 `@SystemInvoked`). + 유스케이스는 검사 대상이 아니다(시스템 호출용은 `@SystemInvoked`). 반대로 `@SystemInvoked` + 유스케이스가 빈으로 등록돼 있으면 같은 가드가 기동을 멈춘다 — 권한 검사가 없는 쓰기를 전송 계층이 + 닿는 곳에 두는 조립이기 때문이다(review 2026-11-03 RR-01-02). - 모두 이 모듈의 `AutoConfiguration.imports`로 등록된다(설정 브리지와 tracing 스위치는 `spring.factories`의 `EnvironmentPostProcessor`·`AutoConfigurationImportFilter`). 구성 루트의 diff --git a/src/composition/service-runtime/src/main/java/dev/caskeleton/composition/runtime/authorization/PermissionEnforcementGuard.java b/src/composition/service-runtime/src/main/java/dev/caskeleton/composition/runtime/authorization/PermissionEnforcementGuard.java index 9f9b0e80..c671f8b8 100644 --- a/src/composition/service-runtime/src/main/java/dev/caskeleton/composition/runtime/authorization/PermissionEnforcementGuard.java +++ b/src/composition/service-runtime/src/main/java/dev/caskeleton/composition/runtime/authorization/PermissionEnforcementGuard.java @@ -2,6 +2,7 @@ package dev.caskeleton.composition.runtime.authorization; import dev.caskeleton.adapter.outbound.support.startup.StartupFailures; import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.security.SystemInvoked; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -17,16 +18,21 @@ import org.springframework.core.annotation.AnnotatedElementUtils; import org.springframework.util.ClassUtils; /** - * Fails startup when a bean declares {@link RequiresPermission} and no advisor in the context - * applies to it. + * Fails startup when a write use case would run with no permission check: a bean that declares + * {@link RequiresPermission} with no advisor in the context that applies to it, or a bean of a + * {@link SystemInvoked} use case at all. * *

    "Applies" is {@link AopUtils#canApply(Advisor, Class)}: the advisor's pointcut matches the * bean's class. That is how the web adapter's advisor reaches a use case, so its absence is exactly * the case this guard is for. The check cannot tell an authorization advisor from any other advisor * whose pointcut happens to match; it closes the "adapter missing" gap, not every misconfiguration. * - *

    Only beans are checked. A use case built with {@code new} is never advised whatever the - * context holds — that is what {@code @SystemInvoked} declares for the use cases the system calls. + *

    A {@code @SystemInvoked} use case carries no permission by design — only the system calls it, + * on a thread with no {@code Authentication} — and its composition constructs it with {@code new} + * and hands it to that caller. Registered as a bean, it is a write any transport can reach with no + * check, whatever advisor the context holds, so its presence fails startup (review 2026-11-03 + * RR-01-02). {@code INBOUND_ADAPTERS_DO_NOT_REACH_SYSTEM_INVOKED_USE_CASES} closes the other path, + * an inbound adapter that constructs one itself. */ final class PermissionEnforcementGuard implements SmartInitializingSingleton, BeanFactoryAware { @@ -40,6 +46,7 @@ final class PermissionEnforcementGuard implements SmartInitializingSingleton, Be @Override public void afterSingletonsInstantiated() { Collection advisors = beanFactory.getBeansOfType(Advisor.class, true, false).values(); + List systemInvoked = new ArrayList<>(); List unenforced = new ArrayList<>(); for (String name : beanFactory.getBeanDefinitionNames()) { Class type = beanFactory.getType(name, false); @@ -47,11 +54,20 @@ final class PermissionEnforcementGuard implements SmartInitializingSingleton, Be continue; } Class userClass = ClassUtils.getUserClass(type); - if (declaresPermission(userClass) + if (AnnotatedElementUtils.hasAnnotation(userClass, SystemInvoked.class)) { + systemInvoked.add(name + " (" + userClass.getName() + ")"); + } else if (declaresPermission(userClass) && advisors.stream().noneMatch(advisor -> AopUtils.canApply(advisor, userClass))) { unenforced.add(name + " (" + userClass.getName() + ")"); } } + if (!systemInvoked.isEmpty()) { + throw StartupFailures.requiredAdapterDisabled( + "@SystemInvoked use cases are registered as beans, where a transport can reach a write" + + " that has no permission check — a system-invoked use case must be constructed by" + + " its composition, not registered as a bean: " + + String.join(", ", systemInvoked)); + } if (!unenforced.isEmpty()) { throw StartupFailures.requiredAdapterDisabled( "beans declare @RequiresPermission but no advisor enforces it — the composition is" diff --git a/src/composition/service-runtime/src/test/java/dev/caskeleton/composition/runtime/authorization/PermissionEnforcementGuardTest.java b/src/composition/service-runtime/src/test/java/dev/caskeleton/composition/runtime/authorization/PermissionEnforcementGuardTest.java index 86ea2242..e8c1acf8 100644 --- a/src/composition/service-runtime/src/test/java/dev/caskeleton/composition/runtime/authorization/PermissionEnforcementGuardTest.java +++ b/src/composition/service-runtime/src/test/java/dev/caskeleton/composition/runtime/authorization/PermissionEnforcementGuardTest.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.adapter.outbound.support.startup.StartupFailureException; import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.security.SystemInvoked; import org.aopalliance.intercept.MethodInterceptor; import org.junit.jupiter.api.Test; import org.springframework.aop.Advisor; @@ -39,6 +40,22 @@ class PermissionEnforcementGuardTest { .run(context -> assertThat(context).hasNotFailed().hasSingleBean(GatedUseCase.class)); } + /** + * A system-invoked use case carries no permission, so a bean of it is a write a transport can + * reach with no check at all — whatever advisor the context holds (review 2026-11-03 RR-01-02). + */ + @Test + void aSystemInvokedUseCaseRegisteredAsABeanFailsStartupEvenWithAnAdvisor() { + runner + .withUserConfiguration(SystemInvokedUseCaseConfig.class, EnforcingAdvisorConfig.class) + .run( + context -> + assertThat(context.getStartupFailure()) + .isInstanceOf(StartupFailureException.class) + .hasMessageContaining("must be constructed by its composition") + .hasMessageContaining(SystemInvokedUseCase.class.getName())); + } + @Test void aContextWithoutGatedBeansNeedsNoAdvisor() { runner.run(context -> assertThat(context).hasNotFailed()); @@ -57,6 +74,19 @@ class PermissionEnforcementGuardTest { } } + @SystemInvoked(reason = "a scheduler calls it") + public static class SystemInvokedUseCase { + public void handle() {} + } + + @Configuration(proxyBeanMethods = false) + static class SystemInvokedUseCaseConfig { + @Bean + SystemInvokedUseCase systemInvokedUseCase() { + return new SystemInvokedUseCase(); + } + } + @Configuration(proxyBeanMethods = false) static class EnforcingAdvisorConfig { @Bean diff --git a/src/config/runtime/compose-profile-contracts.json b/src/config/runtime/compose-profile-contracts.json index d0bef525..a5a000ab 100644 --- a/src/config/runtime/compose-profile-contracts.json +++ b/src/config/runtime/compose-profile-contracts.json @@ -795,6 +795,6 @@ "redis" ], "$vendorComment": "expectedPersistenceVendor is the vendor the application must report, not the one the lane started: info.capabilities.persistence-jpa.vendor while JPA is on, none while it is off. A reported postgresql is the database the queries go to, because the JPA module refuses to start when the selected vendor is postgresql and the datasource it reaches is not PostgreSQL (JpaDataSourceProfileValidator). local-jpa passed for a while against H2 while a PostgreSQL container it never touched sat beside it, because application-local.yml pins an in-memory URL that outranks the lane's environment.", - "$redisComment": "The lanes that switch Redis on authenticate the way a deployment does. Their Redis loads infra/redis-sdk/acl/all-accounts.acl, which turns the passwordless default user off, and the application connects as the two named accounts that file provisions: ca-skeleton-application for ordinary commands and ca-skeleton-application-advanced for registered scripts. Both references use the secret://@environment/ form; without the account part the client authenticates as the disabled default user. The two REDIS_LANE_* values are that file's fixture passwords (infra/redis-sdk/acl/README.md), not deployment credentials, and deliberately carry no APP_ prefix: they are not application settings, only what these references point at. APP_REDIS_NAMESPACE_ENVIRONMENT is prod because the fixture accounts are fenced to ~prod:*, the namespace every other test against that file uses; under the default local namespace the first key a capability wrote would be refused with NOPERM after startup had reported healthy. Before this the lanes declared anonymous-access-accepted against an open Redis, and the two startup warnings that trade prints (no credential, no advanced account) failed their startup-is-silent check — which is right, because in production both are risks. scripts/verify-compose-profile-contracts.sh check 7 holds a Redis lane's references, passwords and namespace to the account lines of the file its Redis actually mounts.", + "$redisComment": "The lanes that switch Redis on authenticate the way a deployment does. Their Redis loads infra/redis-sdk/acl/lane-fixture-accounts.acl, which turns the passwordless default user off, and the application connects as the two named accounts that file provisions: ca-skeleton-application for ordinary commands and ca-skeleton-application-advanced for registered scripts. Both references use the secret://@environment/ form; without the account part the client authenticates as the disabled default user. The two REDIS_LANE_* values are that file's fixture passwords (infra/redis-sdk/acl/README.md), not deployment credentials, and deliberately carry no APP_ prefix: they are not application settings, only what these references point at. APP_REDIS_NAMESPACE_ENVIRONMENT is prod because the fixture accounts are fenced to ~prod:*, the namespace every other test against that file uses; under the default local namespace the first key a capability wrote would be refused with NOPERM after startup had reported healthy. Before this the lanes declared anonymous-access-accepted against an open Redis, and the two startup warnings that trade prints (no credential, no advanced account) failed their startup-is-silent check — which is right, because in production both are risks. scripts/verify-compose-profile-contracts.sh check 7 holds a Redis lane's references, passwords and namespace to the account lines of the file its Redis actually mounts.", "$flywayComment": "SPRING_FLYWAY_LOCATIONS is deliberately absent. Seven lanes listed every capability stream there, and it could never have worked: the db/migration/jpa/* streams each declare a V1, so one Flyway instance over all of them fails with 'Found more than one migration with version 1'. Each stream is its own stream with its own history table by design. The setting was inert until Wave 2 stopped PostgreSqlPersistenceConfig from overriding spring.flyway.locations, and the lanes passed while their own configuration was being discarded. Applying a capability stream needs a Flyway execution per stream — the operator sequence PostgreSqlNotificationPersistenceUnitIntegrationTest and FileserverRoundTripContractTest both perform in their fixtures — and that is what the notification lanes still need." } diff --git a/src/config/spotbugs/exclude.xml b/src/config/spotbugs/exclude.xml index 7276eb5d..2a7d1e5e 100644 --- a/src/config/spotbugs/exclude.xml +++ b/src/config/spotbugs/exclude.xml @@ -16,14 +16,37 @@ - + - + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/grpc/CLAUDE.md b/src/grpc/CLAUDE.md index 19e78b30..5705281a 100644 --- a/src/grpc/CLAUDE.md +++ b/src/grpc/CLAUDE.md @@ -145,7 +145,7 @@ the repository's `.seconds` / `.total` unit naming and the corresponding validat `grpc-spring-boot-starter` does not pull the JPA ledger adapter transitively (11-M9). A deployment that needs durable operation idempotency adds `:grpc:grpc-operation-ledger-jpa` explicitly, defines a `GrpcOperationLedger` bean (`new JpaGrpcOperationLedger(repository)`), includes that adapter's Flyway -location `db/migration/grpc` (V001 table, V002 retention index), and sets the flag below. A deployment +location `db/migration/grpc` (V001 table, V002 retention index, V003 claim attempt), and sets the flag below. A deployment that does not adopt the ledger pays no Spring Data JPA classpath cost. | Key (`app.grpc.platform.*`) | Default | What it does | @@ -165,9 +165,9 @@ optional build is not part of the shipped runtime. They join the registry when a adopts the platform. **Why the starter assembles the interceptor (03-11 (b)), and not (a).** The review proposed (a): -delete the flag and judge by whether a ledger bean exists. The owner chose (b): the flag stays and -means what it says — when true, the decision function is assembled and its TTL comes from -configuration. (a) would have removed the false promise but left every adopter to rebuild the same +delete the flag and judge by whether a ledger bean exists. (b) was chosen and is recorded in +ADR-GRPC-004 (re-review 2026-11-03, RR-02-07): the flag stays and means what it says — when true, the +decision function is assembled and its TTL comes from configuration. (a) would have removed the false promise but left every adopter to rebuild the same assembly and TTL validation by hand; (b) keeps that in one place. The interceptor bean has no `@ConditionalOnMissingBean` on purpose: a replacement would silently ignore `operation-ledger-claim-ttl`. Duplicates of a running operation are refused (`REJECT_IN_PROGRESS`), not told to wait; a @@ -179,7 +179,10 @@ server, so assembling the decision does not wire the platform into `adapter:inbo The adopter injects the bean by type and calls it from its own `IDEMPOTENCY` stage interceptor (`GrpcServerInterceptorChain.builder().stage(GrpcServerInterceptorStage.IDEMPOTENCY, ...)`) or from its typed service adapter: `decide(...)` before the mutation, `recordCommit(...)` inside the mutation's -transaction, `recordFailure(...)` on a terminal failure. +transaction, `recordFailure(...)` on a terminal failure. Both record calls take the `GrpcClaimToken` +the `PROCEED` decision carried: a commit by an attempt whose claim was taken over is refused (its +transaction rolls back) and its failure is ignored, so a slow owner cannot write over the attempt that +took its claim (re-review RR-02-04, ADR-GRPC-004). **Purge scheduling is the adopter's.** `GrpcOperationLedgerRetention.purgeExpired(now)` deletes expired finished rows in bounded batches (500 per statement) until a batch comes back short. The diff --git a/src/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcClaimToken.java b/src/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcClaimToken.java new file mode 100644 index 00000000..7b5fe358 --- /dev/null +++ b/src/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcClaimToken.java @@ -0,0 +1,35 @@ +package dev.caskeleton.grpc.api.ledger; + +/** + * Which attempt holds a claim: the fencing token a commit or a failure record must present. + * + *

    A claim can change hands. A stale one is taken over after the claim TTL, and a failed one is + * re-claimed by the next retry; both make a new attempt of the same operation. The owner that lost + * the claim may still be running, and when it finishes it records its outcome. Without a token that + * record lands on the new attempt's claim: an old owner's failure turned the new attempt's claim + * {@code FAILED_TERMINAL}, a third retry re-claimed it, and two attempts ran the mutation at once + * (re-review 2026-11-03, RR-02-04). With it, the ledger records an outcome only for the attempt + * that still holds the claim. + * + *

    A counter rather than the claim moment: it increases by one with every hand-over, so two + * attempts never share a value, whatever the clocks of the processes that claimed it. + * + * @param attempt one for the first claim of an operation, one more for each hand-over + */ +public record GrpcClaimToken(long attempt) { + + /** The token of the claim that created the record. */ + public static final GrpcClaimToken FIRST = new GrpcClaimToken(1); + + /** Requires a positive attempt number. */ + public GrpcClaimToken { + if (attempt < 1) { + throw new IllegalArgumentException("a claim attempt is numbered from one, was " + attempt); + } + } + + /** The token of the attempt that takes this claim over. */ + public GrpcClaimToken next() { + return new GrpcClaimToken(Math.addExact(attempt, 1)); + } +} diff --git a/src/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java b/src/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java index 2c275151..381979a7 100644 --- a/src/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java +++ b/src/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java @@ -20,7 +20,8 @@ public interface GrpcOperationLedger { * implementation admits both concurrent attempts, which is the exact failure the ledger exists to * prevent, and it does so only under the load where it matters. * - * @return the existing record when one was already claimed, empty when this call created it + * @return the existing record when one was already claimed, empty when this call created it — the + * caller then holds the claim as {@link GrpcClaimToken#FIRST} */ Optional claim( GrpcOperationIdentity identity, String requestFingerprint, Instant now); @@ -34,11 +35,34 @@ public interface GrpcOperationLedger { *

    Called inside the business transaction wherever the datastore allows it. When the ledger and * the mutation commit separately, there is a window in which the mutation is durable and the * claim is not, and a retry in that window runs the mutation twice. + * + *

    Only the attempt that holds the claim commits it. An attempt whose claim was taken over or + * re-claimed since is refused with an exception, so its business transaction rolls back rather + * than committing beside the attempt that now owns the operation. Of two concurrent commits of + * one claim, exactly one succeeds. + * + * @param claimToken the token of the attempt committing, from the claim it won + * @throws IllegalStateException when there is no claim, it already finished, or it is no longer + * held by {@code claimToken} */ - void markCommitted(GrpcOperationIdentity identity, String outcomeReference, Instant now); + void markCommitted( + GrpcOperationIdentity identity, + GrpcClaimToken claimToken, + String outcomeReference, + Instant now); - /** Marks a claim terminally failed. */ - void markFailed(GrpcOperationIdentity identity, Instant now); + /** + * Marks a claim terminally failed, if {@code claimToken} still holds it. + * + *

    An attempt that lost its claim reports a failure the operation no longer has: another + * attempt is running it. That record is ignored rather than written over the running attempt's + * claim. + * + * @param claimToken the token of the attempt that failed, from the claim it won + * @return true when the failure was recorded; false when the claim is missing, already finished, + * or held by another attempt, in which case nothing changed + */ + boolean markFailed(GrpcOperationIdentity identity, GrpcClaimToken claimToken, Instant now); /** * Re-claims a {@code FAILED_TERMINAL} record for a new attempt of the same request. @@ -48,9 +72,11 @@ public interface GrpcOperationLedger { * request, is left untouched. A retry that proceeds without winning this leaves its commit with * no owner and runs concurrently with every other retry of the same key. * - * @return true when this call now owns the record as a fresh attempt claimed at {@code now} + * @return the new attempt's token when this call now owns the record as a fresh attempt claimed + * at {@code now}; empty otherwise */ - boolean reclaimFailed(GrpcOperationIdentity identity, String requestFingerprint, Instant now); + Optional reclaimFailed( + GrpcOperationIdentity identity, String requestFingerprint, Instant now); /** * Takes over an {@code IN_PROGRESS} claim whose owner is presumed dead, for a new attempt of the @@ -73,10 +99,15 @@ public interface GrpcOperationLedger { * When an adopter commits them separately, the abandoned attempt's outcome is unknown and a * takeover can run the mutation twice. * - * @return true when this call now owns the record as a fresh attempt claimed at {@code now} + *

    The owner taken over may be slow rather than dead. Its later commit is refused and its later + * failure ignored, because it no longer holds the claim token; that is what keeps the takeover + * from becoming a second concurrent run. + * + * @return the new attempt's token when this call now owns the record as a fresh attempt claimed + * at {@code now}; empty otherwise * @throws IllegalArgumentException when {@code claimedBefore} is after {@code now} */ - boolean reclaimStale( + Optional reclaimStale( GrpcOperationIdentity identity, String requestFingerprint, Instant claimedBefore, diff --git a/src/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerRecord.java b/src/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerRecord.java index 2bc0942a..22f86322 100644 --- a/src/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerRecord.java +++ b/src/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerRecord.java @@ -10,6 +10,9 @@ import java.util.Optional; * A caller that reuses a key for a different request has made a mistake, and returning the first * request's answer would hide it; comparing fingerprints turns that into a {@code * FAILED_PRECONDITION} the caller can act on. + * + *

    {@code claimToken} names the attempt that holds (or last held) the claim; see {@link + * GrpcClaimToken}. */ public record GrpcOperationLedgerRecord( GrpcOperationIdentity identity, @@ -17,9 +20,10 @@ public record GrpcOperationLedgerRecord( String requestFingerprint, Optional outcomeReference, Instant claimedAt, - Optional completedAt) { + Optional completedAt, + GrpcClaimToken claimToken) { - /** Requires an identity, a state, a fingerprint and a claim moment. */ + /** Requires an identity, a state, a fingerprint, a claim moment and a claim token. */ public GrpcOperationLedgerRecord { if (identity == null || state == null) { throw new IllegalArgumentException("a ledger record needs an identity and a state"); @@ -30,6 +34,9 @@ public record GrpcOperationLedgerRecord( if (claimedAt == null || outcomeReference == null || completedAt == null) { throw new IllegalArgumentException("a ledger record needs its timestamps and Optionals"); } + if (claimToken == null) { + throw new IllegalArgumentException("a ledger record names the attempt that holds it"); + } if (state.terminal() && completedAt.isEmpty()) { throw new IllegalArgumentException("a terminal ledger record records when it finished"); } @@ -40,16 +47,26 @@ public record GrpcOperationLedgerRecord( } } - /** A fresh claim. */ + /** A fresh claim, held by the first attempt. */ public static GrpcOperationLedgerRecord claim( GrpcOperationIdentity identity, String requestFingerprint, Instant claimedAt) { + return claim(identity, requestFingerprint, claimedAt, GrpcClaimToken.FIRST); + } + + /** A claim in progress, held by the attempt {@code claimToken} names. */ + public static GrpcOperationLedgerRecord claim( + GrpcOperationIdentity identity, + String requestFingerprint, + Instant claimedAt, + GrpcClaimToken claimToken) { return new GrpcOperationLedgerRecord( identity, GrpcOperationLedgerState.IN_PROGRESS, requestFingerprint, Optional.empty(), claimedAt, - Optional.empty()); + Optional.empty(), + claimToken); } /** Whether {@code candidateFingerprint} is the same request this claim was made for. */ diff --git a/src/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java b/src/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java index 0c625fc6..d91f8de1 100644 --- a/src/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java +++ b/src/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java @@ -88,7 +88,7 @@ public abstract class GrpcOperationLedgerContract { ledger().claim(ID, FINGERPRINT, NOW); // when - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); // then assertThat(ledger().find(ID)) @@ -106,7 +106,8 @@ public abstract class GrpcOperationLedgerContract { @Test void committingWithoutAClaimIsRefused() { // when - Throwable refused = catchThrowable(() -> ledger().markCommitted(ID, OUTCOME, NOW)); + Throwable refused = + catchThrowable(() -> ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW)); // then assertThat(refused) @@ -118,12 +119,15 @@ public abstract class GrpcOperationLedgerContract { void aCommittedClaimCannotBeCommittedAgain() { // given ledger().claim(ID, FINGERPRINT, NOW); - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); // when Throwable refused = catchThrowable( - () -> ledger().markCommitted(ID, "outcome://document/99", NOW.plusSeconds(2))); + () -> + ledger() + .markCommitted( + ID, GrpcClaimToken.FIRST, "outcome://document/99", NOW.plusSeconds(2))); // then assertThat(refused) @@ -137,7 +141,7 @@ public abstract class GrpcOperationLedgerContract { ledger().claim(ID, FINGERPRINT, NOW); // when - ledger().markFailed(ID, NOW.plusSeconds(1)); + ledger().markFailed(ID, GrpcClaimToken.FIRST, NOW.plusSeconds(1)); // then assertThat(ledger().find(ID)) @@ -153,7 +157,8 @@ public abstract class GrpcOperationLedgerContract { // when Throwable refused = - catchThrowable(() -> ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(2))); + catchThrowable( + () -> ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(2))); // then assertThat(refused) @@ -167,10 +172,11 @@ public abstract class GrpcOperationLedgerContract { failedClaim(); // when - boolean reclaimed = ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + Optional reclaimed = + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); // then - assertThat(reclaimed).isTrue(); + assertThat(reclaimed).isPresent(); } @Test @@ -196,10 +202,11 @@ public abstract class GrpcOperationLedgerContract { void aReclaimedClaimCanBeCommitted() { // given failedClaim(); - ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + GrpcClaimToken reclaimed = + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)).orElseThrow(); // when - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(3)); + ledger().markCommitted(ID, reclaimed, OUTCOME, NOW.plusSeconds(3)); // then assertThat(ledger().find(ID)) @@ -212,27 +219,28 @@ public abstract class GrpcOperationLedgerContract { void aFailedClaimIsNotReclaimedForADifferentRequest() { failedClaim(); - assertThat(ledger().reclaimFailed(ID, OTHER_FINGERPRINT, NOW.plusSeconds(2))).isFalse(); + assertThat(ledger().reclaimFailed(ID, OTHER_FINGERPRINT, NOW.plusSeconds(2))).isEmpty(); } @Test void anInProgressClaimIsNotReclaimed() { ledger().claim(ID, FINGERPRINT, NOW); - assertThat(ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(1))).isFalse(); + assertThat(ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(1))).isEmpty(); } @Test void aCommittedClaimIsNotReclaimed() { // given ledger().claim(ID, FINGERPRINT, NOW); - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); // when - boolean reclaimed = ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + Optional reclaimed = + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); // then - assertThat(reclaimed).isFalse(); + assertThat(reclaimed).isEmpty(); } @Test @@ -242,7 +250,7 @@ public abstract class GrpcOperationLedgerContract { // when List reclaimed = - concurrently(() -> ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2))); + concurrently(() -> ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)).isPresent()); // then assertThat(reclaimed).containsExactlyInAnyOrder(true, false); @@ -255,10 +263,10 @@ public abstract class GrpcOperationLedgerContract { ledger().claim(ID, FINGERPRINT, NOW); // when - boolean reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + Optional reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); // then - assertThat(reclaimed).isTrue(); + assertThat(reclaimed).isPresent(); } @Test @@ -285,10 +293,10 @@ public abstract class GrpcOperationLedgerContract { void aTakenOverStaleClaimCanBeCommitted() { // given ledger().claim(ID, FINGERPRINT, NOW); - ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + GrpcClaimToken takenOver = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER).orElseThrow(); // when - ledger().markCommitted(ID, OUTCOME, LATER.plusSeconds(1)); + ledger().markCommitted(ID, takenOver, OUTCOME, LATER.plusSeconds(1)); // then assertThat(ledger().find(ID)) @@ -304,10 +312,11 @@ public abstract class GrpcOperationLedgerContract { ledger().claim(ID, FINGERPRINT, NOW); // when - boolean reclaimed = ledger().reclaimStale(ID, FINGERPRINT, NOW, NOW.plus(CLAIM_TTL)); + Optional reclaimed = + ledger().reclaimStale(ID, FINGERPRINT, NOW, NOW.plus(CLAIM_TTL)); // then - assertThat(reclaimed).isFalse(); + assertThat(reclaimed).isEmpty(); } @Test @@ -317,10 +326,11 @@ public abstract class GrpcOperationLedgerContract { ledger().claim(ID, FINGERPRINT, NOW); // when - boolean reclaimed = ledger().reclaimStale(ID, OTHER_FINGERPRINT, CUTOFF, LATER); + Optional reclaimed = + ledger().reclaimStale(ID, OTHER_FINGERPRINT, CUTOFF, LATER); // then - assertThat(reclaimed).isFalse(); + assertThat(reclaimed).isEmpty(); } @Test @@ -328,13 +338,13 @@ public abstract class GrpcOperationLedgerContract { void aCommittedClaimIsNotTakenOverAsStale() { // given ledger().claim(ID, FINGERPRINT, NOW); - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); // when - boolean reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + Optional reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); // then - assertThat(reclaimed).isFalse(); + assertThat(reclaimed).isEmpty(); } @Test @@ -344,20 +354,20 @@ public abstract class GrpcOperationLedgerContract { failedClaim(); // when - boolean reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + Optional reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); // then - assertThat(reclaimed).isFalse(); + assertThat(reclaimed).isEmpty(); } @Test @DisplayName("claim이 없으면 stale 회수는 아무것도 만들지 않는다") void takingOverAMissingClaimCreatesNothing() { // when - boolean reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + Optional reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); // then - assertThat(reclaimed).isFalse(); + assertThat(reclaimed).isEmpty(); assertThat(ledger().find(ID)).isEmpty(); } @@ -369,10 +379,10 @@ public abstract class GrpcOperationLedgerContract { ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); // when - boolean again = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + Optional again = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); // then - assertThat(again).isFalse(); + assertThat(again).isEmpty(); } @Test @@ -383,7 +393,7 @@ public abstract class GrpcOperationLedgerContract { // when List reclaimed = - concurrently(() -> ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER)); + concurrently(() -> ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER).isPresent()); // then assertThat(reclaimed).containsExactlyInAnyOrder(true, false); @@ -402,6 +412,126 @@ public abstract class GrpcOperationLedgerContract { assertThat(refused).isInstanceOf(IllegalArgumentException.class).hasMessageContaining("cutoff"); } + @Test + @DisplayName("claim 을 넘겨받을 때마다 claim token 은 다음 시도 번호가 된다") + void eachHandOverIssuesTheNextClaimToken() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + GrpcClaimToken takenOver = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER).orElseThrow(); + + // then + assertThat(takenOver).isEqualTo(GrpcClaimToken.FIRST.next()); + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::claimToken) + .isEqualTo(takenOver); + } + + /** + * The re-review probe's sequence (2026-11-03, RR-02-04): A claims and is slow, B takes the claim + * over after the TTL, then A reports a failure. Recording it would mark B's running attempt + * failed, and the next retry would re-claim it and run beside B. + */ + @Test + @DisplayName("TTL 뒤 넘겨받은 claim 에 옛 owner 가 실패를 기록하면 무시되고 새 시도는 진행 중으로 남는다") + void aFailureReportedByTheOwnerThatLostTheClaimIsIgnored() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // when + boolean recorded = ledger().markFailed(ID, GrpcClaimToken.FIRST, LATER.plusSeconds(1)); + + // then + assertThat(recorded).isFalse(); + assertThat(ledger().find(ID)) + .get() + .extracting( + GrpcOperationLedgerRecord::state, + GrpcOperationLedgerRecord::claimedAt, + GrpcOperationLedgerRecord::completedAt) + .containsExactly(GrpcOperationLedgerState.IN_PROGRESS, LATER, Optional.empty()); + } + + @Test + @DisplayName("옛 owner 의 실패가 무시되므로 다음 재시도는 진행 중인 새 시도를 다시 claim 하지 못한다") + void aRetryCannotReclaimTheRunningAttemptAfterTheLostOwnerFailed() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + ledger().markFailed(ID, GrpcClaimToken.FIRST, LATER.plusSeconds(1)); + + // when + Optional third = ledger().reclaimFailed(ID, FINGERPRINT, LATER.plusSeconds(2)); + + // then + assertThat(third).isEmpty(); + } + + @Test + @DisplayName("TTL 뒤 넘겨받은 claim 에 옛 owner 가 commit 하면 거부되고 새 시도는 진행 중으로 남는다") + void aCommitByTheOwnerThatLostTheClaimIsRefused() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // when + Throwable refused = + catchThrowable( + () -> ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, LATER.plusSeconds(1))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("no longer held by attempt 1"); + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::state) + .isEqualTo(GrpcOperationLedgerState.IN_PROGRESS); + } + + @Test + @DisplayName("실패 뒤 다시 claim 된 기록에 이전 시도가 commit 하면 거부된다") + void aCommitByTheAttemptBeforeAFailedReclaimIsRefused() { + // given + failedClaim(); + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + + // when + Throwable refused = + catchThrowable( + () -> ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(3))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("no longer held by attempt 1"); + } + + @Test + @DisplayName("한 claim 을 동시에 commit 하면 하나만 성공한다") + void concurrentCommitsOfOneClaimHaveExactlyOneWinner() throws Exception { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + List committed = + concurrently( + () -> { + try { + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); + return true; + } catch (IllegalStateException refused) { + return false; + } + }); + + // then + assertThat(committed).containsExactlyInAnyOrder(true, false); + } + @Test @DisplayName("보존 기간이 지난 COMMITTED·FAILED_TERMINAL 행은 정리된다") void finishedRecordsCompletedBeforeTheCutoffArePurged() { @@ -489,12 +619,12 @@ public abstract class GrpcOperationLedgerContract { // cutoff is compared with. private void failedClaim(GrpcOperationIdentity identity, Instant claimedAt) { ledger().claim(identity, FINGERPRINT, claimedAt); - ledger().markFailed(identity, claimedAt.plusSeconds(1)); + ledger().markFailed(identity, GrpcClaimToken.FIRST, claimedAt.plusSeconds(1)); } private void committedClaim(GrpcOperationIdentity identity, Instant claimedAt) { ledger().claim(identity, FINGERPRINT, claimedAt); - ledger().markCommitted(identity, OUTCOME, claimedAt.plusSeconds(1)); + ledger().markCommitted(identity, GrpcClaimToken.FIRST, OUTCOME, claimedAt.plusSeconds(1)); } private static GrpcOperationIdentity identityWithKeyHash(char hexDigit) { diff --git a/src/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java b/src/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java index 55479149..bc26942d 100644 --- a/src/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java +++ b/src/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java @@ -6,15 +6,18 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; /** * A hand-rolled ledger for tests of code that depends on {@link GrpcOperationLedger}. * *

    {@code putIfAbsent} on a concurrent map is the in-memory equivalent of the unique constraint - * the real adapter relies on, and {@code replace(key, expected, next)} of its conditional update, - * so claim, re-claim, stale takeover and purge are atomic here for the same reason they are there. - * The transitions match the JPA adapter's because {@link GrpcOperationLedgerContract} runs against - * both; a fake that allowed more would let the policy tests pass for code the database refuses. + * the real adapter relies on, and {@code replace(key, expected, next)} and {@code computeIfPresent} + * of its conditional updates, so claim, re-claim, stale takeover, commit, failure and purge are + * atomic here for the same reason they are there. A commit or failure is recorded only for the + * attempt whose {@link GrpcClaimToken} still holds the claim. The transitions match the JPA + * adapter's because {@link GrpcOperationLedgerContract} runs against both; a fake that allowed more + * would let the policy tests pass for code the database refuses. */ public final class InMemoryOperationLedger implements GrpcOperationLedger { @@ -46,16 +49,19 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { } @Override - public void markCommitted(GrpcOperationIdentity identity, String outcomeReference, Instant now) { + public void markCommitted( + GrpcOperationIdentity identity, + GrpcClaimToken claimToken, + String outcomeReference, + Instant now) { GrpcOperationLedgerRecord committed = records.computeIfPresent( identity.storageKey(), - (key, existing) -> - finish( - existing, - GrpcOperationLedgerState.COMMITTED, - Optional.of(outcomeReference), - now)); + (key, existing) -> { + requireHeldBy(existing, claimToken); + return finish( + existing, GrpcOperationLedgerState.COMMITTED, Optional.of(outcomeReference), now); + }); if (committed == null) { throw new IllegalStateException( "cannot commit operation '" + identity.storageKey() + "': no claim exists"); @@ -63,28 +69,36 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { } @Override - public void markFailed(GrpcOperationIdentity identity, Instant now) { + public boolean markFailed( + GrpcOperationIdentity identity, GrpcClaimToken claimToken, Instant now) { + AtomicBoolean recorded = new AtomicBoolean(); records.computeIfPresent( identity.storageKey(), - (key, existing) -> - finish(existing, GrpcOperationLedgerState.FAILED_TERMINAL, Optional.empty(), now)); + (key, existing) -> { + if (existing.state() != GrpcOperationLedgerState.IN_PROGRESS + || !existing.claimToken().equals(claimToken)) { + return existing; + } + recorded.set(true); + return finish(existing, GrpcOperationLedgerState.FAILED_TERMINAL, Optional.empty(), now); + }); + return recorded.get(); } @Override - public boolean reclaimFailed( + public Optional reclaimFailed( GrpcOperationIdentity identity, String requestFingerprint, Instant now) { GrpcOperationLedgerRecord current = records.get(identity.storageKey()); - return current != null - && current.state() == GrpcOperationLedgerState.FAILED_TERMINAL - && current.sameRequestAs(requestFingerprint) - && records.replace( - identity.storageKey(), - current, - GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now)); + if (current == null + || current.state() != GrpcOperationLedgerState.FAILED_TERMINAL + || !current.sameRequestAs(requestFingerprint)) { + return Optional.empty(); + } + return handOver(identity, requestFingerprint, current, now); } @Override - public boolean reclaimStale( + public Optional reclaimStale( GrpcOperationIdentity identity, String requestFingerprint, Instant claimedBefore, @@ -94,14 +108,28 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { "a stale-claim cutoff after now would leave the new claim stale as well"); } GrpcOperationLedgerRecord current = records.get(identity.storageKey()); - return current != null - && current.state() == GrpcOperationLedgerState.IN_PROGRESS - && current.claimedAt().isBefore(claimedBefore) - && current.sameRequestAs(requestFingerprint) - && records.replace( + if (current == null + || current.state() != GrpcOperationLedgerState.IN_PROGRESS + || !current.claimedAt().isBefore(claimedBefore) + || !current.sameRequestAs(requestFingerprint)) { + return Optional.empty(); + } + return handOver(identity, requestFingerprint, current, now); + } + + // The conditional replace is the compare-and-set: of two hand-overs of the same record, one wins. + private Optional handOver( + GrpcOperationIdentity identity, + String requestFingerprint, + GrpcOperationLedgerRecord current, + Instant now) { + GrpcClaimToken next = current.claimToken().next(); + return records.replace( identity.storageKey(), current, - GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now)); + GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now, next)) + ? Optional.of(next) + : Optional.empty(); } @Override @@ -132,6 +160,21 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { records.remove(identity.storageKey()); } + // An attempt that lost the claim must not commit beside the attempt that holds it; checked + // before the state, so a lost owner hears that it lost rather than that the row finished. + private static void requireHeldBy(GrpcOperationLedgerRecord existing, GrpcClaimToken claimToken) { + if (!existing.claimToken().equals(claimToken)) { + throw new IllegalStateException( + "operation '" + + existing.identity().storageKey() + + "' is no longer held by attempt " + + claimToken.attempt() + + "; attempt " + + existing.claimToken().attempt() + + " holds it"); + } + } + // Only an IN_PROGRESS claim finishes; a terminal row is re-opened by reclaimFailed alone. private static GrpcOperationLedgerRecord finish( GrpcOperationLedgerRecord existing, @@ -152,6 +195,7 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { existing.requestFingerprint(), outcomeReference, existing.claimedAt(), - Optional.of(now)); + Optional.of(now), + existing.claimToken()); } } diff --git a/src/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java b/src/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java index 6e28dfa2..d84695d5 100644 --- a/src/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java +++ b/src/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java @@ -16,7 +16,8 @@ final class GrpcOperationLedgerMigrations { private static final List IN_ORDER = List.of( "db/migration/grpc/V001__create_grpc_operation_ledger.sql", - "db/migration/grpc/V002__index_grpc_operation_ledger_finished_rows.sql"); + "db/migration/grpc/V002__index_grpc_operation_ledger_finished_rows.sql", + "db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql"); private GrpcOperationLedgerMigrations() {} diff --git a/src/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java b/src/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java index 7888f7e8..ec357db4 100644 --- a/src/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java +++ b/src/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java @@ -2,6 +2,7 @@ package dev.caskeleton.grpc.ledger; import static org.assertj.core.api.Assertions.assertThat; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerContract; @@ -102,14 +103,15 @@ class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract { String fingerprint = "sha256:0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0"; Instant now = Instant.parse("2026-09-25T10:00:00Z"); ledger.claim(id, fingerprint, now); - ledger.markFailed(id, now.plusSeconds(1)); + ledger.markFailed(id, GrpcClaimToken.FIRST, now.plusSeconds(1)); // when transactions.executeWithoutResult( status -> { adapter.claim(id, fingerprint, now.plusSeconds(2)); - adapter.reclaimFailed(id, fingerprint, now.plusSeconds(2)); - adapter.markCommitted(id, "outcome://document/42", now.plusSeconds(3)); + GrpcClaimToken reclaimed = + adapter.reclaimFailed(id, fingerprint, now.plusSeconds(2)).orElseThrow(); + adapter.markCommitted(id, reclaimed, "outcome://document/42", now.plusSeconds(3)); }); // then @@ -142,8 +144,11 @@ class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract { transactions.executeWithoutResult( status -> { adapter.claim(id, fingerprint, retryAt); - adapter.reclaimStale(id, fingerprint, retryAt.minusSeconds(300), retryAt); - adapter.markCommitted(id, "outcome://document/42", retryAt.plusSeconds(1)); + GrpcClaimToken takenOver = + adapter + .reclaimStale(id, fingerprint, retryAt.minusSeconds(300), retryAt) + .orElseThrow(); + adapter.markCommitted(id, takenOver, "outcome://document/42", retryAt.plusSeconds(1)); }); // then @@ -188,33 +193,36 @@ class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract { @Override public void markCommitted( - GrpcOperationIdentity identity, String outcomeReference, Instant now) { + GrpcOperationIdentity identity, + GrpcClaimToken claimToken, + String outcomeReference, + Instant now) { transactions.executeWithoutResult( - status -> delegate.markCommitted(identity, outcomeReference, now)); + status -> delegate.markCommitted(identity, claimToken, outcomeReference, now)); } @Override - public void markFailed(GrpcOperationIdentity identity, Instant now) { - transactions.executeWithoutResult(status -> delegate.markFailed(identity, now)); - } - - @Override - public boolean reclaimFailed( - GrpcOperationIdentity identity, String requestFingerprint, Instant now) { + public boolean markFailed( + GrpcOperationIdentity identity, GrpcClaimToken claimToken, Instant now) { return Boolean.TRUE.equals( - transactions.execute( - status -> delegate.reclaimFailed(identity, requestFingerprint, now))); + transactions.execute(status -> delegate.markFailed(identity, claimToken, now))); } @Override - public boolean reclaimStale( + public Optional reclaimFailed( + GrpcOperationIdentity identity, String requestFingerprint, Instant now) { + return transactions.execute( + status -> delegate.reclaimFailed(identity, requestFingerprint, now)); + } + + @Override + public Optional reclaimStale( GrpcOperationIdentity identity, String requestFingerprint, Instant claimedBefore, Instant now) { - return Boolean.TRUE.equals( - transactions.execute( - status -> delegate.reclaimStale(identity, requestFingerprint, claimedBefore, now))); + return transactions.execute( + status -> delegate.reclaimStale(identity, requestFingerprint, claimedBefore, now)); } @Override diff --git a/src/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerEntity.java b/src/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerEntity.java index 54e26abb..571f36e0 100644 --- a/src/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerEntity.java +++ b/src/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerEntity.java @@ -1,5 +1,6 @@ package dev.caskeleton.grpc.ledger; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerState; @@ -56,6 +57,9 @@ public class GrpcOperationLedgerEntity { @Column(name = "completed_at") private Instant completedAt; + @Column(name = "claim_attempt", nullable = false) + private long claimAttempt; + /** JPA requires a no-argument constructor. */ protected GrpcOperationLedgerEntity() { // for JPA @@ -70,6 +74,7 @@ public class GrpcOperationLedgerEntity { this.requestFingerprint = requestFingerprint; this.state = GrpcOperationLedgerState.IN_PROGRESS; this.claimedAt = claimedAt; + this.claimAttempt = GrpcClaimToken.FIRST.attempt(); } /** A new claim, in progress. */ @@ -114,13 +119,26 @@ public class GrpcOperationLedgerEntity { return Optional.ofNullable(completedAt); } + /** The attempt that holds, or last held, the claim. */ + public GrpcClaimToken getClaimToken() { + return new GrpcClaimToken(claimAttempt); + } + /** - * Records a commit with its replayable outcome. + * Records a commit with its replayable outcome, on a row this code already holds. * - * @throws IllegalStateException when the claim already finished; a second terminal transition - * would overwrite the outcome a retry is entitled to receive + *

    The row-level statement of the rule. {@link JpaGrpcOperationLedger} applies the same rule as + * one conditional {@code UPDATE} ({@link GrpcOperationLedgerRepository#tryMarkCommitted}), so + * that the database, not a value read earlier, decides between concurrent transitions. + * + * @param claimToken the attempt committing + * @throws IllegalStateException when another attempt holds the claim, or the claim already + * finished; a second terminal transition would overwrite the outcome a retry is entitled to + * receive */ - public void markCommitted(String outcomeReference, Instant completedAt) { + public void markCommitted( + GrpcClaimToken claimToken, String outcomeReference, Instant completedAt) { + requireHeldBy(claimToken); requireInProgress(); if (outcomeReference == null || outcomeReference.isBlank()) { throw new IllegalArgumentException( @@ -131,8 +149,16 @@ public class GrpcOperationLedgerEntity { this.completedAt = completedAt; } - /** Records a terminal failure. */ - public void markFailed(Instant completedAt) { + /** + * Records a terminal failure, on a row this code already holds; the row-level statement of {@link + * GrpcOperationLedgerRepository#tryMarkFailed}. + * + * @param claimToken the attempt that failed + * @throws IllegalStateException when another attempt holds the claim, or the claim already + * finished + */ + public void markFailed(GrpcClaimToken claimToken, Instant completedAt) { + requireHeldBy(claimToken); requireInProgress(); this.state = GrpcOperationLedgerState.FAILED_TERMINAL; this.completedAt = completedAt; @@ -146,7 +172,21 @@ public class GrpcOperationLedgerEntity { requestFingerprint, Optional.ofNullable(outcomeReference), claimedAt, - Optional.ofNullable(completedAt)); + Optional.ofNullable(completedAt), + getClaimToken()); + } + + private void requireHeldBy(GrpcClaimToken claimToken) { + if (claimAttempt != claimToken.attempt()) { + throw new IllegalStateException( + "operation '" + + storageKey + + "' is no longer held by attempt " + + claimToken.attempt() + + "; attempt " + + claimAttempt + + " holds it"); + } } private void requireInProgress() { diff --git a/src/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java b/src/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java index d48c61c3..9a043670 100644 --- a/src/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java +++ b/src/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java @@ -11,10 +11,12 @@ import org.springframework.transaction.annotation.Transactional; /** * The narrow persistence interface the ledger adapter needs. * - *

    The claim path is deliberately not {@code save}. A claim entity already has a non-null string - * id, so Spring Data JPA treats it as detached and routes {@code save} through {@code merge}; merge - * can overwrite a committed row back to {@code IN_PROGRESS}. Claiming therefore uses one native - * insert with PostgreSQL conflict handling and decides ownership from the affected-row count. + *

    No path here is {@code save}. A claim entity already has a non-null string id, so Spring Data + * JPA treats it as detached and routes {@code save} through {@code merge}; merge can overwrite a + * committed row back to {@code IN_PROGRESS}, and a load-check-save commit lets two attempts that + * both read {@code IN_PROGRESS} both write. Claiming is therefore one native insert with PostgreSQL + * conflict handling, and every transition is one conditional update; ownership and every race are + * decided by the affected-row count. */ public interface GrpcOperationLedgerRepository extends Repository { @@ -31,7 +33,8 @@ public interface GrpcOperationLedgerRepository state, outcome_reference, claimed_at, - completed_at + completed_at, + claim_attempt ) VALUES ( :storageKey, :callerFingerprint, @@ -41,7 +44,8 @@ public interface GrpcOperationLedgerRepository 'IN_PROGRESS', NULL, :claimedAt, - NULL + NULL, + 1 ) ON CONFLICT DO NOTHING """; @@ -71,7 +75,11 @@ public interface GrpcOperationLedgerRepository * claim's read-on-conflict does) would otherwise still read {@code FAILED_TERMINAL} and refuse * the commit. * - * @return one when this caller re-claimed the row, zero otherwise + *

    Conditional on the attempt the caller read as well, and the new attempt is that one plus + * one: the caller then knows its own fencing token without reading the row again. + * + * @return one when this caller re-claimed the row as attempt {@code readAttempt + 1}, zero + * otherwise */ @Modifying(flushAutomatically = true, clearAutomatically = true) @Transactional @@ -81,15 +89,18 @@ public interface GrpcOperationLedgerRepository UPDATE grpc_operation_ledger SET state = 'IN_PROGRESS', claimed_at = :claimedAt, - completed_at = NULL + completed_at = NULL, + claim_attempt = claim_attempt + 1 WHERE storage_key = :storageKey AND state = 'FAILED_TERMINAL' AND request_fingerprint = :requestFingerprint + AND claim_attempt = :readAttempt """, nativeQuery = true) int tryReclaimFailed( @Param("storageKey") String storageKey, @Param("requestFingerprint") String requestFingerprint, + @Param("readAttempt") long readAttempt, @Param("claimedAt") Instant claimedAt); /** @@ -99,9 +110,11 @@ public interface GrpcOperationLedgerRepository *

    Same shape as {@link #tryReclaimFailed}: one conditional update decided by PostgreSQL's row * lock. A concurrent second takeover waits for the winner, re-evaluates the condition against the * winner's row — whose {@code claimed_at} is now {@code :claimedAt}, not before the cutoff — and - * updates nothing. It clears the persistence context for the same reason. + * updates nothing. It clears the persistence context for the same reason. The attempt grows by + * one, as in {@link #tryReclaimFailed}, and that is what fences the owner taken over. * - * @return one when this caller took the row over, zero otherwise + * @return one when this caller took the row over as attempt {@code readAttempt + 1}, zero + * otherwise */ @Modifying(flushAutomatically = true, clearAutomatically = true) @Transactional @@ -109,19 +122,76 @@ public interface GrpcOperationLedgerRepository value = """ UPDATE grpc_operation_ledger - SET claimed_at = :claimedAt + SET claimed_at = :claimedAt, + claim_attempt = claim_attempt + 1 WHERE storage_key = :storageKey AND state = 'IN_PROGRESS' AND request_fingerprint = :requestFingerprint AND claimed_at < :claimedBefore + AND claim_attempt = :readAttempt """, nativeQuery = true) int tryReclaimStale( @Param("storageKey") String storageKey, @Param("requestFingerprint") String requestFingerprint, + @Param("readAttempt") long readAttempt, @Param("claimedBefore") Instant claimedBefore, @Param("claimedAt") Instant claimedAt); + /** + * Commits the claim if attempt {@code claimAttempt} still holds it. + * + *

    One conditional update rather than load-check-save: two commits of the same claim both read + * {@code IN_PROGRESS} under load-check-save and both wrote; here PostgreSQL's row lock makes the + * second re-evaluate the condition against the first's {@code COMMITTED} row and update nothing. + * An attempt whose claim was taken over or re-claimed matches nothing for the same reason. + * + * @return one when the claim was committed, zero otherwise + */ + @Modifying(flushAutomatically = true, clearAutomatically = true) + @Transactional + @Query( + value = + """ + UPDATE grpc_operation_ledger + SET state = 'COMMITTED', + outcome_reference = :outcomeReference, + completed_at = :completedAt + WHERE storage_key = :storageKey + AND state = 'IN_PROGRESS' + AND claim_attempt = :claimAttempt + """, + nativeQuery = true) + int tryMarkCommitted( + @Param("storageKey") String storageKey, + @Param("claimAttempt") long claimAttempt, + @Param("outcomeReference") String outcomeReference, + @Param("completedAt") Instant completedAt); + + /** + * Marks the claim terminally failed if attempt {@code claimAttempt} still holds it; the same + * conditional shape as {@link #tryMarkCommitted}. + * + * @return one when the failure was recorded, zero otherwise + */ + @Modifying(flushAutomatically = true, clearAutomatically = true) + @Transactional + @Query( + value = + """ + UPDATE grpc_operation_ledger + SET state = 'FAILED_TERMINAL', + completed_at = :completedAt + WHERE storage_key = :storageKey + AND state = 'IN_PROGRESS' + AND claim_attempt = :claimAttempt + """, + nativeQuery = true) + int tryMarkFailed( + @Param("storageKey") String storageKey, + @Param("claimAttempt") long claimAttempt, + @Param("completedAt") Instant completedAt); + /** * Deletes up to {@code limit} finished rows that completed before {@code completedBefore}, oldest * first. Never an {@code IN_PROGRESS} row. @@ -155,9 +225,6 @@ public interface GrpcOperationLedgerRepository int deleteFinishedBefore( @Param("completedBefore") Instant completedBefore, @Param("limit") int limit); - /** Updates an existing claim after it has been loaded. */ - GrpcOperationLedgerEntity save(GrpcOperationLedgerEntity entity); - /** Reads a claim by its derived storage key. */ Optional findById(String storageKey); diff --git a/src/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java b/src/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java index 55d80bcd..9db0c3d8 100644 --- a/src/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java +++ b/src/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java @@ -1,5 +1,6 @@ package dev.caskeleton.grpc.ledger; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; @@ -16,6 +17,11 @@ import java.util.Optional; * {@code save}: this entity has a non-null application-assigned id, so save routes through merge * and can overwrite an existing terminal row. * + *

    Every transition — re-claim, takeover, commit, failure — is one conditional {@code UPDATE} + * that names the claim attempt ({@link GrpcClaimToken}). An attempt whose claim was handed over + * matches nothing: its commit is refused and its failure ignored, which is what keeps a takeover of + * a slow owner from becoming two concurrent runs (re-review 2026-11-03, RR-02-04). + * *

    No transaction annotations here. The commit is meant to happen inside the caller's business * transaction, and a {@code REQUIRES_NEW} on this adapter would put the ledger in its own * transaction — reintroducing the window in which the mutation is durable and the claim is not. @@ -36,6 +42,28 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { public Optional claim( GrpcOperationIdentity identity, String requestFingerprint, Instant now) { requireIdentity(identity); + // Twice at most. Zero rows means another claim holds the key; an empty read after that means + // the row went in between (a purge of an expired record, or a release). Treating that as + // "ours" would run the mutation without a claim row to guard it, so the insert is tried once + // more and decides again. + for (int round = 0; round < 2; round++) { + if (insertClaim(identity, requestFingerprint, now)) { + return Optional.empty(); + } + Optional existing = repository.findById(identity.storageKey()); + if (existing.isPresent()) { + return Optional.of(existing.get().toRecord(identity)); + } + } + throw new IllegalStateException( + "claim for '" + + identity.storageKey() + + "' conflicted twice, but there was no row to read either time; another owner keeps" + + " releasing it. Retry the claim."); + } + + private boolean insertClaim( + GrpcOperationIdentity identity, String requestFingerprint, Instant now) { int inserted = repository.tryInsertClaim( identity.storageKey(), @@ -44,25 +72,10 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { identity.idempotencyKeyHash(), requestFingerprint, now); - if (inserted == 1) { - return Optional.empty(); - } - if (inserted != 0) { + if (inserted != 0 && inserted != 1) { throw new IllegalStateException("claim insert affected an unexpected row count: " + inserted); } - // Zero rows means another claim holds the key. An empty read now means that owner released it - // in between; treating that as "ours" would run the mutation without a claim row to guard it. - return Optional.of( - repository - .findById(identity.storageKey()) - .orElseThrow( - () -> - new IllegalStateException( - "claim for '" - + identity.storageKey() - + "' conflicted, but there was no row to read; the other owner released" - + " it in between. Retry the claim.")) - .toRecord(identity)); + return inserted == 1; } @Override @@ -71,10 +84,30 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { return repository.findById(identity.storageKey()).map(entity -> entity.toRecord(identity)); } + /** + * Commits the claim if {@code claimToken} still holds it, as one conditional update. + * + *

    When nothing matched, the row is read only to say why; the refusal itself was the + * database's, and the exception is what rolls back the caller's business transaction. + */ @Override - public void markCommitted(GrpcOperationIdentity identity, String outcomeReference, Instant now) { + public void markCommitted( + GrpcOperationIdentity identity, + GrpcClaimToken claimToken, + String outcomeReference, + Instant now) { requireIdentity(identity); - GrpcOperationLedgerEntity entity = + requireToken(claimToken); + if (outcomeReference == null || outcomeReference.isBlank()) { + throw new IllegalArgumentException( + "a commit records the outcome to replay; without it the row cannot answer a retry"); + } + if (repository.tryMarkCommitted( + identity.storageKey(), claimToken.attempt(), outcomeReference, now) + == 1) { + return; + } + GrpcOperationLedgerEntity current = repository .findById(identity.storageKey()) .orElseThrow( @@ -84,31 +117,54 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { + identity.storageKey() + "': no claim exists. The claim is what a retry reads; committing " + "without one leaves the mutation durable and unguarded.")); - entity.markCommitted(outcomeReference, now); - repository.save(entity); + if (!current.getClaimToken().equals(claimToken)) { + throw new IllegalStateException( + "operation '" + + identity.storageKey() + + "' is no longer held by attempt " + + claimToken.attempt() + + "; attempt " + + current.getClaimToken().attempt() + + " holds it, so this attempt's transaction must roll back"); + } + throw new IllegalStateException( + "operation '" + + identity.storageKey() + + "' is already " + + current.getState() + + " and cannot transition again"); } @Override - public void markFailed(GrpcOperationIdentity identity, Instant now) { + public boolean markFailed( + GrpcOperationIdentity identity, GrpcClaimToken claimToken, Instant now) { requireIdentity(identity); - repository - .findById(identity.storageKey()) - .ifPresent( - entity -> { - entity.markFailed(now); - repository.save(entity); - }); + requireToken(claimToken); + return repository.tryMarkFailed(identity.storageKey(), claimToken.attempt(), now) == 1; } + /** + * Re-claims a failed record. Conditional on the attempt read just before, so the new token is + * known exactly: that attempt plus one. A record that changed in between matches nothing, and + * this caller has lost the race it would have lost anyway. + */ @Override - public boolean reclaimFailed( + public Optional reclaimFailed( GrpcOperationIdentity identity, String requestFingerprint, Instant now) { requireIdentity(identity); - return repository.tryReclaimFailed(identity.storageKey(), requestFingerprint, now) == 1; + Optional read = currentToken(identity); + if (read.isEmpty()) { + return Optional.empty(); + } + return repository.tryReclaimFailed( + identity.storageKey(), requestFingerprint, read.get().attempt(), now) + == 1 + ? Optional.of(read.get().next()) + : Optional.empty(); } @Override - public boolean reclaimStale( + public Optional reclaimStale( GrpcOperationIdentity identity, String requestFingerprint, Instant claimedBefore, @@ -118,8 +174,19 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { throw new IllegalArgumentException( "a stale-claim cutoff after now would leave the new claim stale as well"); } - return repository.tryReclaimStale(identity.storageKey(), requestFingerprint, claimedBefore, now) - == 1; + Optional read = currentToken(identity); + if (read.isEmpty()) { + return Optional.empty(); + } + return repository.tryReclaimStale( + identity.storageKey(), requestFingerprint, read.get().attempt(), claimedBefore, now) + == 1 + ? Optional.of(read.get().next()) + : Optional.empty(); + } + + private Optional currentToken(GrpcOperationIdentity identity) { + return repository.findById(identity.storageKey()).map(GrpcOperationLedgerEntity::getClaimToken); } @Override @@ -139,6 +206,12 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { repository.deleteById(identity.storageKey()); } + private static void requireToken(GrpcClaimToken claimToken) { + if (claimToken == null) { + throw new IllegalArgumentException("a transition names the attempt that holds the claim"); + } + } + private static void requireIdentity(GrpcOperationIdentity identity) { if (identity == null) { throw new IllegalArgumentException("an operation identity is required"); diff --git a/src/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql b/src/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql new file mode 100644 index 00000000..23b03687 --- /dev/null +++ b/src/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql @@ -0,0 +1,20 @@ +-- Claim attempts: the fencing token of the operation ledger (re-review 2026-11-03, RR-02-04). +-- +-- A claim changes hands when a stale one is taken over after the claim TTL or a failed one is +-- re-claimed. The owner that lost it may still be running, and when it finishes it records an +-- outcome. Without knowing which attempt holds the claim, that record landed on the new attempt's +-- claim: an old owner's failure turned a running attempt FAILED_TERMINAL, the next retry re-claimed +-- it, and two attempts ran the mutation at once. +-- +-- claim_attempt is 1 for the first claim and grows by one with every hand-over. A commit or a +-- failure is one conditional UPDATE that names the attempt (WHERE state = 'IN_PROGRESS' AND +-- claim_attempt = :attempt), so a lost owner's record changes nothing and two commits of the same +-- attempt have one winner. +-- +-- Existing rows are their first attempt: no row has been handed over under a token yet. The table is +-- small and the platform is not adopted anywhere, so a plain ALTER with a default is enough. +ALTER TABLE grpc_operation_ledger + ADD COLUMN claim_attempt BIGINT NOT NULL DEFAULT 1; + +ALTER TABLE grpc_operation_ledger + ADD CONSTRAINT ck_grpc_operation_ledger_claim_attempt_positive CHECK (claim_attempt > 0); diff --git a/src/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java b/src/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java index e3aada69..f2f90758 100644 --- a/src/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java +++ b/src/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; @@ -34,12 +35,12 @@ class GrpcOperationLedgerRepositoryTest { "sha256:aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899"; /** - * An in-memory stand-in for Spring Data's non-null-id save semantics. + * An in-memory stand-in for the repository's native statements. * - *

    A newly constructed ledger entity already has its string id, so a real {@code - * SimpleJpaRepository.save} treats it as detached and merges it. The old fake used {@code - * putIfAbsent} and therefore tested the database constraint we wished save() would hit rather - * than the merge behavior it actually uses. + *

    The insert is {@code putIfAbsent}, the stand-in for the unique constraint. The commit and + * failure are the entity's own transition rules, which state the conditions the native {@code + * UPDATE}s carry (state {@code IN_PROGRESS} and the same claim attempt); a refused transition + * updates zero rows. The re-claims and the purge are certified against PostgreSQL only. */ private static class InMemoryRepository implements GrpcOperationLedgerRepository { private final ConcurrentMap rows = new ConcurrentHashMap<>(); @@ -61,14 +62,19 @@ class GrpcOperationLedgerRepositoryTest { } @Override - public int tryReclaimFailed(String storageKey, String requestFingerprint, Instant claimedAt) { + public int tryReclaimFailed( + String storageKey, String requestFingerprint, long readAttempt, Instant claimedAt) { throw new UnsupportedOperationException( "the re-claim is certified against PostgreSQL by JpaGrpcOperationLedgerContractIT"); } @Override public int tryReclaimStale( - String storageKey, String requestFingerprint, Instant claimedBefore, Instant claimedAt) { + String storageKey, + String requestFingerprint, + long readAttempt, + Instant claimedBefore, + Instant claimedAt) { throw new UnsupportedOperationException( "the stale takeover is certified against PostgreSQL by JpaGrpcOperationLedgerContractIT"); } @@ -80,9 +86,32 @@ class GrpcOperationLedgerRepositoryTest { } @Override - public GrpcOperationLedgerEntity save(GrpcOperationLedgerEntity entity) { - rows.put(entity.getStorageKey(), entity); - return entity; + public int tryMarkCommitted( + String storageKey, long claimAttempt, String outcomeReference, Instant completedAt) { + GrpcOperationLedgerEntity row = rows.get(storageKey); + if (row == null) { + return 0; + } + try { + row.markCommitted(new GrpcClaimToken(claimAttempt), outcomeReference, completedAt); + return 1; + } catch (IllegalStateException refused) { + return 0; + } + } + + @Override + public int tryMarkFailed(String storageKey, long claimAttempt, Instant completedAt) { + GrpcOperationLedgerEntity row = rows.get(storageKey); + if (row == null) { + return 0; + } + try { + row.markFailed(new GrpcClaimToken(claimAttempt), completedAt); + return 1; + } catch (IllegalStateException refused) { + return 0; + } } @Override @@ -139,6 +168,7 @@ class GrpcOperationLedgerRepositoryTest { } @Test + @DisplayName("충돌한 행이 다시 시도해도 계속 사라지면 소유로 보지 않고 거부한다") void aConflictWhoseRowVanishedBeforeTheReadIsNotTakenAsOwnership() { // given InMemoryRepository conflictThenReleased = @@ -166,6 +196,54 @@ class GrpcOperationLedgerRepositoryTest { .hasMessageContaining("no row to read"); } + /** + * A purge (or release) can delete the conflicting row between the insert and the read. The read + * then finds nothing, and the claim tries the insert once more instead of failing the call + * (re-review 2026-11-03, RR-02-04). + */ + @Test + @DisplayName("충돌한 행이 읽기 전에 사라지면 claim 은 insert 를 한 번 더 시도해 소유한다") + void aConflictWhoseRowVanishedOnceIsClaimedByTheRetriedInsert() { + // given + InMemoryRepository conflictOnceThenFree = + new InMemoryRepository() { + private boolean conflicted; + + @Override + public int tryInsertClaim( + String storageKey, + String callerFingerprint, + String fullMethodName, + String idempotencyKeyHash, + String requestFingerprint, + Instant claimedAt) { + if (!conflicted) { + conflicted = true; + return 0; // a row held the key and was purged before this read + } + return super.tryInsertClaim( + storageKey, + callerFingerprint, + fullMethodName, + idempotencyKeyHash, + requestFingerprint, + claimedAt); + } + }; + GrpcOperationLedger racing = new JpaGrpcOperationLedger(conflictOnceThenFree); + + // when + Optional existing = + racing.claim(identity("tenant-1.actor-1"), FINGERPRINT, NOW); + + // then + assertThat(existing).isEmpty(); + assertThat(racing.find(identity("tenant-1.actor-1"))) + .get() + .extracting(GrpcOperationLedgerRecord::state, GrpcOperationLedgerRecord::claimToken) + .containsExactly(GrpcOperationLedgerState.IN_PROGRESS, GrpcClaimToken.FIRST); + } + @Test @DisplayName("one caller's key does not collide with another's") void callersDoNotCollide() { @@ -179,7 +257,7 @@ class GrpcOperationLedgerRepositoryTest { GrpcOperationIdentity id = identity("tenant-1.actor-1"); ledger.claim(id, FINGERPRINT, NOW); - ledger.markCommitted(id, "outcome://document/42", NOW.plusSeconds(1)); + ledger.markCommitted(id, GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); GrpcOperationLedgerRecord record = ledger.find(id).orElseThrow(); assertThat(record.state()).isEqualTo(GrpcOperationLedgerState.COMMITTED); @@ -192,7 +270,7 @@ class GrpcOperationLedgerRepositoryTest { void retryingACommittedClaimPreservesTheWinner() { GrpcOperationIdentity id = identity("tenant-1.actor-1"); assertThat(ledger.claim(id, FINGERPRINT, NOW)).isEmpty(); - ledger.markCommitted(id, "outcome://document/42", NOW.plusSeconds(1)); + ledger.markCommitted(id, GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); GrpcOperationLedgerRecord retry = ledger.claim(id, OTHER_FINGERPRINT, NOW.plusSeconds(2)).orElseThrow(); @@ -206,7 +284,12 @@ class GrpcOperationLedgerRepositoryTest { @DisplayName("a commit without a claim is refused rather than silently inserted") void committingWithoutAClaimIsRefused() { assertThatThrownBy( - () -> ledger.markCommitted(identity("tenant-1.actor-1"), "outcome://document/42", NOW)) + () -> + ledger.markCommitted( + identity("tenant-1.actor-1"), + GrpcClaimToken.FIRST, + "outcome://document/42", + NOW)) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("durable and unguarded"); } @@ -216,9 +299,12 @@ class GrpcOperationLedgerRepositoryTest { void aTerminalRowDoesNotTransitionAgain() { GrpcOperationIdentity id = identity("tenant-1.actor-1"); ledger.claim(id, FINGERPRINT, NOW); - ledger.markCommitted(id, "outcome://document/42", NOW.plusSeconds(1)); + ledger.markCommitted(id, GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); - assertThatThrownBy(() -> ledger.markCommitted(id, "outcome://document/99", NOW.plusSeconds(2))) + assertThatThrownBy( + () -> + ledger.markCommitted( + id, GrpcClaimToken.FIRST, "outcome://document/99", NOW.plusSeconds(2))) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("cannot transition again"); } @@ -240,7 +326,7 @@ class GrpcOperationLedgerRepositoryTest { GrpcOperationIdentity id = identity("tenant-1.actor-1"); ledger.claim(id, FINGERPRINT, NOW); - ledger.markFailed(id, NOW.plusSeconds(1)); + ledger.markFailed(id, GrpcClaimToken.FIRST, NOW.plusSeconds(1)); assertThat(ledger.find(id).orElseThrow().state()) .isEqualTo(GrpcOperationLedgerState.FAILED_TERMINAL); @@ -259,7 +345,8 @@ class GrpcOperationLedgerRepositoryTest { FINGERPRINT, Optional.empty(), NOW, - Optional.of(NOW))) + Optional.of(NOW), + GrpcClaimToken.FIRST)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("cannot be replayed"); } diff --git a/src/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyDecision.java b/src/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyDecision.java index ecc9f400..ca7ebef1 100644 --- a/src/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyDecision.java +++ b/src/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyDecision.java @@ -1,5 +1,6 @@ package dev.caskeleton.grpc.idempotency; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import java.time.Duration; import java.util.Optional; @@ -10,9 +11,18 @@ import java.util.Optional; * duplicate". A duplicate of a committed operation should be answered with the original result; a * duplicate of a running one is a race that has to wait or be refused; and a key reused for a * different request is a caller bug that must be reported rather than absorbed. + * + *

    A {@code PROCEED} that won a claim carries the claim's {@link GrpcClaimToken}. The caller + * hands it back to {@code recordCommit} or {@code recordFailure}: the ledger records an outcome + * only for the attempt that still holds the claim, so a caller whose claim was taken over cannot + * write over the attempt that took it (re-review 2026-11-03, RR-02-04). */ public record GrpcIdempotencyDecision( - Outcome outcome, Optional outcomeReference, Duration retryAfter, String reason) { + Outcome outcome, + Optional outcomeReference, + Duration retryAfter, + String reason, + Optional claimToken) { /** The five ways an idempotency check can end. */ public enum Outcome { @@ -48,11 +58,31 @@ public record GrpcIdempotencyDecision( if (outcome == Outcome.WAIT_AND_POLL && retryAfter.isZero()) { throw new IllegalArgumentException("a WAIT_AND_POLL decision says how long to wait"); } + if (claimToken == null) { + throw new IllegalArgumentException("the claim token Optional must be present"); + } + if (outcome != Outcome.PROCEED && claimToken.isPresent()) { + throw new IllegalArgumentException("only a PROCEED decision can hold a claim"); + } } - /** Proceed with the operation. */ + /** Proceed with an operation that is not claimed: the method takes no idempotency key. */ public static GrpcIdempotencyDecision proceed(String reason) { - return new GrpcIdempotencyDecision(Outcome.PROCEED, Optional.empty(), Duration.ZERO, reason); + return new GrpcIdempotencyDecision( + Outcome.PROCEED, Optional.empty(), Duration.ZERO, reason, Optional.empty()); + } + + /** + * Proceed as the attempt that holds the claim. + * + * @param claimToken the token to hand back when recording the commit or the failure + */ + public static GrpcIdempotencyDecision proceedHolding(GrpcClaimToken claimToken, String reason) { + if (claimToken == null) { + throw new IllegalArgumentException("a claimed PROCEED carries its claim token"); + } + return new GrpcIdempotencyDecision( + Outcome.PROCEED, Optional.empty(), Duration.ZERO, reason, Optional.of(claimToken)); } /** Replay a stored outcome. */ @@ -61,13 +91,18 @@ public record GrpcIdempotencyDecision( Outcome.REPLAY, Optional.of(outcomeReference), Duration.ZERO, - "an identical request already committed; returning the stored outcome"); + "an identical request already committed; returning the stored outcome", + Optional.empty()); } /** Wait and poll for an in-progress claim. */ public static GrpcIdempotencyDecision waitAndPoll(Duration retryAfter) { return new GrpcIdempotencyDecision( - Outcome.WAIT_AND_POLL, Optional.empty(), retryAfter, "an identical request is in progress"); + Outcome.WAIT_AND_POLL, + Optional.empty(), + retryAfter, + "an identical request is in progress", + Optional.empty()); } /** Refuse a duplicate of an in-progress claim. */ @@ -76,7 +111,8 @@ public record GrpcIdempotencyDecision( Outcome.REJECT_IN_PROGRESS, Optional.empty(), Duration.ZERO, - "an identical request is in progress and this method does not wait for it"); + "an identical request is in progress and this method does not wait for it", + Optional.empty()); } /** Report a key reused for a different request. */ @@ -85,7 +121,8 @@ public record GrpcIdempotencyDecision( Outcome.FINGERPRINT_MISMATCH, Optional.empty(), Duration.ZERO, - "the idempotency key was already used for a different request"); + "the idempotency key was already used for a different request", + Optional.empty()); } /** Whether the use case should run. */ diff --git a/src/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java b/src/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java index b48d9a65..5387ffa2 100644 --- a/src/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java +++ b/src/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java @@ -1,5 +1,6 @@ package dev.caskeleton.grpc.idempotency; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; @@ -21,7 +22,9 @@ import java.util.Optional; * *

    A claim does not block its key forever. One still {@code IN_PROGRESS} after the claim TTL is * treated as abandoned by a dead owner and taken over by the next retry of the same request — one - * winner, like the re-claim of a terminal failure. + * winner, like the re-claim of a terminal failure. The owner taken over may only have been slow; it + * still holds the old {@link GrpcClaimToken}, so its later commit is refused and its later failure + * ignored instead of landing on the attempt that took the claim. * *

    The ordering matters and is easy to get subtly wrong. The claim is attempted first, * atomically; only then is the existing record examined. Reading first and claiming second leaves a @@ -124,7 +127,8 @@ public final class GrpcIdempotencyInterceptor { callerFingerprint, method, GrpcRequestFingerprint.hashIdempotencyKey(idempotencyKey)); Optional existing = ledger.claim(identity, requestFingerprint, now); if (existing.isEmpty()) { - return GrpcIdempotencyDecision.proceed("claimed; this caller owns the operation"); + return GrpcIdempotencyDecision.proceedHolding( + GrpcClaimToken.FIRST, "claimed; this caller owns the operation"); } GrpcOperationLedgerRecord record = existing.get(); @@ -142,24 +146,32 @@ public final class GrpcIdempotencyInterceptor { + identity.storageKey() + "' has no outcome reference to replay"))); } - if (record.state() == GrpcOperationLedgerState.FAILED_TERMINAL - && ledger.reclaimFailed(identity, requestFingerprint, now)) { + if (record.state() == GrpcOperationLedgerState.FAILED_TERMINAL) { // A terminal failure is not replayed: the caller asked for the operation, it did not happen, // and repeating it is exactly what the key makes safe — but only for the one retry that // re-claims the row. Proceeding unclaimed runs concurrent retries twice and leaves the // commit without an owner. - return GrpcIdempotencyDecision.proceed( - "the previous attempt failed terminally; this caller re-claimed it for a new attempt"); + Optional reclaimed = ledger.reclaimFailed(identity, requestFingerprint, now); + if (reclaimed.isPresent()) { + return GrpcIdempotencyDecision.proceedHolding( + reclaimed.get(), + "the previous attempt failed terminally; this caller re-claimed it for a new attempt"); + } } if (record.state() == GrpcOperationLedgerState.IN_PROGRESS) { Instant cutoff = now.minus(claimTtl); - if (record.claimedAt().isBefore(cutoff) - && ledger.reclaimStale(identity, requestFingerprint, cutoff, now)) { - // The owner outlived the TTL without committing or failing, so it is presumed dead. Its - // mutation did not commit if the commit was recorded in the mutation's own transaction, - // which the ledger contract requires; when an adopter commits them separately, the - // abandoned attempt's outcome is unknown and this re-run can duplicate it. - return GrpcIdempotencyDecision.proceed( + // The owner outlived the TTL without committing or failing, so it is presumed dead. Its + // mutation did not commit if the commit was recorded in the mutation's own transaction, + // which the ledger contract requires; when an adopter commits them separately, the + // abandoned attempt's outcome is unknown and this re-run can duplicate it. An owner that was + // only slow is fenced by the new claim token: its commit is refused, its failure ignored. + Optional takenOver = + record.claimedAt().isBefore(cutoff) + ? ledger.reclaimStale(identity, requestFingerprint, cutoff, now) + : Optional.empty(); + if (takenOver.isPresent()) { + return GrpcIdempotencyDecision.proceedHolding( + takenOver.get(), "the previous claim was abandoned (older than the " + claimTtl + " claim TTL); this caller took it over for a new attempt"); @@ -178,26 +190,42 @@ public final class GrpcIdempotencyInterceptor { *

    Called from inside the business transaction wherever the datastore permits it. A ledger * committed after the mutation leaves a window in which the write is durable and the claim is * not. + * + * @param claimToken the token of the {@code PROCEED} decision this attempt ran under + * @throws IllegalStateException when this attempt no longer holds the claim — another attempt + * took it over — so the business transaction rolls back instead of committing a second run */ public void recordCommit( GrpcMethodName method, String callerFingerprint, String idempotencyKey, + GrpcClaimToken claimToken, String outcomeReference, Instant now) { ledger.markCommitted( new GrpcOperationIdentity( callerFingerprint, method, GrpcRequestFingerprint.hashIdempotencyKey(idempotencyKey)), + claimToken, outcomeReference, now); } - /** Records a terminal failure against the claim. */ - public void recordFailure( - GrpcMethodName method, String callerFingerprint, String idempotencyKey, Instant now) { - ledger.markFailed( + /** + * Records a terminal failure against the claim, if this attempt still holds it. + * + * @param claimToken the token of the {@code PROCEED} decision this attempt ran under + * @return false when another attempt holds the claim now; its run is not failed by this one + */ + public boolean recordFailure( + GrpcMethodName method, + String callerFingerprint, + String idempotencyKey, + GrpcClaimToken claimToken, + Instant now) { + return ledger.markFailed( new GrpcOperationIdentity( callerFingerprint, method, GrpcRequestFingerprint.hashIdempotencyKey(idempotencyKey)), + claimToken, now); } } diff --git a/src/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java b/src/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java index 90c34214..dd500825 100644 --- a/src/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java +++ b/src/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.grpc.idempotency; import static org.assertj.core.api.Assertions.assertThat; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger; import dev.caskeleton.grpc.core.GrpcMethodName; @@ -42,8 +43,11 @@ class GrpcCompletionReconcilerTest { GrpcCompletionReconciler reconciler = new GrpcCompletionReconciler(query); claim(); reconciler.reconcile(CREATE, CALLER, KEY, null); - ledger.reclaimStale(identity, fingerprint, afterTtl.minus(Duration.ofMinutes(5)), afterTtl); - ledger.markCommitted(identity, "outcome://document/42", afterTtl.plusSeconds(1)); + GrpcClaimToken takenOver = + ledger + .reclaimStale(identity, fingerprint, afterTtl.minus(Duration.ofMinutes(5)), afterTtl) + .orElseThrow(); + ledger.markCommitted(identity, takenOver, "outcome://document/42", afterTtl.plusSeconds(1)); // when GrpcCompletionResolution nextPass = reconciler.reconcile(CREATE, CALLER, KEY, null); @@ -93,6 +97,7 @@ class GrpcCompletionReconcilerTest { ledger.markCommitted( new dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity( CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY)), + GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); @@ -116,6 +121,7 @@ class GrpcCompletionReconcilerTest { ledger.markCommitted( new dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity( CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY)), + GrpcClaimToken.FIRST, "outcome://from-ledger", NOW.plusSeconds(1)); GrpcCompletionReconciler reconciler = new GrpcCompletionReconciler(query); diff --git a/src/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java b/src/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java index df26ea21..99fa0687 100644 --- a/src/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java +++ b/src/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java @@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.grpc.api.deadline.GrpcDeadlineProfile; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerState; @@ -98,8 +99,10 @@ class GrpcIdempotencyInterceptorTest { void aDuplicateOfACommittedOperationReplays() { GrpcIdempotencyInterceptor interceptor = interceptor(false); String fingerprint = GrpcRequestFingerprint.of("body"); - interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); - interceptor.recordCommit(CREATE, CALLER, KEY, "outcome://document/42", NOW.plusSeconds(1)); + GrpcClaimToken claim = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.recordCommit( + CREATE, CALLER, KEY, claim, "outcome://document/42", NOW.plusSeconds(1)); GrpcIdempotencyDecision replayed = interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2)); @@ -139,8 +142,9 @@ class GrpcIdempotencyInterceptorTest { void aTerminallyFailedOperationMayBeRetried() { GrpcIdempotencyInterceptor interceptor = interceptor(false); String fingerprint = GrpcRequestFingerprint.of("body"); - interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); - interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1)); + GrpcClaimToken claim = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.recordFailure(CREATE, CALLER, KEY, claim, NOW.plusSeconds(1)); assertThat(interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2)).outcome()) .isEqualTo(GrpcIdempotencyDecision.Outcome.PROCEED); @@ -151,8 +155,9 @@ class GrpcIdempotencyInterceptorTest { // given GrpcIdempotencyInterceptor interceptor = interceptor(false); String fingerprint = GrpcRequestFingerprint.of("body"); - interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); - interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1)); + GrpcClaimToken claim = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.recordFailure(CREATE, CALLER, KEY, claim, NOW.plusSeconds(1)); // when interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2)); @@ -172,8 +177,9 @@ class GrpcIdempotencyInterceptorTest { // given GrpcIdempotencyInterceptor interceptor = interceptor(false); String fingerprint = GrpcRequestFingerprint.of("body"); - interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); - interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1)); + GrpcClaimToken claim = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.recordFailure(CREATE, CALLER, KEY, claim, NOW.plusSeconds(1)); CountDownLatch start = new CountDownLatch(1); Callable retry = () -> { @@ -235,6 +241,70 @@ class GrpcIdempotencyInterceptorTest { .contains(AFTER_TTL); } + @Test + @DisplayName("claim 을 이긴 PROCEED 는 그 시도의 claim token 을 들고 있다") + void aProceedThatWonAClaimHoldsItsClaimToken() { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); + + // when + GrpcIdempotencyDecision takenOver = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL); + + // then + assertThat(takenOver.claimToken()).contains(GrpcClaimToken.FIRST.next()); + } + + /** + * The re-review probe's sequence through the interceptor (2026-11-03, RR-02-04): A is slow, B + * takes over after the TTL, A then fails. A's failure must not free the key for C while B runs. + */ + @Test + @DisplayName("TTL 뒤 넘겨받힌 옛 owner 의 실패 기록은 무시되어 다음 재시도는 진행 중으로 거부된다") + void aFailureRecordedByTheOwnerThatLostTheClaimDoesNotLetTheNextRetryRun() { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + GrpcClaimToken first = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL); + boolean recorded = + interceptor.recordFailure(CREATE, CALLER, KEY, first, AFTER_TTL.plusSeconds(1)); + + // when + GrpcIdempotencyDecision third = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL.plusSeconds(2)); + + // then + assertThat(recorded).isFalse(); + assertThat(third.outcome()).isEqualTo(GrpcIdempotencyDecision.Outcome.REJECT_IN_PROGRESS); + } + + @Test + @DisplayName("TTL 뒤 넘겨받힌 옛 owner 의 commit 은 거부되어 그 트랜잭션이 롤백된다") + void aCommitRecordedByTheOwnerThatLostTheClaimIsRefused() { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + GrpcClaimToken first = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL); + + // when + Throwable refused = + org.assertj.core.api.Assertions.catchThrowable( + () -> + interceptor.recordCommit( + CREATE, CALLER, KEY, first, "outcome://document/42", AFTER_TTL.plusSeconds(1))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("no longer held"); + } + @Test @DisplayName("TTL이 지나기 직전의 중복 요청은 여전히 대기하거나 거부된다") void aDuplicateAtTheTtlBoundaryIsStillTreatedAsRunning() { diff --git a/src/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetentionTest.java b/src/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetentionTest.java index fca364a6..817514af 100644 --- a/src/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetentionTest.java +++ b/src/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetentionTest.java @@ -3,6 +3,7 @@ package dev.caskeleton.grpc.idempotency; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger; import dev.caskeleton.grpc.core.GrpcMethodName; @@ -78,6 +79,7 @@ class GrpcOperationLedgerRetentionTest { // Finished at the claim moment; the completion moment is what the retention measures. private void committed(GrpcOperationIdentity identity, Instant claimedAt) { ledger.claim(identity, BODY, claimedAt); - ledger.markCommitted(identity, "outcome://document/" + identity.storageKey(), claimedAt); + ledger.markCommitted( + identity, GrpcClaimToken.FIRST, "outcome://document/" + identity.storageKey(), claimedAt); } } diff --git a/src/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidator.java b/src/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidator.java index a2892787..85a3ed85 100644 --- a/src/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidator.java +++ b/src/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidator.java @@ -24,6 +24,11 @@ import java.util.Set; * reflection in production publishes the schema, and a keyed method without a ledger accepts * idempotency keys it cannot honour. None of them fails a smoke test. * + *

    The keyed-method rule has a limit it states in its own message: with the ledger flag on, the + * starter assembles the decision function, and the server stage that calls it is the adopter's. A + * running process cannot see whether that stage exists, so passing this check with the flag on does + * not mean keys are honoured (re-review 2026-11-03, RR-02-07; ADR-GRPC-004). + * *

    Fails once with every violation, so a deployment learns the whole list in one restart. */ public final class GrpcPlatformStartupValidator { @@ -152,7 +157,10 @@ public final class GrpcPlatformStartupValidator { "method '" + method.canonical() + "' requires an idempotency key but app.grpc.platform.operation-ledger-enabled is" - + " false; the key would be accepted and never honoured"); + + " false; the key would be accepted and never honoured. Turning the flag on" + + " assembles the decision function only: the key is honoured once the server's" + + " IDEMPOTENCY stage interceptor or the service adapter calls decide/recordCommit," + + " which this starter does not wire"); } if (!policy.rpcType().stable()) { violations.add( diff --git a/src/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfigurationContextTest.java b/src/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfigurationContextTest.java index a07b65b2..b3fcbd21 100644 --- a/src/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfigurationContextTest.java +++ b/src/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfigurationContextTest.java @@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.grpc.api.deadline.GrpcDeadlineProfile; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger; @@ -196,7 +197,8 @@ class GrpcPlatformAutoConfigurationContextTest { GrpcOperationIdentity identity = new GrpcOperationIdentity(CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY)); ledger.claim(identity, GrpcRequestFingerprint.of("body"), NOW); - ledger.markCommitted(identity, "outcome://document/42", NOW.plusSeconds(1)); + ledger.markCommitted( + identity, GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); // when int purged = diff --git a/src/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidatorTest.java b/src/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidatorTest.java index dcffee91..d7307b1d 100644 --- a/src/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidatorTest.java +++ b/src/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidatorTest.java @@ -200,6 +200,44 @@ class GrpcPlatformStartupValidatorTest { .anySatisfy(violation -> assertThat(violation).contains("never honoured")); } + /** + * The flag assembles the decision function only; the starter builds no server, so nothing calls + * it until the adopter's IDEMPOTENCY stage or service adapter does. The refusal must not read as + * "turn the flag on and keys are honoured" (re-review 2026-11-03, RR-02-07). + */ + @Test + @DisplayName("키 필수 메서드 거부 문구는 플래그가 결정 함수만 조립하고 서버 단계 연결은 따로라고 말한다") + void theKeyedMethodRefusalSaysTheFlagAssemblesTheDecisionOnly() { + // given + GrpcPlatformProperties properties = productionProperties(); + properties.setOperationLedgerEnabled(false); + GrpcMethodPolicyCatalog catalog = + GrpcMethodPolicyCatalog.builder() + .register( + new GrpcMethodPolicy( + CREATE, + RpcType.UNARY, + RpcIdempotencyProfile.IDEMPOTENCY_KEY_REQUIRED, + GrpcDeadlineProfile.of(Duration.ofSeconds(2)), + WaitForReadyPolicy.DISABLED, + false, + 1024, + 1024)) + .build(); + + // when + var violations = + GrpcPlatformStartupValidator.violations(properties, catalog, oneChannel(), stableServer()); + + // then + assertThat(violations) + .anySatisfy( + violation -> + assertThat(violation) + .contains("assembles the decision function only") + .contains("IDEMPOTENCY stage")); + } + @Test @DisplayName("a client-streaming method is refused as an Advanced capability") void clientStreamingIsRefusedAsAdvanced() { diff --git a/src/grpc/grpc-testkit/build.gradle b/src/grpc/grpc-testkit/build.gradle index 3387707d..7dad3194 100644 --- a/src/grpc/grpc-testkit/build.gradle +++ b/src/grpc/grpc-testkit/build.gradle @@ -5,8 +5,9 @@ plugins { // Certification. The Stable plan splits this across four modules (core / in-process / netty / // fault); this repository already expresses "these two runs are not the same kind of evidence" with // strict test lanes rather than with module boundaries, so the four become four lanes over one -// leaf (adaptation design §2). A lane that discovers nothing fails, and none of them can serve an -// up-to-date result — which is the property the split was protecting. +// leaf (adaptation design §2). A lane that discovers nothing fails, and the three hermetic lanes +// reuse a result only when their classes and their lane policy are unchanged — the performance lane +// never does. That is the property the split was protecting. strictTestLanes { lane('grpcInProcessContractTest') { @@ -33,8 +34,8 @@ strictTestLanes { } // The performance lane is excluded from the default `test` task. It measures a running server -// under load, and a measurement in the release gate is a flaky test on a shared CI runner; it runs -// when somebody asks for it, by name. +// under load, so it is not in the pull-request or release gate; it runs in the optional build's +// `performanceCheck`, which integration-main.yml's performance-check job runs on main and nightly. tasks.named('test') { useJUnitPlatform { excludeTags 'grpc-performance' diff --git a/src/messaging/CLAUDE.md b/src/messaging/CLAUDE.md index 8c8d2acf..931c282e 100644 --- a/src/messaging/CLAUDE.md +++ b/src/messaging/CLAUDE.md @@ -40,6 +40,31 @@ JDBC이고 SQL은 `jsonb` · `FOR UPDATE SKIP LOCKED` · `ON CONFLICT` · `TIMES registry edge를 주면 그 경계가 깨진다. 복사본을 고칠 때(키 길이 검사, constant-time 비교 등)는 `shared-contract`의 정본과 같은 규칙인지 함께 본다. +### Publish retryability of authentication and authorization (review 2026-11-01 06-01) + +이 제품 안에는 같은 category에 대한 재시도 규칙이 둘 있다. 의도다. + +- category 기본값: `FailureDescriptor.defaultRetryable(AUTHENTICATION | AUTHORIZATION) = false`. 소비 쪽 + 재시도 엔진(`DefaultRetryDecisionEngine`)은 policy가 category를 따로 정하지 않으면 이 기본값과 + descriptor의 `retryable`을 둘 다 요구한다. 그래서 인가받지 못한 메시지의 재전달은 지금도 재시도하지 않는다. +- publish 거부의 `retryable`: `KafkaPublishFailureClassifier`는 인증·인가 거부에 `retryable = true`를 붙인다. + broker가 레코드를 저장하지 않았고, credential 교체나 application보다 늦게 배포된 ACL은 몇 분 동안 모든 + publish를 거부한 뒤 멈추기 때문이다. `retryable`을 믿는 publisher가 그 몇 분 동안의 레코드를 영구 보류하지 + 않게 하려는 것이다. + +06-01 수정이 이 제품에서 바꾼 것(모두 publish 경로, 소비 경로 동작은 그대로): + +- `KafkaPublishFailureClassifier`: 인증·인가 거부를 `retryable = true`로. +- `DefaultMessagePublisher`: runtime이 없을 때의 거부(`PUBLISH_RUNTIME_UNAVAILABLE`)를 `retryable = true`로. +- `KafkaMessagingTransport`: 닫힌 transport의 거부(`KAFKA_TRANSPORT_CLOSED`)를 `retryable = true`로. +- `MessagingShutdownLifecycle.PHASE`: platform 밖 relay의 순서 테스트가 참조하도록 public. + +버린 대안: vendored 제품은 그대로 두고 application bridge(ACL)가 category를 보고 "인증·인가면 재시도"로 +번역하는 것. 버린 이유: `retryable`은 이 제품이 "같은 레코드를 나중에 다시 보내면 받아질 수 있나"를 말하는 +자리다. bridge가 category로 그 판단을 다시 하면 같은 사실을 제품과 bridge가 따로 정하고, 이 제품의 +`PublishResult`를 읽는 다른 호출자는 bridge와 다른 답을 본다. 대신 이 제품 주석은 소비자 이름(application +relay, `composition:outbox-relay`)을 부르지 않는다 — "flag를 믿는 publisher"로 쓴다(재리뷰 2026-11-03 RR-05-07). + ## Public surface 정책 - **vendor 타입을 public signature에 노출하면 그 의존성은 `api`로 선언한다.** 숨긴다고 타입이 diff --git a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/FailureDescriptor.java b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/FailureDescriptor.java index 5c00b102..ff8dd47a 100644 --- a/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/FailureDescriptor.java +++ b/src/messaging/messaging-core-api/src/main/java/dev/caskeleton/messaging/api/error/FailureDescriptor.java @@ -61,6 +61,14 @@ public record FailureDescriptor( * automatically: each of them fails identically on every redelivery, so retrying only multiplies * the load while the message is still poison. * + *

    A default, not a rule every descriptor follows. A publish rejection may carry {@code + * retryable = true} with an authentication or authorization category: the broker did not store + * the record, and a credential being rotated or an ACL deployed after the application refuses + * every publish for a few minutes and then stops (the Kafka publish classifier). The + * consumer-side retry engine requires this default as well as the flag unless its policy names + * the category, so redelivery of an unauthorised message is still not retried; only a publisher + * that reads {@link #retryable()} sees the difference. + * * @param category the classification * @return true when automatic retry is allowed by default */ diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrar.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrar.java index 4b85f2fc..4e96018c 100644 --- a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrar.java +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrar.java @@ -344,10 +344,17 @@ public final class KafkaConsumerRegistrar implements TransportConsumerRegistrati // A decode failure is not a handler failure and must not be retried forever. The // offset may only advance once the record exists somewhere else, so an unavailable // quarantine leaves it uncommitted rather than dropping the message. - if (quarantine.quarantine(record, undecodable)) { + // It will never decode, however often it is redriven: DESERIALIZATION, not the + // POISON_MESSAGE of a handler that ran out of attempts (review 2026-11-01 06-14). + if (quarantine.quarantine( + record, + dev.caskeleton.messaging.api.error.FailureCategory.DESERIALIZATION, + undecodable)) { settlement.acknowledgeAfterQuarantine(); } else { - settlement.requeueAfterFailure(undecodable); + settlement.requeueAfterFailure( + undecodable, + dev.caskeleton.messaging.api.error.FailureCategory.DESERIALIZATION); } coordinator.release(partition); shutdown.endWork(); @@ -682,6 +689,7 @@ public final class KafkaConsumerRegistrar implements TransportConsumerRegistrati retryOrQuarantine( new IllegalStateException( "the handler asked for another attempt and the retry budget is spent"), + dev.caskeleton.messaging.api.error.FailureCategory.POISON_MESSAGE, delay)); } @@ -715,11 +723,21 @@ public final class KafkaConsumerRegistrar implements TransportConsumerRegistrati * that exists nowhere else — which is why a production destination must have one. */ private void requeueAfterFailure(Throwable reason) { + requeueAfterFailure(reason, exhaustedCategory(reason)); + } + + /** + * {@link #requeueAfterFailure(Throwable)} under a category the caller already knows: a record + * that could not be decoded stays {@code DESERIALIZATION} when its first quarantine write was + * not confirmed and it reaches quarantine again through the retry path. + */ + private void requeueAfterFailure( + Throwable reason, dev.caskeleton.messaging.api.error.FailureCategory category) { if (!terminal.compareAndSet(null, "requeue")) { // The handler settled before it failed or before its late completion; that answer stands. return; } - retryOrQuarantine(reason, Duration.ZERO); + retryOrQuarantine(reason, category, Duration.ZERO); } /** @@ -730,11 +748,13 @@ public final class KafkaConsumerRegistrar implements TransportConsumerRegistrati * being retried — the offset may not advance past a message that exists nowhere else. */ private dev.caskeleton.messaging.api.settlement.SettlementResult retryOrQuarantine( - Throwable reason, Duration requested) { + Throwable reason, + dev.caskeleton.messaging.api.error.FailureCategory category, + Duration requested) { FailedOffset key = new FailedOffset(partition, offset); int attempt = failures.merge(key, 1, Integer::sum); dev.caskeleton.messaging.policy.RetryPolicy retry = spec.profile().retry(); - if (attempt >= retry.maxAttempts() && quarantine.quarantine(record, reason)) { + if (attempt >= retry.maxAttempts() && quarantine.quarantine(record, category, reason)) { failures.remove(key); settlements.enqueue( new KafkaSettlementCommand( @@ -751,6 +771,19 @@ public final class KafkaConsumerRegistrar implements TransportConsumerRegistrati return enqueueRequeue(backoff); } + /** + * The category a record that ran out of attempts is quarantined under: its last failure's own, + * when that failure is a platform one and so carries a classification — a transient one tells a + * redrive the record may pass now — and {@code POISON_MESSAGE}, the platform's name for "keeps + * failing its consumer", when it is not. + */ + private static dev.caskeleton.messaging.api.error.FailureCategory exhaustedCategory( + Throwable reason) { + return reason instanceof dev.caskeleton.messaging.api.error.MessagingException classified + ? classified.category() + : dev.caskeleton.messaging.api.error.FailureCategory.POISON_MESSAGE; + } + private dev.caskeleton.messaging.api.settlement.SettlementResult enqueueRequeue( Duration delay) { retries.pauseUntil(partition, offset, delay, clock.instant()); diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifier.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifier.java index 0eea955a..d6569fdc 100644 --- a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifier.java +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifier.java @@ -67,10 +67,14 @@ public final class KafkaPublishFailureClassifier { *

    Authentication and authorization prove the broker did not store the record, but not that it * will refuse it again: a credential being rotated, or an ACL deployed after the application, * fails every publish this way for a few minutes and then stops. Marked not retryable, a - * publisher that trusts the flag — the application's outbox relay does — parks every record sent - * in those minutes. A credential that is simply wrong is retried too, and the outbox then holds - * its rows FAILED with an alert every retry — recoverable once the credential is fixed, where - * DEAD rows needed a manual reset each. + * publisher that trusts the flag parks every record sent in those minutes. A credential that is + * simply wrong is retried too, for as long as it stays wrong; the descriptor's code names it, so + * a caller can tell that retry from a rotation and stop it. + * + *

    This flag departs from {@link FailureDescriptor#defaultRetryable} on purpose, for publish + * only: that default says an authentication or authorization failure is not retried, and it still + * governs consumption, where the retry engine requires both. See {@code src/messaging/CLAUDE.md} + * "Publish retryability of authentication and authorization". */ private static boolean mayPassOnRetry(Throwable cause) { return cause instanceof AuthenticationException || cause instanceof AuthorizationException; diff --git a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaQuarantinePublisher.java b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaQuarantinePublisher.java index 20489c95..af09f0e8 100644 --- a/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaQuarantinePublisher.java +++ b/src/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaQuarantinePublisher.java @@ -1,5 +1,6 @@ package dev.caskeleton.messaging.kafka; +import dev.caskeleton.messaging.api.error.FailureCategory; import org.apache.kafka.clients.consumer.ConsumerRecord; /** @@ -18,11 +19,22 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; public interface KafkaQuarantinePublisher { /** - * Writes an undecodable record somewhere durable. + * Writes a record the consumer will not deliver somewhere durable, under the reason it gives. * + *

    The category is the caller's, because only the caller knows which of two very different + * things happened: the record could not be rebuilt at all ({@link + * FailureCategory#DESERIALIZATION}), or its handler failed until the attempt budget ran out — + * under the failure's own category when it carries one, {@link FailureCategory#POISON_MESSAGE} + * when it does not. Both were written as {@code POISON_MESSAGE}, so a redrive could not tell a + * message that will never decode from one an outage kept failing (review 2026-11-01 06-14). + * + * @param record the record as it arrived + * @param category why it is quarantined, written as the failure-category header + * @param reason the failure; its type is written as the failure-code header * @return true when the write is confirmed, so the source offset may be committed */ - boolean quarantine(ConsumerRecord record, Throwable reason); + boolean quarantine( + ConsumerRecord record, FailureCategory category, Throwable reason); /** * The default: nothing is configured, so nothing may be committed past. @@ -37,7 +49,7 @@ public interface KafkaQuarantinePublisher { /** Holds the single "not configured" quarantine. */ final class Unavailable { - private static final KafkaQuarantinePublisher INSTANCE = (record, reason) -> false; + private static final KafkaQuarantinePublisher INSTANCE = (record, category, reason) -> false; private Unavailable() {} } diff --git a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrarTest.java b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrarTest.java index f5580a52..edd77699 100644 --- a/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrarTest.java +++ b/src/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaConsumerRegistrarTest.java @@ -3,6 +3,7 @@ package dev.caskeleton.messaging.kafka; import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.messaging.api.MessageId; +import dev.caskeleton.messaging.api.error.FailureCategory; import dev.caskeleton.messaging.api.header.ReservedHeaders; import dev.caskeleton.messaging.policy.OrderingImpact; import dev.caskeleton.messaging.policy.RetryMode; @@ -258,7 +259,7 @@ class KafkaConsumerRegistrarTest { registrar( KafkaContractHarness.profile(), this::fail, - (record, reason) -> quarantined.add(record.offset()), + (record, category, reason) -> quarantined.add(record.offset()), () -> NOW); assign(); addRecord(10); @@ -274,6 +275,29 @@ class KafkaConsumerRegistrarTest { .hasValue(11L); } + @Test + @DisplayName("handler가 시도 예산을 다 쓴 레코드는 그 실패의 분류로 격리하고 분류가 없으면 POISON_MESSAGE로 적는다") + void aDeliveryThatSpentItsBudgetIsQuarantinedUnderItsFailuresCategory() { + // given: a handler that fails with a plain exception every time + List categories = new ArrayList<>(); + KafkaConsumerRegistrar registrar = + registrar( + KafkaContractHarness.profile(), + this::fail, + (record, category, reason) -> categories.add(category), + () -> NOW); + assign(); + addRecord(10); + registrar.pollOnce(NOW); + handlers.runAll(); + + // when + registrar.pollOnce(NOW); + + // then + assertThat(categories).containsExactly(FailureCategory.POISON_MESSAGE); + } + @Test @DisplayName("handler가 요청한 requeue도 시도 횟수를 세어 예산을 쓰면 quarantine 후 commit된다") void aDeliveryItsHandlerKeepsRequeueingSpendsTheBudgetAndIsQuarantined() { @@ -283,7 +307,7 @@ class KafkaConsumerRegistrarTest { registrar( KafkaContractHarness.profile(), this::requeue, - (record, reason) -> quarantined.add(record.offset()), + (record, category, reason) -> quarantined.add(record.offset()), () -> NOW); assign(); addRecord(10); @@ -470,7 +494,7 @@ class KafkaConsumerRegistrarTest { handlers, new GracefulShutdownCoordinator(Duration.ofSeconds(30)), Duration.ofMillis(1), - (record, reason) -> { + (record, category, reason) -> { quarantined.add(record.offset()); return true; })) { @@ -486,6 +510,65 @@ class KafkaConsumerRegistrarTest { } } + /** + * A record that cannot be rebuilt into an envelope will never decode, however often it is + * redriven; the one whose handler ran out of attempts may pass once the outage is over. They were + * both written as {@code POISON_MESSAGE} (review 2026-11-01 06-14). + */ + @Test + @DisplayName("envelope로 되살릴 수 없는 레코드는 시도 예산 소진과 다른 DESERIALIZATION으로 격리한다") + void anUndecodableRecordIsQuarantinedAsDeserializationNotAsPoison() { + // given + List categories = new ArrayList<>(); + try (KafkaConsumerRegistrar registrar = + new KafkaConsumerRegistrar( + new TransportConsumerSpec(KafkaContractHarness.profile(), this::settleImmediately), + consumer, + handlers, + new GracefulShutdownCoordinator(Duration.ofSeconds(30)), + Duration.ofMillis(1), + (record, category, reason) -> categories.add(category))) { + assign(); + addRecordWithoutIdentityHeaders(10); + registrar.pollOnce(NOW); + + // when + handlers.runAll(); + + // then + assertThat(categories).containsExactly(FailureCategory.DESERIALIZATION); + } + } + + @Test + @DisplayName("격리 쓰기가 한 번 실패해 재시도로 넘어간 읽을 수 없는 레코드도 DESERIALIZATION으로 격리한다") + void anUndecodableRecordStaysDeserializationWhenItsQuarantineIsRetried() { + // given: the first quarantine write is not confirmed, so the record goes round the retry path + List categories = new ArrayList<>(); + try (KafkaConsumerRegistrar registrar = + new KafkaConsumerRegistrar( + new TransportConsumerSpec(KafkaContractHarness.profile(), this::settleImmediately), + consumer, + handlers, + new GracefulShutdownCoordinator(Duration.ofSeconds(30)), + Duration.ofMillis(1), + (record, category, reason) -> { + categories.add(category); + return categories.size() > 1; + })) { + assign(); + addRecordWithoutIdentityHeaders(10); + registrar.pollOnce(NOW); + + // when + handlers.runAll(); + + // then + assertThat(categories) + .containsExactly(FailureCategory.DESERIALIZATION, FailureCategory.DESERIALIZATION); + } + } + @Test @DisplayName("one partition hitting its ceiling does not abandon the other's records") void oneSaturatedPartitionDoesNotAbandonAnother() { diff --git a/src/messaging/messaging-runtime-core/src/main/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisher.java b/src/messaging/messaging-runtime-core/src/main/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisher.java index a51f9fef..5c4341e3 100644 --- a/src/messaging/messaging-runtime-core/src/main/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisher.java +++ b/src/messaging/messaging-runtime-core/src/main/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisher.java @@ -265,7 +265,7 @@ public final class DefaultMessagePublisher implements MessagePublisher, EncodedM admission.complete(destination.name().value()); // No runtime to send through — the registry closed during shutdown, or none is installed // yet. Nothing about the message was wrong, so the refusal is marked retryable: a caller that - // trusts the flag, as the application's outbox relay does, retries it instead of parking it. + // trusts the flag retries it instead of parking it. return refused( new FailureDescriptor( FailureCategory.TRANSIENT_INFRASTRUCTURE, diff --git a/src/messaging/messaging-spring-boot-starter-kafka/src/main/java/dev/caskeleton/messaging/autoconfigure/KafkaDeadLetterQuarantine.java b/src/messaging/messaging-spring-boot-starter-kafka/src/main/java/dev/caskeleton/messaging/autoconfigure/KafkaDeadLetterQuarantine.java index 8185ef6c..85ad9c2e 100644 --- a/src/messaging/messaging-spring-boot-starter-kafka/src/main/java/dev/caskeleton/messaging/autoconfigure/KafkaDeadLetterQuarantine.java +++ b/src/messaging/messaging-spring-boot-starter-kafka/src/main/java/dev/caskeleton/messaging/autoconfigure/KafkaDeadLetterQuarantine.java @@ -1,5 +1,6 @@ package dev.caskeleton.messaging.autoconfigure; +import dev.caskeleton.messaging.api.error.FailureCategory; import dev.caskeleton.messaging.api.header.ReservedHeaders; import dev.caskeleton.messaging.kafka.KafkaQuarantinePublisher; import java.nio.charset.StandardCharsets; @@ -38,11 +39,14 @@ final class KafkaDeadLetterQuarantine implements KafkaQuarantinePublisher { } @Override - public boolean quarantine(ConsumerRecord record, Throwable reason) { + public boolean quarantine( + ConsumerRecord record, FailureCategory category, Throwable reason) { RecordHeaders headers = new RecordHeaders(record.headers().toArray()); headers.remove(ReservedHeaders.FAILURE_CATEGORY); headers.remove(ReservedHeaders.FAILURE_CODE); - headers.add(ReservedHeaders.FAILURE_CATEGORY, bytes("POISON_MESSAGE")); + headers.add( + ReservedHeaders.FAILURE_CATEGORY, + bytes(category == null ? FailureCategory.POISON_MESSAGE.name() : category.name())); headers.add( ReservedHeaders.FAILURE_CODE, bytes(reason == null ? "UNKNOWN" : reason.getClass().getSimpleName())); diff --git a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingShutdownLifecycle.java b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingShutdownLifecycle.java index 7efaa471..147c2661 100644 --- a/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingShutdownLifecycle.java +++ b/src/messaging/messaging-spring-boot-starter/src/main/java/dev/caskeleton/messaging/autoconfigure/MessagingShutdownLifecycle.java @@ -29,14 +29,14 @@ import org.springframework.context.SmartLifecycle; */ public final class MessagingShutdownLifecycle implements SmartLifecycle { - /** How often the drain is re-checked. Short enough that a fast drain is not padded. */ /** - * The phase this drain stops in: {@link #getPhase()}. Public so a relay outside the platform — - * the application's own outbox relay, which may not name this type in production code — can be - * held above it by a test that does name it. + * The phase this drain stops in: {@link #getPhase()}. Public so a relay outside the platform — a + * publisher whose production code may not name this type — can be held above it by a test that + * does name it. */ public static final int PHASE = Integer.MAX_VALUE - 1024; + /** How often the drain is re-checked. Short enough that a fast drain is not padded. */ private static final Duration POLL = Duration.ofMillis(50); private final MessagingAdmissionController admission; @@ -120,11 +120,10 @@ public final class MessagingShutdownLifecycle implements SmartLifecycle { * drain ahead of the transports it is draining, because closing a connection first would strand * the settlements the drain exists to flush. It sits below the outbox relays' phase for the * mirror-image reason: refusing admission while a relay pass is mid-publish turns that pass's - * rows into refusals. Two relays stand above it: the platform's own ({@code - * MessagingOutboxRelayLifecycle}, built only when a platform outbox repository exists) and the - * application's ({@code composition:outbox-relay} {@code OutboxRelayScheduler}, phase {@code - * Integer.MAX_VALUE}), which finishes the publish it is in and releases its other claims before - * this phase runs. + * rows into refusals. The platform's own relay ({@code MessagingOutboxRelayLifecycle}, built only + * when a platform outbox repository exists) stands above it, and so must any relay outside the + * platform that publishes through it: a phase above {@link #PHASE}, finishing the publish it is + * in before this phase runs. * * @return the lifecycle phase, {@link #PHASE} */ diff --git a/src/optional-platforms/build.gradle b/src/optional-platforms/build.gradle index 59808a93..1c5ff1b0 100644 --- a/src/optional-platforms/build.gradle +++ b/src/optional-platforms/build.gradle @@ -1,5 +1,9 @@ plugins { id 'ca.architecture' + // Every strict lane of this build belongs to a stage a workflow executes, checked here exactly + // as in the main build: the main build's check cannot see this build's lanes, and while only it + // applied the plugin, grpcPerformanceTest ran nowhere (review 2026-11-03 RR-06-03). + id 'ca.strict-lane-stages' } // No `ca.release-provenance` any more: each gRPC leaf's `ca.java-conventions` derives its version @@ -47,3 +51,33 @@ tasks.register('integrationCheck') { description = 'Runs every optional-platform leaf integration test lane. Needs a container runtime.' dependsOn subprojects.findAll { it.childProjects.isEmpty() }.collect { "${it.path}:integrationTestLaneCheck" } } + +// The performance stage of this build: grpcPerformanceTest. It asserts the shape of a measurement +// (ordered percentiles, a gate that reads them) with a generous budget, not a number, so it can run +// on a shared runner. integration-main.yml's performance-check job runs it next to the main build's +// `:performanceCheck`; before this aggregate it ran when somebody asked for it by name, which was +// nobody (review 2026-11-03 RR-06-03). +tasks.register('performanceCheck') { + group = 'verification' + description = 'Runs every optional-platform leaf performance test lane.' + dependsOn subprojects.findAll { it.childProjects.isEmpty() }.collect { "${it.path}:performanceTestLaneCheck" } +} + +// The tag-only stage of this build: the hermetic strict lanes (the three gRPC certification lanes). +// Their tests already run inside each leaf's `test`; the lanes add the one thing `test` cannot say — +// that a tag still selects something — which is worth asserting once per release tag. release.yml +// runs this aggregate instead of naming the lanes, so a lane added here is in the release gate +// without a workflow edit. +tasks.register('releaseCheck') { + group = 'verification' + description = 'Runs every optional-platform leaf hermetic strict test lane (the release gate).' + dependsOn subprojects.findAll { it.childProjects.isEmpty() }.collect { "${it.path}:testLaneCheck" } +} + +// The stages this build's strict lanes may belong to, executed by workflows as +// `./gradlew -p optional-platforms ` from `src`. +strictLaneStages { + stage ':ci', ':integrationCheck', ':performanceCheck', ':releaseCheck' + workflowDirectory = layout.projectDirectory.dir('../../.github/workflows') + workflowProjectDirectory = 'optional-platforms' +} diff --git a/src/sample-portfolio/README.md b/src/sample-portfolio/README.md index fd427c76..eb63c231 100644 --- a/src/sample-portfolio/README.md +++ b/src/sample-portfolio/README.md @@ -48,8 +48,13 @@ 비밀번호가 문서에 적혀 있는 공개 데모 계정입니다. 비밀이 아니며, 실제 서비스는 이 방식을 쓰지 않고 `adapter:inbound:web`의 JWT 체인으로 인증합니다. -그래서 `prod` 프로필로 띄우면 기동을 거부합니다(`SampleDemoAccountsProductionGuard`, 종료 코드 71). -샘플을 복제해 운영에 올릴 때 이 계정이 남아 있으면 누구나 관리자로 들어올 수 있기 때문입니다. 운영으로 +그래서 데모 프로필(`local`·`test`·`demo`)이 명시적으로 켜져 있고 그 밖의 프로필이 하나도 없을 때만 +기동합니다. 프로필 없음, `prod`, `production`, `staging` 등은 기동을 거부합니다 +(`SampleDemoAccountsProductionGuard`, 종료 코드 71). 샘플을 복제해 운영에 올릴 때 이 계정이 남아 있으면 +누구나 관리자로 들어올 수 있기 때문입니다. 거부 목록(`prod`)이 아니라 허용 목록인 이유는 프로필 이름이 +하나만 달라도 계정이 열리지 않게 하려는 것입니다. `./gradlew :sample-portfolio:bootRun`은 +`SPRING_PROFILES_ACTIVE`가 비어 있으면 `local`을 넘기고, jar·이미지로 띄울 때는 +`SPRING_PROFILES_ACTIVE=local`(또는 `test`·`demo`)을 직접 줍니다. 운영으로 가져가려면 `SampleDemoSecurityConfig`·`SampleDemoAccounts`를 지우고 `SamplePortfolioApplication`에서 `WebSecurityAutoConfiguration` 제외를 풉니다. 이 규칙은 데모 설정 옆에 선언되어 있어 함께 지워집니다. diff --git a/src/sample-portfolio/build.gradle b/src/sample-portfolio/build.gradle index d1b079e4..a494b00e 100644 --- a/src/sample-portfolio/build.gradle +++ b/src/sample-portfolio/build.gradle @@ -196,7 +196,6 @@ strictTestLanes { requires('dev.caskeleton.sample.portfolio.adapter.inbound.web.contract.OpenApiDriftContractTest') systemProperty 'openapi.snapshot.write', project.hasProperty('approveOpenApiChange') ? 'true' : 'false' - // Pin UTC like the main test task for host-locale independence. } } @@ -228,7 +227,8 @@ strictTestLanes { lane('messagingSampleContractQualificationTest') { sourceSet = 'test' description = 'Runs exact Messaging sample contract qualification tests.' - requires('dev.caskeleton.sample.portfolio.application.worklog.WorkLogReservedContractContributionTest') + requires('dev.caskeleton.sample.portfolio.application.worklog.WorkLogReservedContractContributionTest', + 'dev.caskeleton.sample.portfolio.application.event.SamplePayloadSchemaReleaseTest') rejectSkipped = true } } @@ -245,6 +245,11 @@ tasks.register('stageDockerJar', Sync) { } // Run from the repo's src/ root so src/.env is picked up (same as app-bootstrap). +// bootRun is the local developer command, so it says `local` explicitly unless SPRING_PROFILES_ACTIVE +// names a profile: the demo accounts refuse to start without a demo profile, and application.yml no +// longer defaults one (RR-07-02). The jar and the image get no such default. bootRun { workingDir = rootProject.projectDir + systemProperty 'spring.profiles.active', + providers.environmentVariable('SPRING_PROFILES_ACTIVE').filter { !it.isBlank() }.orElse('local').get() } diff --git a/src/sample-portfolio/src/integrationTest/java/dev/caskeleton/sample/portfolio/adapter/outbound/persistence/repository/PosterRepositoryAdapterContractTest.java b/src/sample-portfolio/src/integrationTest/java/dev/caskeleton/sample/portfolio/adapter/outbound/persistence/repository/PosterRepositoryAdapterContractTest.java new file mode 100644 index 00000000..efda28c1 --- /dev/null +++ b/src/sample-portfolio/src/integrationTest/java/dev/caskeleton/sample/portfolio/adapter/outbound/persistence/repository/PosterRepositoryAdapterContractTest.java @@ -0,0 +1,71 @@ +package dev.caskeleton.sample.portfolio.adapter.outbound.persistence.repository; + +import dev.caskeleton.sample.portfolio.adapter.outbound.persistence.config.JpaConfig; +import dev.caskeleton.sample.portfolio.domain.poster.PosterRepository; +import dev.caskeleton.sample.portfolio.domain.poster.testkit.PosterRepositoryContract; +import dev.caskeleton.sample.portfolio.testinfra.PostgreSqlTestContainer; +import java.util.function.Supplier; +import org.junit.jupiter.api.BeforeEach; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest; +import org.springframework.boot.jdbc.test.autoconfigure.AutoConfigureTestDatabase; +import org.springframework.boot.testcontainers.service.connection.ServiceConnection; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.TestPropertySource; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionTemplate; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.postgresql.PostgreSQLContainer; + +/** + * The shipped {@link PosterRepositoryAdapter} against real PostgreSQL, held to the same {@link + * PosterRepositoryContract} as the in-memory fake. The test-managed transaction is switched off so + * every contract call commits on its own: the title rule is the unique index's, and it answers at + * the statement the adapter flushes. + */ +@DataJpaTest +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@Testcontainers(disabledWithoutDocker = true) +@Transactional(propagation = Propagation.NOT_SUPPORTED) +@Import({ + JpaConfig.class, + PosterRepositoryAdapter.class, + PosterRepositoryAdapterIntegrationTest.TestAuditConfig.class +}) +@TestPropertySource( + properties = { + "spring.flyway.enabled=true", + "spring.flyway.locations=classpath:db/migration/postgresql,classpath:db/sample-migration", + "spring.jpa.hibernate.ddl-auto=validate" + }) +class PosterRepositoryAdapterContractTest extends PosterRepositoryContract { + + @Container @ServiceConnection + static final PostgreSQLContainer POSTGRES = PostgreSqlTestContainer.create(); + + @Autowired private PosterRepositoryAdapter adapter; + @Autowired private PosterJpaRepository jpa; + @Autowired private PlatformTransactionManager transactionManager; + + @BeforeEach + void empty() { + inTransaction( + () -> { + jpa.deleteAll(); + return null; + }); + } + + @Override + protected PosterRepository repository() { + return adapter; + } + + @Override + protected T inTransaction(Supplier work) { + return new TransactionTemplate(transactionManager).execute(status -> work.get()); + } +} diff --git a/src/sample-portfolio/src/integrationTest/java/dev/caskeleton/sample/portfolio/adapter/outbound/persistence/repository/PosterRepositoryAdapterIntegrationTest.java b/src/sample-portfolio/src/integrationTest/java/dev/caskeleton/sample/portfolio/adapter/outbound/persistence/repository/PosterRepositoryAdapterIntegrationTest.java index fae67951..9f7f9628 100644 --- a/src/sample-portfolio/src/integrationTest/java/dev/caskeleton/sample/portfolio/adapter/outbound/persistence/repository/PosterRepositoryAdapterIntegrationTest.java +++ b/src/sample-portfolio/src/integrationTest/java/dev/caskeleton/sample/portfolio/adapter/outbound/persistence/repository/PosterRepositoryAdapterIntegrationTest.java @@ -5,13 +5,13 @@ import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.persistence.audit.AuditContextPort; import dev.caskeleton.sample.portfolio.adapter.outbound.persistence.config.JpaConfig; -import dev.caskeleton.sample.portfolio.application.poster.PosterTitleAlreadyExistsException; import dev.caskeleton.sample.portfolio.domain.poster.Poster; import dev.caskeleton.sample.portfolio.domain.poster.PosterId; import dev.caskeleton.sample.portfolio.domain.poster.PosterImageReference; import dev.caskeleton.sample.portfolio.domain.poster.PosterPage; import dev.caskeleton.sample.portfolio.domain.poster.PosterSortField; import dev.caskeleton.sample.portfolio.domain.poster.PosterStatus; +import dev.caskeleton.sample.portfolio.domain.poster.PosterTitleAlreadyExistsException; import dev.caskeleton.sample.portfolio.domain.poster.PublishedPosterImageAttachment; import dev.caskeleton.sample.portfolio.testinfra.PostgreSqlTestContainer; import java.time.Clock; diff --git a/src/sample-portfolio/src/integrationTest/java/dev/caskeleton/sample/portfolio/application/poster/CreatePosterConcurrencyIntegrationTest.java b/src/sample-portfolio/src/integrationTest/java/dev/caskeleton/sample/portfolio/application/poster/CreatePosterConcurrencyIntegrationTest.java index d31647f0..26fd857e 100644 --- a/src/sample-portfolio/src/integrationTest/java/dev/caskeleton/sample/portfolio/application/poster/CreatePosterConcurrencyIntegrationTest.java +++ b/src/sample-portfolio/src/integrationTest/java/dev/caskeleton/sample/portfolio/application/poster/CreatePosterConcurrencyIntegrationTest.java @@ -7,10 +7,12 @@ import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction import dev.caskeleton.application.messaging.event.testkit.FakeIntegrationEventEncoder; import dev.caskeleton.application.outbox.testkit.InMemoryOutboxAggregateSequences; import dev.caskeleton.application.outbox.testkit.InMemoryOutboxStore; +import dev.caskeleton.sample.portfolio.adapter.outbound.identifier.UuidOutboxEventIdFactory; import dev.caskeleton.sample.portfolio.adapter.outbound.identifier.UuidPosterIdFactory; import dev.caskeleton.sample.portfolio.adapter.outbound.persistence.config.JpaConfig; import dev.caskeleton.sample.portfolio.adapter.outbound.persistence.repository.PosterRepositoryAdapter; import dev.caskeleton.sample.portfolio.application.event.PortfolioEventOutbox; +import dev.caskeleton.sample.portfolio.domain.poster.PosterTitleAlreadyExistsException; import dev.caskeleton.sample.portfolio.testinfra.PostgreSqlTestContainer; import java.time.Clock; import java.util.ArrayList; @@ -150,7 +152,7 @@ class CreatePosterConcurrencyIntegrationTest { return encoder.encode(draft); } }, - () -> UUID.randomUUID().toString(), + new UuidOutboxEventIdFactory(), Optional::empty, Clock.systemUTC(), new InMemoryOutboxAggregateSequences()); diff --git a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/error/DomainExceptionHandler.java b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/error/DomainExceptionHandler.java index ffb4f4ce..42dc6adf 100644 --- a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/error/DomainExceptionHandler.java +++ b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/error/DomainExceptionHandler.java @@ -2,12 +2,12 @@ package dev.caskeleton.sample.portfolio.adapter.inbound.web.error; import dev.caskeleton.adapter.inbound.web.error.ErrorResponseFactory; import dev.caskeleton.sample.portfolio.application.poster.PosterNotFoundException; -import dev.caskeleton.sample.portfolio.application.poster.PosterTitleAlreadyExistsException; import dev.caskeleton.sample.portfolio.application.posterimage.PosterImageAlreadyPublishedException; import dev.caskeleton.sample.portfolio.application.worklog.WorkLogNotFoundException; import dev.caskeleton.sample.portfolio.application.worklog.WorkLogNotOwnedException; import dev.caskeleton.sample.portfolio.application.worklog.WorkLogVersionMismatchException; import dev.caskeleton.sample.portfolio.domain.poster.PosterInvariantException; +import dev.caskeleton.sample.portfolio.domain.poster.PosterTitleAlreadyExistsException; import dev.caskeleton.sample.portfolio.domain.worklog.WorkLogInvariantException; import dev.caskeleton.shared.error.OperationalError; import dev.caskeleton.shared.response.Envelope; diff --git a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/adapter/outbound/persistence/repository/PosterRepositoryAdapter.java b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/adapter/outbound/persistence/repository/PosterRepositoryAdapter.java index 2b42d19c..4f62d964 100644 --- a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/adapter/outbound/persistence/repository/PosterRepositoryAdapter.java +++ b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/adapter/outbound/persistence/repository/PosterRepositoryAdapter.java @@ -3,13 +3,13 @@ package dev.caskeleton.sample.portfolio.adapter.outbound.persistence.repository; import dev.caskeleton.adapter.outbound.persistence.audit.AuditContextPort; import dev.caskeleton.sample.portfolio.adapter.outbound.persistence.entity.PosterEntity; import dev.caskeleton.sample.portfolio.adapter.outbound.persistence.mapper.PosterPersistenceMapper; -import dev.caskeleton.sample.portfolio.application.poster.PosterTitleAlreadyExistsException; import dev.caskeleton.sample.portfolio.domain.poster.Poster; import dev.caskeleton.sample.portfolio.domain.poster.PosterId; import dev.caskeleton.sample.portfolio.domain.poster.PosterPage; import dev.caskeleton.sample.portfolio.domain.poster.PosterRepository; import dev.caskeleton.sample.portfolio.domain.poster.PosterSortField; import dev.caskeleton.sample.portfolio.domain.poster.PosterStatus; +import dev.caskeleton.sample.portfolio.domain.poster.PosterTitleAlreadyExistsException; import java.time.Clock; import java.time.Instant; import java.util.List; @@ -66,8 +66,8 @@ public class PosterRepositoryAdapter implements PosterRepository { * The unique index {@value #TITLE_UNIQUE_INDEX} (sample migration V8.1) is what keeps poster * titles unique; the use case's {@code existsByTitle} check only gives the common case an early * answer and loses races. Flushing here makes a violation surface inside this call, where it is - * turned into the same {@link PosterTitleAlreadyExistsException} the check throws, instead of at - * commit as a generic unique-violation failure. + * turned into the domain's {@link PosterTitleAlreadyExistsException} that the port declares, + * instead of at commit as a generic unique-violation failure. */ private PosterEntity saveHoldingTheTitleRule(PosterEntity entity, String title) { try { diff --git a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/poster/CreatePosterUseCase.java b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/poster/CreatePosterUseCase.java index 9fd982cb..53650119 100644 --- a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/poster/CreatePosterUseCase.java +++ b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/poster/CreatePosterUseCase.java @@ -11,6 +11,7 @@ import dev.caskeleton.sample.portfolio.domain.poster.Poster; import dev.caskeleton.sample.portfolio.domain.poster.PosterCreated; import dev.caskeleton.sample.portfolio.domain.poster.PosterIdFactory; import dev.caskeleton.sample.portfolio.domain.poster.PosterRepository; +import dev.caskeleton.sample.portfolio.domain.poster.PosterTitleAlreadyExistsException; /** * Creates a new {@link Poster} (server-assigned id via {@link PosterIdFactory}) and appends a diff --git a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/poster/PosterTitleAlreadyExistsException.java b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/poster/PosterTitleAlreadyExistsException.java deleted file mode 100644 index d697beec..00000000 --- a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/poster/PosterTitleAlreadyExistsException.java +++ /dev/null @@ -1,11 +0,0 @@ -package dev.caskeleton.sample.portfolio.application.poster; - -/** Raised when creating a poster whose title is already taken. Web handler maps it to 409. */ -public class PosterTitleAlreadyExistsException extends RuntimeException { - - private static final long serialVersionUID = 1L; - - public PosterTitleAlreadyExistsException(String title) { - super("Poster title already in use: " + title); - } -} diff --git a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/poster/UpdatePosterUseCase.java b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/poster/UpdatePosterUseCase.java index d2bf7cba..2e001d50 100644 --- a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/poster/UpdatePosterUseCase.java +++ b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/poster/UpdatePosterUseCase.java @@ -9,6 +9,7 @@ import dev.caskeleton.application.transaction.TransactionPort; import dev.caskeleton.application.usecase.CommandUseCase; import dev.caskeleton.sample.portfolio.domain.poster.Poster; import dev.caskeleton.sample.portfolio.domain.poster.PosterRepository; +import dev.caskeleton.sample.portfolio.domain.poster.PosterTitleAlreadyExistsException; /** * Partial update of a poster's title/caption using {@code Patch} 3-state semantics. Mirrors {@code diff --git a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/worklog/WorkLogReservedPayload.java b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/worklog/WorkLogReservedPayload.java index e02c25e7..b3bd0f26 100644 --- a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/worklog/WorkLogReservedPayload.java +++ b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/application/worklog/WorkLogReservedPayload.java @@ -1,17 +1,18 @@ package dev.caskeleton.sample.portfolio.application.worklog; import dev.caskeleton.application.messaging.contract.IntegrationPayload; +import dev.caskeleton.application.messaging.event.IntegrationIdentity; -/** Immutable sample payload for the {@code portfolio.worklog.reserved} v1 contract. */ +/** + * Immutable sample payload for the {@code portfolio.worklog.reserved} v1 contract. + * + *

    The work log id is the event's aggregate id, so it takes the aggregate id grammar and bound + * from {@link IntegrationIdentity} rather than a copy of them (review 2026-11-03 RR-01-08). + */ public record WorkLogReservedPayload(String workLogId) implements IntegrationPayload { - private static final int MAXIMUM_WORK_LOG_ID_LENGTH = 160; - private static final String WORK_LOG_ID_GRAMMAR = "[A-Za-z0-9][A-Za-z0-9._:-]*"; - public WorkLogReservedPayload { - if (workLogId == null - || workLogId.length() > MAXIMUM_WORK_LOG_ID_LENGTH - || !workLogId.matches(WORK_LOG_ID_GRAMMAR)) { + if (!IntegrationIdentity.isCanonical(workLogId, IntegrationIdentity.AGGREGATE_ID_MAX_LENGTH)) { throw new IllegalArgumentException( "workLogId must be a 1-160 character canonical US-ASCII identifier"); } diff --git a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/bootstrap/management/SampleManagementSecurityConfig.java b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/bootstrap/management/SampleManagementSecurityConfig.java index 2905046e..9a0c29f7 100644 --- a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/bootstrap/management/SampleManagementSecurityConfig.java +++ b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/bootstrap/management/SampleManagementSecurityConfig.java @@ -21,7 +21,12 @@ import org.springframework.security.web.SecurityFilterChain; @ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.SERVLET) public class SampleManagementSecurityConfig { - /** Actuator security chain ordered at {@code 0} to run BEFORE the main app chain. */ + /** + * The actuator chain. Its order ({@code 0}) does not put it in front of the demo chain, which is + * {@code @Order(-100)} and matches {@code /**}: requests to the management port reach the demo + * chain first, which permits every read. The bean exists for its name, which replaces the + * template's restricted chain from {@code composition:service-runtime}. + */ @Bean @Order(0) public SecurityFilterChain actuatorSecurityFilterChain(HttpSecurity http) throws Exception { diff --git a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoAccountsProductionGuard.java b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoAccountsProductionGuard.java index 5d64db3b..943e2542 100644 --- a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoAccountsProductionGuard.java +++ b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoAccountsProductionGuard.java @@ -1,28 +1,34 @@ package dev.caskeleton.sample.portfolio.bootstrap.security; -import dev.caskeleton.adapter.outbound.support.runtime.RuntimeProfileSnapshot; import dev.caskeleton.adapter.outbound.support.startup.StartupPhase; import dev.caskeleton.adapter.outbound.support.startup.StartupRule; +import java.util.Arrays; import java.util.List; +import java.util.Set; import org.springframework.core.env.Environment; /** - * Refuses the {@code prod} profile while the {@link SampleDemoAccounts} authenticate requests. + * Lets the {@link SampleDemoAccounts} authenticate requests only under a profile that names a demo: + * {@code local}, {@code test} or {@code demo}, switched on explicitly and alone. * *

    The demo accounts are published on purpose: each password is the account name and the sample * README prints both, so a reader can watch {@code @RequiresPermission} allow one caller and refuse * another. That is harmless in a demo and an open admin account anywhere else. The sample is also - * the code adopters copy first, so the copy that matters is the one that reaches a production - * deployment with this configuration still in it. The README says not to; this rule makes the - * deployment say so too, before any datasource, migration or listener is created. + * the code adopters copy first, so the copy that matters is the one that reaches a deployment with + * this configuration still in it. The README says not to; this rule makes the deployment say so + * too, before any datasource, migration or listener is created. * - *

    "Production" is the template's own answer ({@link RuntimeProfileSnapshot#isProduction()}), not - * a profile name compared here, so the sample and every module agree on what it means. The rule is - * declared beside the accounts in {@link SampleDemoSecurityConfig}: replacing the demo chain with a - * real one removes the refusal with it. + *

    An allow list, not a deny list (review 2026-11-03 RR-07-02, owner decision). Refusing exactly + * {@code prod} left the accounts open under no profile, {@code production}, {@code staging} or any + * other name a deployment happened to use: one differently spelled profile opened an admin account. + * Here a deployment is refused unless every active profile is a demo profile, so no profile at all + * — the sample's image sets none — is refused too. */ final class SampleDemoAccountsProductionGuard implements StartupRule { + /** The profiles that say "this is a demo", and nothing else may be active beside them. */ + static final Set DEMO_PROFILES = Set.of("local", "test", "demo"); + @Override public StartupPhase phase() { return StartupPhase.PROFILE_CHECK; @@ -30,23 +36,31 @@ final class SampleDemoAccountsProductionGuard implements StartupRule { @Override public String summary() { - return "the sample's demo accounts must not authenticate under the prod profile"; + return "the sample's demo accounts authenticate only under the local, test or demo profile"; } @Override public String remedy() { - return "run the sample under local or dev; a production service authenticates through the JWT" - + " chain — delete SampleDemoSecurityConfig and SampleDemoAccounts and stop excluding" - + " WebSecurityAutoConfiguration in SamplePortfolioApplication"; + return "run the sample with SPRING_PROFILES_ACTIVE=local (or test or demo); a real service" + + " authenticates through the JWT chain — delete SampleDemoSecurityConfig and" + + " SampleDemoAccounts and stop excluding WebSecurityAutoConfiguration in" + + " SamplePortfolioApplication"; } @Override public List violations(Environment environment) { - if (!RuntimeProfileSnapshot.from(environment).isProduction()) { + List active = Arrays.asList(environment.getActiveProfiles()); + String risk = + "demo-user and demo-admin sign in over HTTP Basic, each password is the account name, and" + + " the sample README prints both — anyone could act as an admin"; + if (active.isEmpty()) { + return List.of("no profile is active, which a deployment cannot be told apart from: " + risk); + } + List outside = + active.stream().filter(profile -> !DEMO_PROFILES.contains(profile)).toList(); + if (outside.isEmpty()) { return List.of(); } - return List.of( - "demo-user and demo-admin sign in over HTTP Basic, each password is the account name, and" - + " the sample README prints both — anyone could act as an admin"); + return List.of("active profile(s) " + outside + " are not a demo profile: " + risk); } } diff --git a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoSecurityConfig.java b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoSecurityConfig.java index 0fad9aab..15d15e80 100644 --- a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoSecurityConfig.java +++ b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoSecurityConfig.java @@ -3,6 +3,7 @@ package dev.caskeleton.sample.portfolio.bootstrap.security; import dev.caskeleton.adapter.inbound.web.auth.EnvelopeAuthenticationEntryPoint; import dev.caskeleton.adapter.inbound.web.auth.SecurityErrorClassifier; import dev.caskeleton.adapter.outbound.support.startup.StartupRule; +import jakarta.servlet.DispatcherType; import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -25,7 +26,8 @@ import tools.jackson.databind.ObjectMapper; * {@code @RequiresPermission} is checked against {@code app.authz} exactly as it is for a JWT * caller. This chain only answers who the caller is. * - *

    The accounts are public, so they refuse the prod profile at startup ({@link + *

    The accounts are public, so the sample refuses to start unless a demo profile ({@code local}, + * {@code test} or {@code demo}) is explicitly active and no other is ({@link * SampleDemoAccountsProductionGuard}). */ @Configuration @@ -48,10 +50,10 @@ public class SampleDemoSecurityConfig { } /** - * Refuses to start under the prod profile while these accounts are the way in. + * Refuses to start outside a demo profile while these accounts are the way in. * *

    Static and argument-free, as every startup rule is: the support module's runner creates it - * before any other singleton, so a production start is refused before a datasource opens or a + * before any other singleton, so a non-demo start is refused before a datasource opens or a * migration runs. Declared here, beside the accounts, so the refusal leaves with them. * * @return the rule @@ -74,7 +76,13 @@ public class SampleDemoSecurityConfig { .exceptionHandling(ex -> ex.authenticationEntryPoint(unauthenticated)) .authorizeHttpRequests( auth -> - auth.requestMatchers(HttpMethod.GET) + // The container's error dispatch for a request that already failed, as the web + // leaf's chain allows it: authorising it again would answer a rejected anonymous + // write with a 401 that blames authentication. Only the dispatch type is opened; a + // request that names /error itself is still a request (re-review RR-02-02). + auth.dispatcherTypeMatchers(DispatcherType.ERROR) + .permitAll() + .requestMatchers(HttpMethod.GET) .permitAll() .requestMatchers(HttpMethod.HEAD) .permitAll() diff --git a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/domain/poster/PosterRepository.java b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/domain/poster/PosterRepository.java index 420f08f5..e3ba130f 100644 --- a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/domain/poster/PosterRepository.java +++ b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/domain/poster/PosterRepository.java @@ -2,13 +2,23 @@ package dev.caskeleton.sample.portfolio.domain.poster; import java.util.Optional; -/** Outbound port for Poster persistence. Implemented in adapter-persistence. */ +/** + * Outbound port for Poster persistence. Implemented in adapter-persistence; {@code + * PosterRepositoryContract} (test fixtures) holds the adapter and the in-memory fake to the same + * promises. + */ public interface PosterRepository { /** - * Stores the poster. Titles are unique ignoring case, and the store enforces it: a save that - * would duplicate a stored title fails with the application's {@code - * PosterTitleAlreadyExistsException}, even when {@link #existsByTitle} said the title was free. + * Stores the poster. Titles are unique ignoring case, and the store enforces it, even when {@link + * #existsByTitle} said the title was free. + * + *

    A business-rule conflict is a domain exception the port declares and the adapter raises, so + * every implementation answers it the same way. A technical conflict — a stale {@code @Version}, + * as in {@code WorkLogRepository} — stays the framework's exception, which the transaction port + * classifies as {@code DB_OPTIMISTIC_CONFLICT}. + * + * @throws PosterTitleAlreadyExistsException when another poster already has this title */ Poster save(Poster poster); diff --git a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/domain/poster/PosterTitleAlreadyExistsException.java b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/domain/poster/PosterTitleAlreadyExistsException.java new file mode 100644 index 00000000..0d90e3fd --- /dev/null +++ b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/domain/poster/PosterTitleAlreadyExistsException.java @@ -0,0 +1,20 @@ +package dev.caskeleton.sample.portfolio.domain.poster; + +/** + * A poster title is already taken, ignoring case — the rule {@link PosterRepository#save} enforces + * and the create use case asks about first. The web adapter answers it 409 {@code + * POSTER_ALREADY_EXISTS}. + * + *

    A domain type, beside the port that raises it, because the port's contract has to name it in + * code (review 2026-11-03 RR-06-12): it used to be an application exception that the domain port + * could only mention in Javadoc. One type for both paths — the use case's early answer and the + * store's unique index — so the two cannot drift into different answers. + */ +public class PosterTitleAlreadyExistsException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public PosterTitleAlreadyExistsException(String title) { + super("Poster title already in use: " + title); + } +} diff --git a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/domain/worklog/WorkLogRepository.java b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/domain/worklog/WorkLogRepository.java index 4ae6dbfb..17cc9c1e 100644 --- a/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/domain/worklog/WorkLogRepository.java +++ b/src/sample-portfolio/src/main/java/dev/caskeleton/sample/portfolio/domain/worklog/WorkLogRepository.java @@ -8,6 +8,13 @@ import java.util.Optional; * this port. See README. */ public interface WorkLogRepository { + /** + * Stores the work log. Saving one read at an older version than the stored row is a technical + * conflict, not a business rule: the adapter raises the framework's optimistic-locking failure — + * the same one Hibernate raises when the version check fails at flush — and the transaction port + * classifies both as {@code DB_OPTIMISTIC_CONFLICT}. A business-rule conflict is a domain + * exception the port declares instead ({@code PosterRepository#save}). + */ WorkLog save(WorkLog workLog); Optional findById(WorkLogId id); diff --git a/src/sample-portfolio/src/main/resources/application.yml b/src/sample-portfolio/src/main/resources/application.yml index 7c5ec9fe..4ce4bf16 100644 --- a/src/sample-portfolio/src/main/resources/application.yml +++ b/src/sample-portfolio/src/main/resources/application.yml @@ -26,8 +26,10 @@ spring: error: include-stacktrace: ${APP_SERVER_ERROR_INCLUDE_STACKTRACE:never} include-message: ${APP_SERVER_ERROR_INCLUDE_MESSAGE:never} - profiles: - active: ${SPRING_PROFILES_ACTIVE:local} + # No spring.profiles.active default. The demo accounts open only under an explicitly active + # local, test or demo profile (SampleDemoAccountsProductionGuard), and a default here would turn + # "no profile" — what the image and a copied deployment start with — into `local` (RR-07-02). + # SPRING_PROFILES_ACTIVE still sets it; `./gradlew :sample-portfolio:bootRun` passes `local`. datasource: url: ${APP_DATASOURCE_URL:jdbc:postgresql://localhost:5432/ca_skeleton} username: ${APP_DATASOURCE_USERNAME:ca_skeleton} @@ -153,8 +155,10 @@ logging: org.hibernate.SQL: ${APP_LOG_LEVEL_SQL:WARN} # --------------------------------------------------------------------------- -# Application-owned settings (app.*). -# These are bound by @ConfigurationProperties in the respective modules. +# Application-owned settings (app.*), each read by a module on this sample's classpath. +# SampleRuntimeDefaultsContractTest (app-bootstrap repository-contract lane) fails on a group nothing +# here reads: the messaging, notification and HTTP-client blocks that used to sit below configured +# modules the sample does not have (review 2026-11-03 RR-06-06). # --------------------------------------------------------------------------- app: capabilities: @@ -197,7 +201,7 @@ app: published-retention: P7D # ------------------------------------------------------------------------- - # Optional integration adapter toggles (all disabled for lean demo). + # The poster image workflow, off for the lean demo. # ------------------------------------------------------------------------- poster-image: api: @@ -207,32 +211,6 @@ app: enabled: false publication: enabled: false - messaging: - broker: ${APP_MESSAGING_BROKER:} - notification: - slack-webhook: - enabled: ${APP_NOTIFICATION_SLACK_WEBHOOK_ENABLED:false} - google-email: - enabled: ${APP_NOTIFICATION_GOOGLE_EMAIL_ENABLED:false} - outbound: - http: - connect-timeout: ${APP_OUTBOUND_HTTP_CONNECT_TIMEOUT:2s} - read-timeout: ${APP_OUTBOUND_HTTP_READ_TIMEOUT:5s} - global-call-timeout: ${APP_OUTBOUND_HTTP_GLOBAL_CALL_TIMEOUT:10s} - maximum-in-flight-calls: ${APP_OUTBOUND_HTTP_MAXIMUM_IN_FLIGHT_CALLS:128} - retry-enabled: ${APP_OUTBOUND_HTTP_RETRY_ENABLED:false} - retry: - max-attempts: ${APP_OUTBOUND_HTTP_RETRY_MAX_ATTEMPTS:3} - initial-backoff: ${APP_OUTBOUND_HTTP_RETRY_INITIAL_BACKOFF:100ms} - backoff-multiplier: ${APP_OUTBOUND_HTTP_RETRY_BACKOFF_MULTIPLIER:2.0} - circuit-breaker-enabled: ${APP_OUTBOUND_HTTP_CIRCUIT_BREAKER_ENABLED:false} - circuit-breaker: - failure-rate-threshold: ${APP_OUTBOUND_HTTP_CIRCUIT_BREAKER_FAILURE_RATE_THRESHOLD:50} - sliding-window-size: ${APP_OUTBOUND_HTTP_CIRCUIT_BREAKER_SLIDING_WINDOW_SIZE:100} - minimum-number-of-calls: ${APP_OUTBOUND_HTTP_CIRCUIT_BREAKER_MINIMUM_NUMBER_OF_CALLS:100} - wait-duration-in-open-state: ${APP_OUTBOUND_HTTP_CIRCUIT_BREAKER_WAIT_DURATION_IN_OPEN_STATE:60s} - permitted-calls-in-half-open: ${APP_OUTBOUND_HTTP_CIRCUIT_BREAKER_PERMITTED_CALLS_IN_HALF_OPEN:10} - response-size-limit: ${APP_OUTBOUND_HTTP_RESPONSE_SIZE_LIMIT:10MB} --- spring: diff --git a/src/sample-portfolio/src/systemTest/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleWorkLogAuthorizationSystemTest.java b/src/sample-portfolio/src/systemTest/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleWorkLogAuthorizationSystemTest.java index d5f6c7d7..69c015b9 100644 --- a/src/sample-portfolio/src/systemTest/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleWorkLogAuthorizationSystemTest.java +++ b/src/sample-portfolio/src/systemTest/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleWorkLogAuthorizationSystemTest.java @@ -21,6 +21,7 @@ import java.time.LocalDate; import java.util.Base64; import java.util.List; import java.util.UUID; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -144,6 +145,43 @@ class SampleWorkLogAuthorizationSystemTest { assertThat(List.of(response.statusCode(), title(id))).containsExactly(403, "stored"); } + /** + * A write the firewall rejects (a {@code ;} in the path) never reaches authentication, so the + * container's error dispatch to {@code /error} carries no caller. The demo chain lets that + * dispatch through, as the web leaf's chain does; otherwise the rejection reached the client as a + * 401 that blamed authentication for a malformed request (re-review RR-02-02). + */ + @Test + @DisplayName("방화벽이 거절한 익명 쓰기 요청은 오류 디스패치에서 401로 바뀌지 않고 원래 400으로 나간다") + void aRejectedAnonymousWriteKeepsItsOwnStatusThroughTheErrorDispatch() throws Exception { + // given + HttpRequest.Builder rejected = + HttpRequest.newBuilder(uri("/api/worklogs;x=1")) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(CREATE_BODY)); + + // when + HttpResponse response = send(rejected, null); + + // then + assertThat(response.statusCode()).as(response.body()).isEqualTo(400); + } + + @Test + @DisplayName("오류 디스패치만 열린다 — /error에 직접 보낸 익명 쓰기는 여전히 401이다") + void anAnonymousWriteNamingTheErrorPathStillNeedsADemoAccount() throws Exception { + // when + HttpResponse response = + send( + HttpRequest.newBuilder(uri("/error")) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(CREATE_BODY)), + null); + + // then + assertThat(response.statusCode()).isEqualTo(401); + } + private WorkLogId storedWorkLog() { return storedWorkLog("sample-owner"); } diff --git a/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/controller/PosterControllerWireTest.java b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/controller/PosterControllerWireTest.java index cc319c83..1e64bbf4 100644 --- a/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/controller/PosterControllerWireTest.java +++ b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/controller/PosterControllerWireTest.java @@ -21,7 +21,6 @@ import dev.caskeleton.sample.portfolio.application.poster.GetPosterQuery; import dev.caskeleton.sample.portfolio.application.poster.GetPosterUseCase; import dev.caskeleton.sample.portfolio.application.poster.ListPostersUseCase; import dev.caskeleton.sample.portfolio.application.poster.PosterNotFoundException; -import dev.caskeleton.sample.portfolio.application.poster.PosterTitleAlreadyExistsException; import dev.caskeleton.sample.portfolio.application.poster.PublishPosterUseCase; import dev.caskeleton.sample.portfolio.application.poster.UpdatePosterUseCase; import dev.caskeleton.sample.portfolio.domain.poster.Poster; @@ -29,6 +28,7 @@ import dev.caskeleton.sample.portfolio.domain.poster.PosterId; import dev.caskeleton.sample.portfolio.domain.poster.PosterInvariantException; import dev.caskeleton.sample.portfolio.domain.poster.PosterPage; import dev.caskeleton.sample.portfolio.domain.poster.PosterStatus; +import dev.caskeleton.sample.portfolio.domain.poster.PosterTitleAlreadyExistsException; import java.util.List; import java.util.Locale; import org.junit.jupiter.api.Nested; diff --git a/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/error/DomainExceptionHandlerTest.java b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/error/DomainExceptionHandlerTest.java index 718181fc..4c19e0e0 100644 --- a/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/error/DomainExceptionHandlerTest.java +++ b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/error/DomainExceptionHandlerTest.java @@ -3,11 +3,11 @@ package dev.caskeleton.sample.portfolio.adapter.inbound.web.error; import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.sample.portfolio.application.poster.PosterNotFoundException; -import dev.caskeleton.sample.portfolio.application.poster.PosterTitleAlreadyExistsException; import dev.caskeleton.sample.portfolio.application.posterimage.PosterImageAlreadyPublishedException; import dev.caskeleton.sample.portfolio.application.worklog.WorkLogNotFoundException; import dev.caskeleton.sample.portfolio.domain.poster.PosterId; import dev.caskeleton.sample.portfolio.domain.poster.PosterInvariantException; +import dev.caskeleton.sample.portfolio.domain.poster.PosterTitleAlreadyExistsException; import dev.caskeleton.sample.portfolio.domain.worklog.WorkLogId; import dev.caskeleton.sample.portfolio.domain.worklog.WorkLogInvariantException; import dev.caskeleton.shared.error.Category; diff --git a/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/application/event/SamplePayloadSchemaReleaseTest.java b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/application/event/SamplePayloadSchemaReleaseTest.java new file mode 100644 index 00000000..36ce65ca --- /dev/null +++ b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/application/event/SamplePayloadSchemaReleaseTest.java @@ -0,0 +1,18 @@ +package dev.caskeleton.sample.portfolio.application.event; + +import dev.caskeleton.application.messaging.contract.testkit.ReleasedPayloadSchemaContract; +import java.nio.file.Path; + +/** The payload schemas the sample publishes (poster and work-log events). */ +class SamplePayloadSchemaReleaseTest extends ReleasedPayloadSchemaContract { + + @Override + protected Path schemaRoot() { + return Path.of("src/main/resources/contracts/messaging"); + } + + @Override + protected Path manifest() { + return Path.of("src/test/resources/contracts/released-payload-schemas.sha256"); + } +} diff --git a/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/application/worklog/WorkLogUseCasesTest.java b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/application/worklog/WorkLogUseCasesTest.java index 42e9167e..8c2b645e 100644 --- a/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/application/worklog/WorkLogUseCasesTest.java +++ b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/application/worklog/WorkLogUseCasesTest.java @@ -56,7 +56,7 @@ class WorkLogUseCasesTest { new PortfolioEventOutbox( NO_OP_OUTBOX, new FakeIntegrationEventEncoder(new WorkLogReservedContractContribution()), - () -> "01FAKEEVENTIDTSV4RRFFQ6900", + () -> "0190c9a2-1b3c-7def-8abc-1234567890ac", NO_CORRELATION, Clock.systemUTC(), new InMemoryOutboxAggregateSequences()), diff --git a/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoAccountsProductionGuardTest.java b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoAccountsProductionGuardTest.java index c56ed6d1..c570821b 100644 --- a/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoAccountsProductionGuardTest.java +++ b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoAccountsProductionGuardTest.java @@ -53,8 +53,9 @@ class SampleDemoAccountsProductionGuardTest { } @ParameterizedTest - @ValueSource(strings = {"local", "dev", "test"}) - void theDemoAccountsAreAllowedOutsideProduction(String profile) { + @ValueSource(strings = {"local", "test", "demo"}) + @DisplayName("local·test·demo 프로필이 명시적으로 켜졌을 때만 데모 계정을 허용한다") + void theDemoAccountsAreAllowedOnlyUnderAnExplicitDemoProfile(String profile) { // given MockEnvironment environment = new MockEnvironment(); environment.setActiveProfiles(profile); @@ -66,13 +67,29 @@ class SampleDemoAccountsProductionGuardTest { assertThat(violations).isEmpty(); } + @ParameterizedTest + @ValueSource(strings = {"production", "staging", "prd", "dev", "local,prod", "test,staging"}) + @DisplayName("허용 목록 밖의 프로필이 하나라도 켜져 있으면 데모 계정을 거부한다") + void theDemoAccountsAreRefusedUnderAnyProfileOutsideTheAllowList(String profiles) { + // given + MockEnvironment environment = new MockEnvironment(); + environment.setActiveProfiles(profiles.split(",")); + + // when + List violations = new SampleDemoAccountsProductionGuard().violations(environment); + + // then + assertThat(violations).singleElement().asString().contains("demo-admin"); + } + @Test - void theDemoAccountsAreAllowedWithNoProfileAtAll() { + @DisplayName("프로필이 하나도 없으면 운영 배포와 구별할 수 없으므로 데모 계정을 거부한다") + void theDemoAccountsAreRefusedWithNoProfileAtAll() { // when List violations = new SampleDemoAccountsProductionGuard().violations(new MockEnvironment()); // then - assertThat(violations).isEmpty(); + assertThat(violations).singleElement().asString().contains("no profile"); } } diff --git a/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoSecurityConfigTest.java b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoSecurityConfigTest.java index 98067db3..e5b8f976 100644 --- a/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoSecurityConfigTest.java +++ b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/bootstrap/security/SampleDemoSecurityConfigTest.java @@ -43,6 +43,7 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Import; import org.springframework.core.annotation.AnnotationUtils; import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.bean.override.mockito.MockitoBean; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.ResultActions; @@ -53,9 +54,14 @@ import org.springframework.test.web.servlet.ResultActions; properties = { "app.persistence-jpa.enabled=false", // The PostgreSQL idempotency store needs the relational module this test leaves off. - "app.capabilities.idempotency.provider=disabled" + "app.capabilities.idempotency.provider=disabled", + // Object storage is on while present and has no destination here; the `local` profile used to + // switch it off by default, and this test runs under `test` (RR-07-02). + "app.object-storage.enabled=false" }) @AutoConfigureMockMvc +// The demo accounts start only under a demo profile (SampleDemoAccountsProductionGuard). +@ActiveProfiles("test") class SampleDemoSecurityConfigTest { private static final String ID = "0190bd6e-7c3e-7abc-8def-0123456789ab"; diff --git a/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/domain/poster/testkit/InMemoryPosterRepositoryContractTest.java b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/domain/poster/testkit/InMemoryPosterRepositoryContractTest.java new file mode 100644 index 00000000..ff29433e --- /dev/null +++ b/src/sample-portfolio/src/test/java/dev/caskeleton/sample/portfolio/domain/poster/testkit/InMemoryPosterRepositoryContractTest.java @@ -0,0 +1,19 @@ +package dev.caskeleton.sample.portfolio.domain.poster.testkit; + +import dev.caskeleton.sample.portfolio.domain.poster.PosterRepository; +import java.util.function.Supplier; + +class InMemoryPosterRepositoryContractTest extends PosterRepositoryContract { + + private final InMemoryPosterRepository repository = new InMemoryPosterRepository(); + + @Override + protected PosterRepository repository() { + return repository; + } + + @Override + protected T inTransaction(Supplier work) { + return work.get(); + } +} diff --git a/src/sample-portfolio/src/test/resources/contracts/released-payload-schemas.sha256 b/src/sample-portfolio/src/test/resources/contracts/released-payload-schemas.sha256 new file mode 100644 index 00000000..bb3f2a39 --- /dev/null +++ b/src/sample-portfolio/src/test/resources/contracts/released-payload-schemas.sha256 @@ -0,0 +1,10 @@ +# SHA-256 (LF-normalized) of every released payload schema version in src/main/resources/contracts/messaging. +# A released version is never edited: a change goes into a new version (application-core README). +# Add a line when a version is released; change a line only for a version no one can have consumed. +7264efd4e2531e6fd00010bb3deca96af2f1d0408d0021d014feaf90190f2eac portfolio.worklog.reserved/v1.schema.json +c71019a727f351aa26afbf6c9246b6cf1a7abcfd2b6bb3ee1185dfce6be504d8 poster.archived/v1.schema.json +8be3a72b7b469d894d06bd30f79f565feb23fbf698674d5caee094de5a2300fb poster.created/v1.schema.json +c5310821c338be5aa916d126008a0a0f4be6c6999775018ce29cbefc44c8a942 poster.deleted/v1.schema.json +179160ec458b710535c7846a47a1abceab1a743a3a5f11633c47c2115e82e2c4 poster.image-attachment-prepared/v1.schema.json +1a256fa47c382a4900ed8f852da9256a01ef1ddb2dc5b0b75047f827ae81644e poster.image-attachment-ready/v2.schema.json +d89954e3074fc015a20d616bf0ec1cb46c088fa417d78e006163f1c2dc61170c poster.published/v1.schema.json diff --git a/src/sample-portfolio/src/testFixtures/java/dev/caskeleton/sample/portfolio/domain/poster/testkit/InMemoryPosterRepository.java b/src/sample-portfolio/src/testFixtures/java/dev/caskeleton/sample/portfolio/domain/poster/testkit/InMemoryPosterRepository.java new file mode 100644 index 00000000..15cf66be --- /dev/null +++ b/src/sample-portfolio/src/testFixtures/java/dev/caskeleton/sample/portfolio/domain/poster/testkit/InMemoryPosterRepository.java @@ -0,0 +1,114 @@ +package dev.caskeleton.sample.portfolio.domain.poster.testkit; + +import dev.caskeleton.sample.portfolio.domain.poster.Poster; +import dev.caskeleton.sample.portfolio.domain.poster.PosterId; +import dev.caskeleton.sample.portfolio.domain.poster.PosterPage; +import dev.caskeleton.sample.portfolio.domain.poster.PosterRepository; +import dev.caskeleton.sample.portfolio.domain.poster.PosterSortField; +import dev.caskeleton.sample.portfolio.domain.poster.PosterStatus; +import dev.caskeleton.sample.portfolio.domain.poster.PosterTitleAlreadyExistsException; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +/** + * In-memory {@link PosterRepository} with the semantics of {@code PosterRepositoryAdapter}, held to + * them by {@link PosterRepositoryContract}: snapshots (a caller's later mutation is not saved), a + * new poster at version 0 and an update incrementing it, and the title rule — a save that would + * duplicate another poster's title, ignoring case, raises {@link PosterTitleAlreadyExistsException} + * as the unique index does. + * + *

    It does not model optimistic-lock conflicts on a stale version. + */ +public final class InMemoryPosterRepository implements PosterRepository { + + private final Map rows = new LinkedHashMap<>(); + + @Override + public Poster save(Poster poster) { + long version; + if (poster.version() == null) { + version = 0L; + } else { + Poster existing = rows.get(poster.id()); + if (existing == null) { + throw new IllegalStateException( + "no persisted poster row to carry audit creation from: " + poster.id()); + } + version = existing.version() + 1; + } + String key = titleKey(poster.title()); + boolean taken = + rows.values().stream() + .anyMatch( + other -> !other.id().equals(poster.id()) && titleKey(other.title()).equals(key)); + if (taken) { + throw new PosterTitleAlreadyExistsException(poster.title()); + } + Poster stored = copy(poster, version); + rows.put(stored.id(), stored); + return copy(stored, version); + } + + @Override + public Optional findById(PosterId id) { + return Optional.ofNullable(rows.get(id)).map(p -> copy(p, p.version())); + } + + @Override + public PosterPage findPage( + int page, int size, PosterSortField sortField, boolean ascending, PosterStatus status) { + int safeSize = size <= 0 ? 20 : size; + int safePage = Math.max(0, page); + List matching = + rows.values().stream().filter(p -> status == null || p.status() == status).toList(); + if (sortField != null) { + Comparator order = + sortField == PosterSortField.TITLE + ? Comparator.comparing(Poster::title) + : Comparator.comparing(Poster::status); + matching = matching.stream().sorted(ascending ? order : order.reversed()).toList(); + } + List items = + matching.stream() + .skip((long) safePage * safeSize) + .limit(safeSize) + .map(p -> copy(p, p.version())) + .toList(); + return new PosterPage(items, matching.size()); + } + + @Override + public boolean existsById(PosterId id) { + return rows.containsKey(id); + } + + @Override + public boolean existsByTitle(String title) { + String key = titleKey(title); + return rows.values().stream().anyMatch(p -> titleKey(p.title()).equals(key)); + } + + @Override + public void deleteById(PosterId id) { + rows.remove(id); + } + + /** The unique index's key: {@code lower(title)} (sample migration V8.1). */ + private static String titleKey(String title) { + return title.toLowerCase(Locale.ROOT); + } + + private static Poster copy(Poster poster, long version) { + return Poster.rehydrate( + poster.id(), + poster.title(), + poster.caption(), + poster.imageReference(), + poster.status(), + version); + } +} diff --git a/src/sample-portfolio/src/testFixtures/java/dev/caskeleton/sample/portfolio/domain/poster/testkit/PosterRepositoryContract.java b/src/sample-portfolio/src/testFixtures/java/dev/caskeleton/sample/portfolio/domain/poster/testkit/PosterRepositoryContract.java new file mode 100644 index 00000000..2835061e --- /dev/null +++ b/src/sample-portfolio/src/testFixtures/java/dev/caskeleton/sample/portfolio/domain/poster/testkit/PosterRepositoryContract.java @@ -0,0 +1,119 @@ +package dev.caskeleton.sample.portfolio.domain.poster.testkit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.sample.portfolio.domain.poster.Poster; +import dev.caskeleton.sample.portfolio.domain.poster.PosterId; +import dev.caskeleton.sample.portfolio.domain.poster.PosterRepository; +import dev.caskeleton.sample.portfolio.domain.poster.PosterTitleAlreadyExistsException; +import java.util.Optional; +import java.util.function.Supplier; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * What every {@link PosterRepository} promises, run against {@link InMemoryPosterRepository} and + * the PostgreSQL {@code PosterRepositoryAdapter} alike (review 2026-11-03 RR-06-12). + * + *

    The promise that matters is the title rule: titles are unique ignoring case, and {@link + * PosterRepository#save} enforces it by raising the domain's {@link + * PosterTitleAlreadyExistsException}. It used to be a sentence in the port's Javadoc that pointed + * at an application type, so only the PostgreSQL adapter kept it and a unit test's mock could not. + * + *

    Every call goes through {@link #inTransaction}, so each one commits before the next reads it. + */ +public abstract class PosterRepositoryContract { + + private static final PosterId FIRST = PosterId.of("0190bd6e-7c3e-7abc-8def-0123456789ab"); + private static final PosterId SECOND = PosterId.of("0190bd6e-7c3e-7abc-8def-0123456789ac"); + + /** The repository under test, empty at the start of each test. */ + protected abstract PosterRepository repository(); + + /** Runs {@code work} in one committed transaction of the repository under test. */ + protected abstract T inTransaction(Supplier work); + + @Test + @DisplayName("저장한 포스터는 id로 다시 읽으면 같은 제목과 버전 0으로 나온다") + void aSavedPosterIsReadBackByIdAtVersionZero() { + // given + save(Poster.create(FIRST, "여름 세일 배너", "7월 한정")); + + // when + Optional found = inTransaction(() -> repository().findById(FIRST)); + + // then + assertThat(found) + .get() + .extracting(Poster::title, Poster::version) + .containsExactly("여름 세일 배너", 0L); + } + + @Test + @DisplayName("대소문자만 다른 같은 제목의 두 번째 포스터 저장은 domain 제목 충돌 예외로 거절된다") + void aSecondPosterWithTheSameTitleInAnotherCaseIsRejected() { + // given + save(Poster.create(FIRST, "Summer Sale Banner", null)); + + // when + Throwable failure = + catchThrowable(() -> save(Poster.create(SECOND, "SUMMER SALE banner", null))); + + // then + assertThat(failure) + .isInstanceOf(PosterTitleAlreadyExistsException.class) + .hasMessage("Poster title already in use: SUMMER SALE banner"); + } + + @Test + @DisplayName("다른 포스터의 제목으로 바꾸는 저장도 같은 제목 충돌 예외로 거절된다") + void renamingAPosterToAnotherPostersTitleIsRejected() { + // given + save(Poster.create(FIRST, "Summer Sale Banner", null)); + save(Poster.create(SECOND, "Autumn Banner", null)); + Poster second = inTransaction(() -> repository().findById(SECOND)).orElseThrow(); + second.rename("summer sale banner"); + + // when + Throwable failure = catchThrowable(() -> save(second)); + + // then + assertThat(failure).isInstanceOf(PosterTitleAlreadyExistsException.class); + } + + @Test + @DisplayName("자기 제목을 대소문자만 바꿔 저장하는 것은 허용된다") + void renamingAPosterToItsOwnTitleInAnotherCaseIsAccepted() { + // given + save(Poster.create(FIRST, "Summer Sale Banner", null)); + Poster loaded = inTransaction(() -> repository().findById(FIRST)).orElseThrow(); + loaded.rename("SUMMER SALE BANNER"); + + // when + save(loaded); + + // then + assertThat(inTransaction(() -> repository().findById(FIRST))) + .get() + .extracting(Poster::title) + .isEqualTo("SUMMER SALE BANNER"); + } + + @Test + @DisplayName("existsByTitle은 대소문자를 무시하고 저장된 제목을 찾는다") + void existsByTitleIgnoresCase() { + // given + save(Poster.create(FIRST, "Summer Sale Banner", null)); + + // when + boolean exists = inTransaction(() -> repository().existsByTitle("SUMMER sale BANNER")); + + // then + assertThat(exists).isTrue(); + } + + private Poster save(Poster poster) { + return inTransaction(() -> repository().save(poster)); + } +} diff --git a/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/contract/OpenApiDriftContractTest.java b/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/contract/OpenApiDriftContractTest.java index 993d1921..550dba62 100644 --- a/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/contract/OpenApiDriftContractTest.java +++ b/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/contract/OpenApiDriftContractTest.java @@ -40,6 +40,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.web.server.LocalServerPort; import org.springframework.context.annotation.Import; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.bean.override.mockito.MockitoBean; /** @@ -58,6 +59,9 @@ import org.springframework.test.context.bean.override.mockito.MockitoBean; * ./gradlew openapiCheckSnapshot -PapproveOpenApiChange} (sets {@code * -Dopenapi.snapshot.write=true}). */ +// The sample's local composition (object storage off), which application.yml used to select by +// default before the demo-profile allow list removed the default (RR-07-02). +@ActiveProfiles("local") @SpringBootTest( classes = OpenApiDriftContractTest.OpenApiDriftTestApp.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, diff --git a/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/date/DateHeaderContractTest.java b/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/date/DateHeaderContractTest.java index b6367ddb..92355b96 100644 --- a/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/date/DateHeaderContractTest.java +++ b/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/date/DateHeaderContractTest.java @@ -21,6 +21,7 @@ import org.springframework.boot.security.oauth2.server.resource.autoconfigure.se import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.web.server.LocalServerPort; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; @@ -30,6 +31,9 @@ import org.springframework.web.bind.annotation.RestController; * container and checks the header across a success (200) and an error (404) response — the * status-matrix that a mock MVC test cannot exercise (MockMvc never goes through Tomcat). */ +// The sample's local composition (object storage off), which application.yml used to select by +// default before the demo-profile allow list removed the default (RR-07-02). +@ActiveProfiles("local") @SpringBootTest( classes = DateHeaderContractTest.DateProbeApp.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, diff --git a/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/filter/VirtualThreadMdcE2ETest.java b/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/filter/VirtualThreadMdcE2ETest.java index 8440a8c1..0e2d519a 100644 --- a/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/filter/VirtualThreadMdcE2ETest.java +++ b/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/filter/VirtualThreadMdcE2ETest.java @@ -35,6 +35,7 @@ import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; @@ -56,6 +57,9 @@ import org.springframework.web.bind.annotation.RestController; *

    Auto-configuration is narrowed to skip security so the demo endpoint is reachable without a * JWT — security is owned by a sibling branch and is not the contract under test here. */ +// The sample's local composition (object storage off), which application.yml used to select by +// default before the demo-profile allow list removed the default (RR-07-02). +@ActiveProfiles("local") @SpringBootTest( classes = VirtualThreadMdcE2ETest.TestBootstrap.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, diff --git a/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/openapi/OpenApiSnapshotTest.java b/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/openapi/OpenApiSnapshotTest.java index 7ea15eba..2cd61b73 100644 --- a/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/openapi/OpenApiSnapshotTest.java +++ b/src/sample-portfolio/src/webSystemTest/java/dev/caskeleton/sample/portfolio/adapter/inbound/web/openapi/OpenApiSnapshotTest.java @@ -30,6 +30,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.web.server.LocalServerPort; import org.springframework.context.annotation.Import; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.bean.override.mockito.MockitoBean; import tools.jackson.databind.ObjectMapper; @@ -40,6 +41,9 @@ import tools.jackson.databind.ObjectMapper; * schema cannot pass). The release-blocking drift gate against a committed snapshot is * owned by feature-contract-verification-test-suite (planned). */ +// The sample's local composition (object storage off), which application.yml used to select by +// default before the demo-profile allow list removed the default (RR-07-02). +@ActiveProfiles("local") @SpringBootTest( classes = OpenApiSnapshotTest.OpenApiTestApp.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, diff --git a/src/shared-contract/README.md b/src/shared-contract/README.md index d3b2e6bb..e825d9e2 100644 --- a/src/shared-contract/README.md +++ b/src/shared-contract/README.md @@ -366,7 +366,8 @@ Spring/Lettuce 타입을 노출하지 않는다. - **`ERROR_CODE`=100 은 `error-codes.yaml` 의 행 수 상한과 동기화한다.** 레지스트리가 100행을 넘으면 이 상수와 레지스트리 메모를 함께 갱신해야 한다. - **`TENANT_ID`=1000:** 메트릭 라벨은 bounded mapping-table ID 나 cohort bucket 을 써야 한다 — - raw UUID tenant 식별자는 라벨로 금지. 1001번째 tenant 부터는 bucket folding 이 자동 적용된다. + raw UUID tenant 식별자는 라벨로 금지. 이 상수는 상한 값일 뿐이다: 1000개를 넘는 tenant 값을 접어 + 주는(bucket folding) 코드는 없으므로, 상한을 지키는 매핑은 메트릭을 등록하는 쪽이 만든다. - **tag-key 매핑 quirk 2가지**(`limitFor` 에서 명시 처리): (1) HTTP 메트릭 레지스트리 행의 tag 이름은 `status_code` 가 아니라 `status` 다 — 혼동을 막으려 `status`/`status_code` 둘 다 `STATUS_CODE`(7)로 매핑. (2) Resilience4j bound 는 `outcome` tag(5값)에 적용 — 상수명은 명료성을 위해 diff --git a/src/shared-contract/src/main/java/dev/caskeleton/shared/error/OperationalError.java b/src/shared-contract/src/main/java/dev/caskeleton/shared/error/OperationalError.java index 5bd8e8c6..db6e1d52 100644 --- a/src/shared-contract/src/main/java/dev/caskeleton/shared/error/OperationalError.java +++ b/src/shared-contract/src/main/java/dev/caskeleton/shared/error/OperationalError.java @@ -47,6 +47,8 @@ public enum OperationalError implements ApiErrorCode { IDEMPOTENT_IN_FLIGHT(Category.CONFLICT, 409, true), IDEMPOTENT_REQUEST_MISMATCH(Category.VALIDATION, 422, false), IDEMPOTENCY_UNAVAILABLE(Category.TRANSIENT_DEPENDENCY, 503, true), + // the action ran and recording its result failed; a resend with the same key is safe + IDEMPOTENCY_COMPLETION_UNKNOWN(Category.TRANSIENT_DEPENDENCY, 503, true), IDEMPOTENCY_RECOVERY_REQUIRED(Category.DATA_INTEGRITY, 500, false), // adapter runtime lifecycle ADAPTER_DISABLED(Category.INTERNAL, 500, false), diff --git a/src/shared-contract/src/main/java/dev/caskeleton/shared/runtime/RuntimeEnvironment.java b/src/shared-contract/src/main/java/dev/caskeleton/shared/runtime/RuntimeEnvironment.java index cd3bcd08..5ad8aed2 100644 --- a/src/shared-contract/src/main/java/dev/caskeleton/shared/runtime/RuntimeEnvironment.java +++ b/src/shared-contract/src/main/java/dev/caskeleton/shared/runtime/RuntimeEnvironment.java @@ -29,11 +29,11 @@ import java.util.Optional; *

    Three module families ask this question — the composition root, the Fileserver and HTTP client * platform configurations, and the object-storage adapter. {@code shared-contract} is the one leaf * all of them may depend on. It is also Java-stdlib-only, so this type cannot accept Spring's - * {@code Environment}. Spring-aware composition code captures active profiles through - * app-bootstrap's {@code RuntimeProfileSnapshot} bridge and passes the resulting plain collection - * here. That keeps the vocabulary testable without a container and usable from the {@code - * EnvironmentPostProcessor} phase, where no application context exists yet, while preventing ad-hoc - * profile reads from reappearing throughout bootstrap code. + * {@code Environment}. Spring-aware composition code captures active profiles through the {@code + * RuntimeProfileSnapshot} bridge in {@code adapter:outbound:support} and passes the resulting plain + * collection here. That keeps the vocabulary testable without a container and usable from the + * {@code EnvironmentPostProcessor} phase, where no application context exists yet, while preventing + * ad-hoc profile reads from reappearing throughout bootstrap code. * *

    The vendored {@code messaging:*} platform deliberately does not use this type. It depends on * no {@code shared-contract} type by design, and its own profile vocabulary is its own contract. diff --git a/src/shared-contract/src/test/java/dev/caskeleton/shared/error/OperationalErrorTest.java b/src/shared-contract/src/test/java/dev/caskeleton/shared/error/OperationalErrorTest.java index 3de1a863..af634e14 100644 --- a/src/shared-contract/src/test/java/dev/caskeleton/shared/error/OperationalErrorTest.java +++ b/src/shared-contract/src/test/java/dev/caskeleton/shared/error/OperationalErrorTest.java @@ -239,6 +239,21 @@ class OperationalErrorTest { assertThat(OperationalError.IDEMPOTENT_IN_FLIGHT.retryable()).isTrue(); } + @Test + @DisplayName("action 뒤 결과를 기록하지 못한 결과 불명은 같은 키로 다시 보내면 되므로 retryable 503이다") + void idempotencyCompletionUnknownIs503TransientRetryableWithTheSameKey() { + // docs/web/http-contract.md §2 (re-review 2026-11-03, RR-02-03) — the action ran and its result + // was not recorded. Distinct from IDEMPOTENCY_UNAVAILABLE (refused before the action) by code; + // a + // resend with the same key is judged by the server (replay / in-flight / recovery required). + assertThat(OperationalError.IDEMPOTENCY_COMPLETION_UNKNOWN.code()) + .isEqualTo("IDEMPOTENCY_COMPLETION_UNKNOWN"); + assertThat(OperationalError.IDEMPOTENCY_COMPLETION_UNKNOWN.httpStatus()).isEqualTo(503); + assertThat(OperationalError.IDEMPOTENCY_COMPLETION_UNKNOWN.category()) + .isEqualTo(Category.TRANSIENT_DEPENDENCY); + assertThat(OperationalError.IDEMPOTENCY_COMPLETION_UNKNOWN.retryable()).isTrue(); + } + @Test void idempotentRequestMismatchIs422ValidationNotRetryable() { // D8 / error-codes.yaml#IDEMPOTENT_REQUEST_MISMATCH — same key + different body is a client