diff --git a/build.gradle b/build.gradle index 59808a9..c73bdc2 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,9 @@ plugins { id 'ca.architecture' + // Every strict lane of this build belongs to a stage a workflow executes, checked here exactly + // as in the main build: the main build's check cannot see this build's lanes, and while only it + // applied the plugin, grpcPerformanceTest ran nowhere (review 2026-11-03 RR-06-03). + id 'ca.strict-lane-stages' } // No `ca.release-provenance` any more: each gRPC leaf's `ca.java-conventions` derives its version @@ -24,11 +28,20 @@ tasks.register('qualityCheck') { dependsOn subprojects.findAll { it.childProjects.isEmpty() }.collect { "${it.path}:qualityCheck" } } +// SpotBugs + FindSecBugs over the test source sets (config/spotbugs/exclude-test.xml). This build has +// no main stage, and the run is short (eleven tasks), so it joins `ci`. +tasks.register('testQualityCheck') { + group = 'verification' + description = 'Runs test-code static analysis for every optional-platform leaf.' + dependsOn subprojects.findAll { it.childProjects.isEmpty() }.collect { "${it.path}:testQualityCheck" } +} + tasks.register('ci') { group = 'verification' description = 'Runs the optional gRPC platform verification surface.' dependsOn tasks.named('checkAll') dependsOn tasks.named('qualityCheck') + dependsOn tasks.named('testQualityCheck') // architectureCheck, not only the direction gate: it also runs verifyNoIgnoredSourcePackages, so // a gRPC source Git would drop fails this build's ci the way it fails the main build's. dependsOn tasks.named('architectureCheck') @@ -47,3 +60,33 @@ tasks.register('integrationCheck') { description = 'Runs every optional-platform leaf integration test lane. Needs a container runtime.' dependsOn subprojects.findAll { it.childProjects.isEmpty() }.collect { "${it.path}:integrationTestLaneCheck" } } + +// The performance stage of this build: grpcPerformanceTest. It asserts the shape of a measurement +// (ordered percentiles, a gate that reads them) with a generous budget, not a number, so it can run +// on a shared runner. integration-main.yml's performance-check job runs it next to the main build's +// `:performanceCheck`; before this aggregate it ran when somebody asked for it by name, which was +// nobody (review 2026-11-03 RR-06-03). +tasks.register('performanceCheck') { + group = 'verification' + description = 'Runs every optional-platform leaf performance test lane.' + dependsOn subprojects.findAll { it.childProjects.isEmpty() }.collect { "${it.path}:performanceTestLaneCheck" } +} + +// The tag-only stage of this build: the hermetic strict lanes (the three gRPC certification lanes). +// Their tests already run inside each leaf's `test`; the lanes add the one thing `test` cannot say — +// that a tag still selects something — which is worth asserting once per release tag. release.yml +// runs this aggregate instead of naming the lanes, so a lane added here is in the release gate +// without a workflow edit. +tasks.register('releaseCheck') { + group = 'verification' + description = 'Runs every optional-platform leaf hermetic strict test lane (the release gate).' + dependsOn subprojects.findAll { it.childProjects.isEmpty() }.collect { "${it.path}:testLaneCheck" } +} + +// The stages this build's strict lanes may belong to, executed by workflows as +// `./gradlew -p optional-platforms ` from `src`. A command counts when it starts in this +// directory (working-directory joined with -p), which the plugin reads from where this build is. +strictLaneStages { + stage ':ci', ':integrationCheck', ':performanceCheck', ':releaseCheck' + workflowDirectory = layout.projectDirectory.dir('../../.github/workflows') +} diff --git a/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md b/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md index a7e2ebf..7898463 100644 --- a/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md +++ b/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md @@ -47,6 +47,80 @@ retries when everything is failing, which is the behaviour that lets a dependenc transaction annotations, deliberately: a `REQUIRES_NEW` would put the claim in its own transaction and reintroduce the window where the write is durable and the claim is not. +**A claim expires instead of blocking its key forever** (review 2026-11-01, 03-10). A claim still +`IN_PROGRESS` after the claim TTL is presumed abandoned by a dead owner, and the next retry of the same +request takes it over with `GrpcOperationLedger.reclaimStale` — one conditional +`UPDATE … WHERE state = 'IN_PROGRESS' AND claimed_at < :cutoff`, so of two concurrent takeovers exactly +one wins, the same shape as `reclaimFailed`. The TTL is the `claimTtl` argument of +`GrpcIdempotencyInterceptor`, refused at construction when it is not longer than every keyed method's +deadline. Since the 03-11 follow-up the starter assembles the interceptor when +`app.grpc.platform.operation-ledger-enabled=true` and takes the TTL from +`app.grpc.platform.operation-ledger-claim-ttl` (default 5m); a TTL the constructor refuses fails startup +naming that key, and an enabled flag without a `GrpcOperationLedger` bean fails startup too. + +The takeover is safe only because the commit is recorded in the mutation's own transaction: then a +claim still `IN_PROGRESS` means the mutation never committed. An adopter that commits them separately +turns an abandoned claim into an unknown outcome, and a takeover can run the mutation twice. A takeover +of an owner that is merely slow does the same, which is why the TTL must exceed the slowest keyed +method and not just its deadline. + +Rejected: an owner token with a heartbeat lease (the outbox lease model) — needed only when a keyed +method can run longer than any fixed TTL; revisit when one exists. A sweeper job that deletes stale +rows — it needs a scheduler this family does not have, and deleting a claim makes the next retry a +fresh claim with no record that an attempt was abandoned. + +**A taken-over owner is fenced by a claim token** (re-review 2026-11-03, RR-02-04). The owner a +takeover presumes dead may only be slow. Without knowing which attempt held the claim, its late +`markFailed` turned the new attempt's claim `FAILED_TERMINAL`, the next retry re-claimed it, and two +attempts ran the mutation at once; its late `markCommitted` committed beside the new attempt. Every +claim now carries a `GrpcClaimToken` (`claim_attempt`, migration V003): 1 for the first claim, one more +for each `reclaimFailed` or `reclaimStale`. `decide` returns it on `PROCEED`, and `recordCommit` / +`recordFailure` hand it back. A commit or a failure is one conditional +`UPDATE … WHERE state = 'IN_PROGRESS' AND claim_attempt = :attempt`: a commit by an attempt that lost +the claim is refused with an exception, so its business transaction rolls back; a failure by one is +ignored (`markFailed` returns false). The same conditional update closes the JPA commit race — +load-check-save let two commits that both read `IN_PROGRESS` both write; now exactly one matches. +A claim whose conflicting row is purged between insert and read retries the insert once instead of +failing the call. + +`release(identity, claimToken)` is fenced the same way (re-review 2026-11-03 round 2, RR-01-15): it is a +conditional `DELETE … WHERE state = 'IN_PROGRESS' AND claim_attempt = :attempt` and returns whether it +deleted. It used to delete by key alone, so any caller could erase the claim of the attempt that held +it, or a finished record a retry replays. The counter lives in the row, so a deleted row's key starts +again at attempt 1: the token separates the attempts of one row, not attempts across a delete. Both +deletions are limited accordingly — release only for a claim whose owner is known to be gone, purge +only for rows finished longer ago than any retry. Rejected: dropping `release` from the port +(`markFailed` then `reclaimFailed` keeps the counter, but a deployment that must free a key for a +different request has no other path), and a tombstone row that keeps the counter (a second state to +purge, for a window the two limits already close). + +Chosen over the claim moment as the token (`WHERE claimed_at = :claimedAt`), which needs no column: +PostgreSQL stores microseconds while `Instant` carries nanoseconds, so the token a caller holds and +the one stored can differ by rounding, and two processes' clocks can give two attempts the same +moment. The counter has neither problem. The heartbeat lease above stays rejected for the same reason +as before; the token fences without one. + +**What `operation-ledger-enabled=true` promises (03-11 (b), recorded 2026-11-03, RR-02-07).** The +2026-11-01 review offered (a) delete the flag and judge by whether a ledger bean exists, or (b) keep the +flag and have the starter assemble the decision function from it. (b) is kept: the assembly and the +TTL validation live in one place instead of in every adopter. (a) was rejected because it removes the +false promise only by removing the assembly with it. The limit stays and is stated where an operator +meets it: the starter builds no server, so the flag assembles the decision function only, and keys are +honoured once the adopter's `IDEMPOTENCY` stage interceptor or service adapter calls `decide` and +`recordCommit`. `GrpcPlatformStartupValidator`'s refusal for a keyed method says exactly that; a running +process cannot see whether the stage exists, so the validator does not claim to check it. + +**Finished rows are purged after a retention; in-progress rows never are** (03-10 follow-up). +`GrpcOperationLedger.purgeCompletedBefore(cutoff, limit)` deletes at most `limit` `COMMITTED` or +`FAILED_TERMINAL` rows that completed before the cutoff, oldest first — on PostgreSQL one bounded +`DELETE … WHERE storage_key IN (SELECT … LIMIT … FOR UPDATE SKIP LOCKED)` served by the partial index +of migration V002. `GrpcOperationLedgerRetention.purgeExpired(now)` applies +`app.grpc.platform.operation-ledger-retention` (default 24h, the HTTP idempotency replay TTL) in +batches. A retry after its record was purged is a fresh claim and runs again, so the retention must +exceed the callers' retry window. The adopter schedules the purge; the starter assembles the policy +only, because a scheduled bean would force a scheduler onto every deployment and the starter cannot +know which replica should run it. + **A key reused for a different request is a caller error, not a duplicate.** The stored request fingerprint turns that into `FAILED_PRECONDITION` rather than silently returning the first request's answer. diff --git a/docs/runbooks/grpc-platform-operations.md b/docs/runbooks/grpc-platform-operations.md index 07d0639..aa896ec 100644 --- a/docs/runbooks/grpc-platform-operations.md +++ b/docs/runbooks/grpc-platform-operations.md @@ -38,8 +38,45 @@ the time, and which half is not knowable from the status. 6. If the method is not keyed, there is no ledger row. Resolve it against the business resource, or escalate to the service owner. This is the case the keyed profile exists to avoid. +7. `IN_PROGRESS` means a claim exists and has not finished. If its owner died, the claim is not + stuck forever: once it is older than the claim TTL (`app.grpc.platform.operation-ledger-claim-ttl`, + default 5m), the next retry of the same request takes it over and runs again, + and a later reconciler pass sees that attempt's `COMMITTED` or `FAILED_TERMINAL`. Until a retry + arrives, the case stays pending. The re-run is safe only when the ledger commit is recorded in the + mutation's own transaction; if this service commits them separately, check the business resource + before letting the retry through. An owner that was only slow cannot spoil the takeover: its claim + attempt (`claim_attempt`) is behind, so its late commit is refused and its late failure ignored. + A log line saying an operation "is no longer held by attempt N" is that refusal, not an outage. + **Escalate when** the reconciler's pending list grows across passes. That means the ledger is -unreachable rather than slow. +unreachable rather than slow. A case pending as `IN_PROGRESS` for longer than the claim TTL with no +retry arriving is a caller that gave up; it needs the business resource checked, not a ledger fix. + +--- + +## The operation ledger table keeps growing + +**What you are seeing.** `grpc_operation_ledger` grows without bound, or a retry of an old key runs +the mutation again instead of replaying. + +**What it means.** Finished rows (`COMMITTED`, `FAILED_TERMINAL`) are deleted only when something +calls `GrpcOperationLedgerRetention.purgeExpired(now)`; the starter assembles that bean +(`app.grpc.platform.operation-ledger-enabled=true`) but does not schedule it. A growing table means +the service's scheduler is not calling it. A retry that re-ran after the retention +(`app.grpc.platform.operation-ledger-retention`, default 24h) is the trade the retention makes: its +record was purged, so it was a new operation. + +**What to do.** + +1. Check that the service schedules `purgeExpired`. Each call deletes in batches of 500 until a batch + comes back short, never touching `IN_PROGRESS` rows. +2. For a large backlog, call it repeatedly; each batch is a short transaction that skips rows another + transaction holds. +3. If retried keys re-run after the retention, raise the retention above the callers' retry window + rather than skipping the purge. + +**Escalate when** old `IN_PROGRESS` rows accumulate. Those are not purged by design; each is an +abandoned claim no retry has taken over (see `IN_PROGRESS` above). --- diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..0d1f586 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,9 @@ +# optional-platforms is a build of its own (`./gradlew -p optional-platforms ...`), and Gradle reads +# gradle.properties from the build's root directory only, so src/gradle.properties never reached it: +# the optional build ran serially and without the build cache (review 2026-11-01 08-06). +# The values and their reasons are src/gradle.properties'. settings.gradle fails the build when the +# two files disagree, so this is a copy that cannot drift, not a second source. +org.gradle.workers.max=4 +org.gradle.jvmargs=-Xmx1g -XX:MaxMetaspaceSize=512m -Dfile.encoding=UTF-8 +org.gradle.caching=true +org.gradle.parallel=true diff --git a/grpc-advanced/CLAUDE.md b/grpc-advanced/CLAUDE.md index 3955a52..12b53b2 100644 --- a/grpc-advanced/CLAUDE.md +++ b/grpc-advanced/CLAUDE.md @@ -18,6 +18,12 @@ grpc:* → grpc-advanced:* (금지 — registry가 거부한다) `grpc-spring-boot-starter`의 registry 엔트리에는 어떤 advanced id도 없다. `verifyCleanArchitectureDependencies`가 build time에 이 규칙을 강제한다. runtime 검사는 없다. +### 공용 다이제스트 헬퍼를 복사해 두는 이유 (review 2026-11-01 01-11) + +`grpc-advanced`(`GrpcDiagnosticsRedactor`)는 SHA-256 계산을 자기 안에 둔다. 의존 방향이 +`grpc-advanced:* → grpc:*`로 닫혀 있고 `shared-contract` edge가 없는 것은 의도다. `grpc/CLAUDE.md`의 +같은 절과 이유가 같다(vendored 플랫폼, MSG-023 §6.2와 같은 배치). + ## Capability grade와 feature flag capability마다 별도 flag를 갖는다. 하나의 "advanced" 스위치로 묶지 않는 이유는, gRPC-Web을 켜는 diff --git a/grpc/CLAUDE.md b/grpc/CLAUDE.md index 7bb05da..5705281 100644 --- a/grpc/CLAUDE.md +++ b/grpc/CLAUDE.md @@ -41,6 +41,15 @@ server profile·named channel profile·method size profile·method policy 기본 `caskeleton.grpc.common.v1.generated` — 손으로 쓴 `dev.caskeleton`과 겹치지 않는다). 템플릿을 재사용할 때 바꾸는 방법은 `AGENTS.md` 「템플릿 재사용 체크리스트」에 있다. +### 공용 다이제스트 헬퍼를 복사해 두는 이유 (review 2026-11-01 01-11) + +`grpc-runtime`(`GrpcRequestFingerprint`, `GrpcResumeTokenCodec`)은 SHA-256·HMAC-SHA256 계산을 자기 +안에 둔다. 템플릿의 `shared-contract` `digest/Sha256`·`HmacSha256`을 쓰지 않는 것은 의도다. 이 +family는 optional build에 있는 vendored 플랫폼이고 registry edge는 `grpc:*` 안에서만 닫힌다(위 표). +`shared-contract` edge를 주면 플랫폼이 템플릿 계약 leaf에 묶여 따로 떼어 쓸 수 없게 된다 — messaging +family가 같은 이유로 고른 배치(MSG-023 §6.2)와 같다. 복사본을 고칠 때는 `shared-contract`의 정본과 +같은 규칙인지 함께 본다. + ## 증거 등급 — testkit이 하나의 leaf인 이유 원본 계획은 testkit을 core/in-process/netty/fault 4개 모듈로 쪼갠다. 목적은 "in-process 결과를 @@ -96,6 +105,9 @@ application-owned port → adapter:inbound:grpc (typed service adapter) → :grp 직접 닿지 않는 규칙은 app-bootstrap ArchUnit(`INBOUND_ADAPTERS_DO_NOT_DEPEND_ON_OUTBOUND_ADAPTERS`)이 검사한다. 현재 `adapter:inbound:grpc`는 이 family에 의존하지 않는다 — registry의 `allowed_dependencies`를 보라. 배선은 별도 결정이며, 그때 registry edge와 브리지가 함께 들어간다. +starter가 `operation-ledger-enabled=true`일 때 멱등성 판단 bean을 조립하는 것(03-11)은 이 배선이 아니다 — +starter는 서버를 만들지 않고, 그 bean을 서버에 붙이는 일은 채택하는 쪽의 몫이다(아래 「Durable operation +ledger adoption」). ## 테스트 명령 @@ -130,7 +142,54 @@ the repository's `.seconds` / `.total` unit naming and the corresponding validat ### Durable operation ledger adoption -`grpc-spring-boot-starter` does not pull the JPA ledger adapter transitively. A deployment that needs -durable operation idempotency adds `:grpc:grpc-operation-ledger-jpa` explicitly, enables the ledger -configuration, and includes that adapter's Flyway location `db/migration/grpc`. A deployment that -does not adopt the ledger therefore pays no Spring Data JPA classpath cost. +`grpc-spring-boot-starter` does not pull the JPA ledger adapter transitively (11-M9). A deployment that +needs durable operation idempotency adds `:grpc:grpc-operation-ledger-jpa` explicitly, defines a +`GrpcOperationLedger` bean (`new JpaGrpcOperationLedger(repository)`), includes that adapter's Flyway +location `db/migration/grpc` (V001 table, V002 retention index, V003 claim attempt), and sets the flag below. A deployment +that does not adopt the ledger pays no Spring Data JPA classpath cost. + +| Key (`app.grpc.platform.*`) | Default | What it does | +| --- | --- | --- | +| `operation-ledger-enabled` | `false` | `true`: the starter assembles `GrpcIdempotencyInterceptor` (bean `grpcIdempotencyInterceptor`) and `GrpcOperationLedgerRetention` (bean `grpcOperationLedgerRetention`) over the deployment's `GrpcOperationLedger` bean, and refuses startup when there is none. `false`: neither bean exists, and a method that is `IDEMPOTENCY_KEY_REQUIRED` refuses startup | +| `operation-ledger-claim-ttl` | `5m` | How long an `IN_PROGRESS` claim is trusted to have a live owner. After it, a retry of the same request takes the claim over (`GrpcOperationLedger.reclaimStale`). Refused at startup unless longer than every keyed method's deadline | +| `operation-ledger-retention` | `24h` | How long a finished (`COMMITTED`/`FAILED_TERMINAL`) record is kept before `GrpcOperationLedgerRetention.purgeExpired` may delete it. `IN_PROGRESS` rows are never purged | + +Defaults: the 5-minute TTL is the value the ledger contract and interceptor tests certify, far beyond +a unary deadline, and short enough that a key whose owner died unblocks within a client's retry +patience. The 24-hour retention matches the HTTP idempotency replay TTL +(`app.capabilities.idempotency.replay-ttl`). A retry of a key after its record was purged is a new +operation, so the retention must exceed the callers' retry window. + +These keys are not in `docs/registries/env-keys.yaml`: no `app.grpc.platform.*` key is, because the +optional build is not part of the shipped runtime. They join the registry when a composition root +adopts the platform. + +**Why the starter assembles the interceptor (03-11 (b)), and not (a).** The review proposed (a): +delete the flag and judge by whether a ledger bean exists. (b) was chosen and is recorded in +ADR-GRPC-004 (re-review 2026-11-03, RR-02-07): the flag stays and means what it says — when true, the +decision function is assembled and its TTL comes from configuration. (a) would have removed the false promise but left every adopter to rebuild the same +assembly and TTL validation by hand; (b) keeps that in one place. The interceptor bean has no +`@ConditionalOnMissingBean` on purpose: a replacement would silently ignore `operation-ledger-claim-ttl`. +Duplicates of a running operation are refused (`REJECT_IN_PROGRESS`), not told to wait; a +wait-and-poll option becomes a key when a keyed method needs it. + +**Server wiring is still the adopter's, and stays a separate decision.** The starter builds no +server, so assembling the decision does not wire the platform into `adapter:inbound:grpc` (see "배선은 +별도 결정" above). `GrpcIdempotencyInterceptor` is a decision function, not an `io.grpc.ServerInterceptor`. +The adopter injects the bean by type and calls it from its own `IDEMPOTENCY` stage interceptor +(`GrpcServerInterceptorChain.builder().stage(GrpcServerInterceptorStage.IDEMPOTENCY, ...)`) or from +its typed service adapter: `decide(...)` before the mutation, `recordCommit(...)` inside the mutation's +transaction, `recordFailure(...)` on a terminal failure. Both record calls take the `GrpcClaimToken` +the `PROCEED` decision carried: a commit by an attempt whose claim was taken over is refused (its +transaction rolls back) and its failure is ignored, so a slow owner cannot write over the attempt that +took its claim (re-review RR-02-04, ADR-GRPC-004). + +**Purge scheduling is the adopter's.** `GrpcOperationLedgerRetention.purgeExpired(now)` deletes +expired finished rows in bounded batches (500 per statement) until a batch comes back short. The +starter does not schedule it: it owns no scheduler and cannot know which replica should run it, and a +scheduled bean would force a task scheduler onto every deployment. The adopter calls it from its own +scheduler (for example hourly); concurrent runs on several replicas are safe because the delete skips +locked rows. Running it never is safe but lets the table grow without bound. + +The commit must be recorded in the mutation's transaction and the TTL must be longer than the slowest +keyed method; both reasons are in ADR-GRPC-004. diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcClaimToken.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcClaimToken.java new file mode 100644 index 0000000..269b00e --- /dev/null +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcClaimToken.java @@ -0,0 +1,40 @@ +package dev.caskeleton.grpc.api.ledger; + +/** + * Which attempt holds a claim: the fencing token a commit or a failure record must present. + * + *

A claim can change hands. A stale one is taken over after the claim TTL, and a failed one is + * re-claimed by the next retry; both make a new attempt of the same operation. The owner that lost + * the claim may still be running, and when it finishes it records its outcome. Without a token that + * record lands on the new attempt's claim: an old owner's failure turned the new attempt's claim + * {@code FAILED_TERMINAL}, a third retry re-claimed it, and two attempts ran the mutation at once + * (re-review 2026-11-03, RR-02-04). With it, the ledger records an outcome only for the attempt + * that still holds the claim. + * + *

A counter rather than the claim moment: it increases by one with every hand-over, so two + * attempts of one record never share a value, whatever the clocks of the processes that claimed it. + * The counter lives in the record: once the record is deleted ({@code release}, or the purge of a + * finished record), the next claim of the key starts again at {@link #FIRST}. Both deletions are + * therefore limited to records no live attempt can still write to — a released claim's owner is + * known to be gone, and a purged record finished longer ago than any retry (review 2026-11-03 + * RR-01-15). + * + * @param attempt one for the first claim of an operation, one more for each hand-over + */ +public record GrpcClaimToken(long attempt) { + + /** The token of the claim that created the record. */ + public static final GrpcClaimToken FIRST = new GrpcClaimToken(1); + + /** Requires a positive attempt number. */ + public GrpcClaimToken { + if (attempt < 1) { + throw new IllegalArgumentException("a claim attempt is numbered from one, was " + attempt); + } + } + + /** The token of the attempt that takes this claim over. */ + public GrpcClaimToken next() { + return new GrpcClaimToken(Math.addExact(attempt, 1)); + } +} diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java index 4342bde..19c239c 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java @@ -20,7 +20,8 @@ public interface GrpcOperationLedger { * implementation admits both concurrent attempts, which is the exact failure the ledger exists to * prevent, and it does so only under the load where it matters. * - * @return the existing record when one was already claimed, empty when this call created it + * @return the existing record when one was already claimed, empty when this call created it — the + * caller then holds the claim as {@link GrpcClaimToken#FIRST} */ Optional claim( GrpcOperationIdentity identity, String requestFingerprint, Instant now); @@ -34,11 +35,34 @@ public interface GrpcOperationLedger { *

Called inside the business transaction wherever the datastore allows it. When the ledger and * the mutation commit separately, there is a window in which the mutation is durable and the * claim is not, and a retry in that window runs the mutation twice. + * + *

Only the attempt that holds the claim commits it. An attempt whose claim was taken over or + * re-claimed since is refused with an exception, so its business transaction rolls back rather + * than committing beside the attempt that now owns the operation. Of two concurrent commits of + * one claim, exactly one succeeds. + * + * @param claimToken the token of the attempt committing, from the claim it won + * @throws IllegalStateException when there is no claim, it already finished, or it is no longer + * held by {@code claimToken} */ - void markCommitted(GrpcOperationIdentity identity, String outcomeReference, Instant now); + void markCommitted( + GrpcOperationIdentity identity, + GrpcClaimToken claimToken, + String outcomeReference, + Instant now); - /** Marks a claim terminally failed. */ - void markFailed(GrpcOperationIdentity identity, Instant now); + /** + * Marks a claim terminally failed, if {@code claimToken} still holds it. + * + *

An attempt that lost its claim reports a failure the operation no longer has: another + * attempt is running it. That record is ignored rather than written over the running attempt's + * claim. + * + * @param claimToken the token of the attempt that failed, from the claim it won + * @return true when the failure was recorded; false when the claim is missing, already finished, + * or held by another attempt, in which case nothing changed + */ + boolean markFailed(GrpcOperationIdentity identity, GrpcClaimToken claimToken, Instant now); /** * Re-claims a {@code FAILED_TERMINAL} record for a new attempt of the same request. @@ -48,15 +72,85 @@ public interface GrpcOperationLedger { * request, is left untouched. A retry that proceeds without winning this leaves its commit with * no owner and runs concurrently with every other retry of the same key. * - * @return true when this call now owns the record as a fresh attempt claimed at {@code now} + * @return the new attempt's token when this call now owns the record as a fresh attempt claimed + * at {@code now}; empty otherwise */ - boolean reclaimFailed(GrpcOperationIdentity identity, String requestFingerprint, Instant now); + Optional reclaimFailed( + GrpcOperationIdentity identity, String requestFingerprint, Instant now); /** - * Releases a claim that will never complete, so the caller may try again. + * Takes over an {@code IN_PROGRESS} claim whose owner is presumed dead, for a new attempt of the + * same request. + * + *

A claim made before {@code claimedBefore} has outlived the time its owner could still be + * working (the claim TTL), so its owner is treated as gone. Without this, a process that dies + * between claim and commit leaves its key blocked forever: every retry waits on, or is refused + * by, a claim nobody will finish. + * + *

A compare-and-set like {@link #reclaimFailed}: the record becomes a fresh {@code + * IN_PROGRESS} claim at {@code now}, and of two concurrent takeovers exactly one returns true — + * the winner's new claim moment is no longer before the cutoff, so the loser's condition fails. + * That is why {@code claimedBefore} after {@code now} is refused: the new claim would itself be + * stale and a second takeover would win too. A record in any other state, claimed at or after the + * cutoff, or made for a different request, is left untouched. + * + *

Safe only because the commit is recorded in the same transaction as the mutation (see {@link + * #markCommitted}): then a claim still {@code IN_PROGRESS} means the mutation never committed. + * When an adopter commits them separately, the abandoned attempt's outcome is unknown and a + * takeover can run the mutation twice. + * + *

The owner taken over may be slow rather than dead. Its later commit is refused and its later + * failure ignored, because it no longer holds the claim token; that is what keeps the takeover + * from becoming a second concurrent run. + * + * @return the new attempt's token when this call now owns the record as a fresh attempt claimed + * at {@code now}; empty otherwise + * @throws IllegalArgumentException when {@code claimedBefore} is after {@code now} + */ + Optional reclaimStale( + GrpcOperationIdentity identity, + String requestFingerprint, + Instant claimedBefore, + Instant now); + + /** + * Deletes up to {@code limit} finished records ({@code COMMITTED} or {@code FAILED_TERMINAL}) + * that completed before {@code completedBefore}, oldest first. + * + *

Retention, not recovery. A finished row only answers retries, so once it is older than any + * retry a caller will still send, keeping it only grows the table. Deleting it is a trade the + * deployment makes knowingly: a retry of the same key arriving after the purge is a fresh claim + * and runs the mutation again. The retention must therefore exceed the callers' retry window. + * + *

An {@code IN_PROGRESS} record is never deleted, however old. It is either running or + * abandoned, and an abandoned one is taken over by {@link #reclaimStale}, which keeps the record + * that an attempt started; deleting it would erase that. + * + *

Bounded so a backlog is drained in short transactions rather than one long delete that holds + * locks on the hottest table of the idempotency path. + * + * @return how many records were deleted, at most {@code limit} + * @throws IllegalArgumentException when {@code limit} is not positive + */ + int purgeCompletedBefore(Instant completedBefore, int limit); + + /** + * Releases a claim that will never complete, so the caller may try again: deletes the record if + * it is still {@code IN_PROGRESS} and {@code claimToken} still holds it. * *

Only for a claim whose owner is known to be gone. Releasing one whose owner is merely slow - * re-admits the duplicate. + * re-admits the duplicate. Fenced like every other transition (review 2026-11-03 RR-01-15): an + * attempt that lost the claim cannot delete the claim of the attempt that took it over, and a + * finished record — the answer a retry replays — is never released. + * + *

The record is gone afterwards, so the next claim of the key starts again at {@link + * GrpcClaimToken#FIRST}. An earlier attempt that lost the claim and is still running would then + * hold a token equal to the new claim's; releasing is safe only when no such attempt can still + * record an outcome — which is what "the owner is known to be gone" has to cover. + * + * @param claimToken the token of the attempt that holds the claim + * @return true when the record was deleted; false when it is missing, finished, or held by + * another attempt, in which case nothing changed */ - void release(GrpcOperationIdentity identity); + boolean release(GrpcOperationIdentity identity, GrpcClaimToken claimToken); } diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerRecord.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerRecord.java index 2bc0942..22f8632 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerRecord.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerRecord.java @@ -10,6 +10,9 @@ import java.util.Optional; * A caller that reuses a key for a different request has made a mistake, and returning the first * request's answer would hide it; comparing fingerprints turns that into a {@code * FAILED_PRECONDITION} the caller can act on. + * + *

{@code claimToken} names the attempt that holds (or last held) the claim; see {@link + * GrpcClaimToken}. */ public record GrpcOperationLedgerRecord( GrpcOperationIdentity identity, @@ -17,9 +20,10 @@ public record GrpcOperationLedgerRecord( String requestFingerprint, Optional outcomeReference, Instant claimedAt, - Optional completedAt) { + Optional completedAt, + GrpcClaimToken claimToken) { - /** Requires an identity, a state, a fingerprint and a claim moment. */ + /** Requires an identity, a state, a fingerprint, a claim moment and a claim token. */ public GrpcOperationLedgerRecord { if (identity == null || state == null) { throw new IllegalArgumentException("a ledger record needs an identity and a state"); @@ -30,6 +34,9 @@ public record GrpcOperationLedgerRecord( if (claimedAt == null || outcomeReference == null || completedAt == null) { throw new IllegalArgumentException("a ledger record needs its timestamps and Optionals"); } + if (claimToken == null) { + throw new IllegalArgumentException("a ledger record names the attempt that holds it"); + } if (state.terminal() && completedAt.isEmpty()) { throw new IllegalArgumentException("a terminal ledger record records when it finished"); } @@ -40,16 +47,26 @@ public record GrpcOperationLedgerRecord( } } - /** A fresh claim. */ + /** A fresh claim, held by the first attempt. */ public static GrpcOperationLedgerRecord claim( GrpcOperationIdentity identity, String requestFingerprint, Instant claimedAt) { + return claim(identity, requestFingerprint, claimedAt, GrpcClaimToken.FIRST); + } + + /** A claim in progress, held by the attempt {@code claimToken} names. */ + public static GrpcOperationLedgerRecord claim( + GrpcOperationIdentity identity, + String requestFingerprint, + Instant claimedAt, + GrpcClaimToken claimToken) { return new GrpcOperationLedgerRecord( identity, GrpcOperationLedgerState.IN_PROGRESS, requestFingerprint, Optional.empty(), claimedAt, - Optional.empty()); + Optional.empty(), + claimToken); } /** Whether {@code candidateFingerprint} is the same request this claim was made for. */ diff --git a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java index 1ff769c..ac84f97 100644 --- a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java +++ b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.grpc.core.GrpcMethodName; +import java.time.Duration; import java.time.Instant; import java.util.List; import java.util.Optional; @@ -12,6 +13,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.stream.Stream; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; /** @@ -34,6 +37,14 @@ public abstract class GrpcOperationLedgerContract { "sha256:1122334455667788990011223344556677889900112233445566778899001122"; private static final String OUTCOME = "outcome://document/42"; private static final Instant NOW = Instant.parse("2026-09-25T10:00:00Z"); + private static final Duration CLAIM_TTL = Duration.ofMinutes(5); + // A retry arriving after the TTL: its cutoff (LATER - TTL) is past the original claim at NOW. + private static final Instant LATER = NOW.plus(CLAIM_TTL).plusSeconds(1); + private static final Instant CUTOFF = LATER.minus(CLAIM_TTL); + // A purge run a day after NOW: rows finished around NOW are past retention. + private static final Instant RETENTION_CUTOFF = NOW.plus(Duration.ofDays(1)); + private static final GrpcOperationIdentity SECOND_ID = identityWithKeyHash('b'); + private static final GrpcOperationIdentity THIRD_ID = identityWithKeyHash('c'); /** The ledger under test, holding no records when a test starts. */ protected abstract GrpcOperationLedger ledger(); @@ -77,7 +88,7 @@ public abstract class GrpcOperationLedgerContract { ledger().claim(ID, FINGERPRINT, NOW); // when - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); // then assertThat(ledger().find(ID)) @@ -95,7 +106,8 @@ public abstract class GrpcOperationLedgerContract { @Test void committingWithoutAClaimIsRefused() { // when - Throwable refused = catchThrowable(() -> ledger().markCommitted(ID, OUTCOME, NOW)); + Throwable refused = + catchThrowable(() -> ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW)); // then assertThat(refused) @@ -107,12 +119,15 @@ public abstract class GrpcOperationLedgerContract { void aCommittedClaimCannotBeCommittedAgain() { // given ledger().claim(ID, FINGERPRINT, NOW); - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); // when Throwable refused = catchThrowable( - () -> ledger().markCommitted(ID, "outcome://document/99", NOW.plusSeconds(2))); + () -> + ledger() + .markCommitted( + ID, GrpcClaimToken.FIRST, "outcome://document/99", NOW.plusSeconds(2))); // then assertThat(refused) @@ -126,7 +141,7 @@ public abstract class GrpcOperationLedgerContract { ledger().claim(ID, FINGERPRINT, NOW); // when - ledger().markFailed(ID, NOW.plusSeconds(1)); + ledger().markFailed(ID, GrpcClaimToken.FIRST, NOW.plusSeconds(1)); // then assertThat(ledger().find(ID)) @@ -142,7 +157,8 @@ public abstract class GrpcOperationLedgerContract { // when Throwable refused = - catchThrowable(() -> ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(2))); + catchThrowable( + () -> ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(2))); // then assertThat(refused) @@ -156,10 +172,11 @@ public abstract class GrpcOperationLedgerContract { failedClaim(); // when - boolean reclaimed = ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + Optional reclaimed = + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); // then - assertThat(reclaimed).isTrue(); + assertThat(reclaimed).isPresent(); } @Test @@ -185,10 +202,11 @@ public abstract class GrpcOperationLedgerContract { void aReclaimedClaimCanBeCommitted() { // given failedClaim(); - ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + GrpcClaimToken reclaimed = + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)).orElseThrow(); // when - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(3)); + ledger().markCommitted(ID, reclaimed, OUTCOME, NOW.plusSeconds(3)); // then assertThat(ledger().find(ID)) @@ -201,27 +219,28 @@ public abstract class GrpcOperationLedgerContract { void aFailedClaimIsNotReclaimedForADifferentRequest() { failedClaim(); - assertThat(ledger().reclaimFailed(ID, OTHER_FINGERPRINT, NOW.plusSeconds(2))).isFalse(); + assertThat(ledger().reclaimFailed(ID, OTHER_FINGERPRINT, NOW.plusSeconds(2))).isEmpty(); } @Test void anInProgressClaimIsNotReclaimed() { ledger().claim(ID, FINGERPRINT, NOW); - assertThat(ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(1))).isFalse(); + assertThat(ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(1))).isEmpty(); } @Test void aCommittedClaimIsNotReclaimed() { // given ledger().claim(ID, FINGERPRINT, NOW); - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); // when - boolean reclaimed = ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + Optional reclaimed = + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); // then - assertThat(reclaimed).isFalse(); + assertThat(reclaimed).isEmpty(); } @Test @@ -231,15 +250,440 @@ public abstract class GrpcOperationLedgerContract { // when List reclaimed = - concurrently(() -> ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2))); + concurrently(() -> ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)).isPresent()); // then assertThat(reclaimed).containsExactlyInAnyOrder(true, false); } - private void failedClaim() { + @Test + @DisplayName("TTL이 지난 IN_PROGRESS claim은 같은 요청의 재시도가 넘겨받는다") + void aStaleInProgressClaimIsTakenOverByARetryOfTheSameRequest() { + // given ledger().claim(ID, FINGERPRINT, NOW); - ledger().markFailed(ID, NOW.plusSeconds(1)); + + // when + Optional reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // then + assertThat(reclaimed).isPresent(); + } + + @Test + @DisplayName("넘겨받은 stale claim은 넘겨받은 시각부터 진행 중인 새 시도다") + void aTakenOverStaleClaimIsANewAttemptInProgressSinceTheTakeover() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // then + assertThat(ledger().find(ID)) + .get() + .extracting( + GrpcOperationLedgerRecord::state, + GrpcOperationLedgerRecord::claimedAt, + GrpcOperationLedgerRecord::completedAt) + .containsExactly(GrpcOperationLedgerState.IN_PROGRESS, LATER, Optional.empty()); + } + + @Test + @DisplayName("넘겨받은 stale claim은 commit할 수 있다") + void aTakenOverStaleClaimCanBeCommitted() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + GrpcClaimToken takenOver = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER).orElseThrow(); + + // when + ledger().markCommitted(ID, takenOver, OUTCOME, LATER.plusSeconds(1)); + + // then + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::state, GrpcOperationLedgerRecord::outcomeReference) + .containsExactly(GrpcOperationLedgerState.COMMITTED, Optional.of(OUTCOME)); + } + + @Test + @DisplayName("TTL 안의 IN_PROGRESS claim은 넘겨받지 않는다") + void anInProgressClaimWithinTheTtlIsNotTakenOver() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + Optional reclaimed = + ledger().reclaimStale(ID, FINGERPRINT, NOW, NOW.plus(CLAIM_TTL)); + + // then + assertThat(reclaimed).isEmpty(); + } + + @Test + @DisplayName("다른 요청은 TTL이 지난 claim도 넘겨받지 못한다") + void aStaleClaimIsNotTakenOverForADifferentRequest() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + Optional reclaimed = + ledger().reclaimStale(ID, OTHER_FINGERPRINT, CUTOFF, LATER); + + // then + assertThat(reclaimed).isEmpty(); + } + + @Test + @DisplayName("commit된 claim은 오래되어도 stale로 넘겨받지 않는다") + void aCommittedClaimIsNotTakenOverAsStale() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); + + // when + Optional reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // then + assertThat(reclaimed).isEmpty(); + } + + @Test + @DisplayName("실패한 claim은 stale 회수가 아니라 실패 재claim으로만 다시 연다") + void aFailedClaimIsNotTakenOverAsStale() { + // given + failedClaim(); + + // when + Optional reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // then + assertThat(reclaimed).isEmpty(); + } + + @Test + @DisplayName("claim이 없으면 stale 회수는 아무것도 만들지 않는다") + void takingOverAMissingClaimCreatesNothing() { + // when + Optional reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // then + assertThat(reclaimed).isEmpty(); + assertThat(ledger().find(ID)).isEmpty(); + } + + @Test + @DisplayName("한 번 넘겨받은 claim은 같은 cutoff로 다시 넘겨받지 못한다") + void aTakenOverClaimIsNotTakenOverAgainWithTheSameCutoff() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // when + Optional again = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // then + assertThat(again).isEmpty(); + } + + @Test + @DisplayName("TTL이 지난 claim을 동시에 넘겨받으려 하면 승자는 하나다") + void concurrentTakeoversOfAStaleClaimHaveExactlyOneWinner() throws Exception { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + List reclaimed = + concurrently(() -> ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER).isPresent()); + + // then + assertThat(reclaimed).containsExactlyInAnyOrder(true, false); + } + + @Test + @DisplayName("cutoff가 현재 시각보다 뒤면 거부한다 — 새 claim이 곧바로 stale이 되기 때문이다") + void aCutoffAfterNowIsRefused() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + Throwable refused = catchThrowable(() -> ledger().reclaimStale(ID, FINGERPRINT, LATER, CUTOFF)); + + // then + assertThat(refused).isInstanceOf(IllegalArgumentException.class).hasMessageContaining("cutoff"); + } + + @Test + @DisplayName("claim 을 넘겨받을 때마다 claim token 은 다음 시도 번호가 된다") + void eachHandOverIssuesTheNextClaimToken() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + GrpcClaimToken takenOver = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER).orElseThrow(); + + // then + assertThat(takenOver).isEqualTo(GrpcClaimToken.FIRST.next()); + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::claimToken) + .isEqualTo(takenOver); + } + + /** + * Release deletes the row, so it is fenced like every other transition: only the attempt that + * holds an unfinished claim may give it up (review 2026-11-03 RR-01-15). + */ + @Test + @DisplayName("진행 중 claim 을 가진 시도가 release 하면 키가 비고 다음 claim 이 첫 시도가 된다") + void releaseByTheAttemptHoldingAnUnfinishedClaimFreesTheKey() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + boolean released = ledger().release(ID, GrpcClaimToken.FIRST); + + // then + assertThat(List.of(released, ledger().claim(ID, FINGERPRINT, NOW.plusSeconds(1)).isEmpty())) + .containsExactly(true, true); + } + + @Test + @DisplayName("claim 을 넘겨준 옛 시도의 release 는 무시되고 새 시도의 claim 이 남는다") + void releaseByTheAttemptThatLostTheClaimChangesNothing() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // when + boolean released = ledger().release(ID, GrpcClaimToken.FIRST); + + // then + assertThat(released).isFalse(); + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::state, GrpcOperationLedgerRecord::claimToken) + .containsExactly(GrpcOperationLedgerState.IN_PROGRESS, GrpcClaimToken.FIRST.next()); + } + + @Test + @DisplayName("끝난 claim 은 release 로 지워지지 않는다 — 재시도에 결과를 돌려줄 기록이다") + void releaseOfAFinishedClaimChangesNothing() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); + + // when + boolean released = ledger().release(ID, GrpcClaimToken.FIRST); + + // then + assertThat(released).isFalse(); + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::state) + .isEqualTo(GrpcOperationLedgerState.COMMITTED); + } + + /** + * The re-review probe's sequence (2026-11-03, RR-02-04): A claims and is slow, B takes the claim + * over after the TTL, then A reports a failure. Recording it would mark B's running attempt + * failed, and the next retry would re-claim it and run beside B. + */ + @Test + @DisplayName("TTL 뒤 넘겨받은 claim 에 옛 owner 가 실패를 기록하면 무시되고 새 시도는 진행 중으로 남는다") + void aFailureReportedByTheOwnerThatLostTheClaimIsIgnored() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // when + boolean recorded = ledger().markFailed(ID, GrpcClaimToken.FIRST, LATER.plusSeconds(1)); + + // then + assertThat(recorded).isFalse(); + assertThat(ledger().find(ID)) + .get() + .extracting( + GrpcOperationLedgerRecord::state, + GrpcOperationLedgerRecord::claimedAt, + GrpcOperationLedgerRecord::completedAt) + .containsExactly(GrpcOperationLedgerState.IN_PROGRESS, LATER, Optional.empty()); + } + + @Test + @DisplayName("옛 owner 의 실패가 무시되므로 다음 재시도는 진행 중인 새 시도를 다시 claim 하지 못한다") + void aRetryCannotReclaimTheRunningAttemptAfterTheLostOwnerFailed() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + ledger().markFailed(ID, GrpcClaimToken.FIRST, LATER.plusSeconds(1)); + + // when + Optional third = ledger().reclaimFailed(ID, FINGERPRINT, LATER.plusSeconds(2)); + + // then + assertThat(third).isEmpty(); + } + + @Test + @DisplayName("TTL 뒤 넘겨받은 claim 에 옛 owner 가 commit 하면 거부되고 새 시도는 진행 중으로 남는다") + void aCommitByTheOwnerThatLostTheClaimIsRefused() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // when + Throwable refused = + catchThrowable( + () -> ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, LATER.plusSeconds(1))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("no longer held by attempt 1"); + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::state) + .isEqualTo(GrpcOperationLedgerState.IN_PROGRESS); + } + + @Test + @DisplayName("실패 뒤 다시 claim 된 기록에 이전 시도가 commit 하면 거부된다") + void aCommitByTheAttemptBeforeAFailedReclaimIsRefused() { + // given + failedClaim(); + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + + // when + Throwable refused = + catchThrowable( + () -> ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(3))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("no longer held by attempt 1"); + } + + @Test + @DisplayName("한 claim 을 동시에 commit 하면 하나만 성공한다") + void concurrentCommitsOfOneClaimHaveExactlyOneWinner() throws Exception { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + List committed = + concurrently( + () -> { + try { + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); + return true; + } catch (IllegalStateException refused) { + return false; + } + }); + + // then + assertThat(committed).containsExactlyInAnyOrder(true, false); + } + + @Test + @DisplayName("보존 기간이 지난 COMMITTED·FAILED_TERMINAL 행은 정리된다") + void finishedRecordsCompletedBeforeTheCutoffArePurged() { + // given + committedClaim(ID, NOW); + failedClaim(SECOND_ID, NOW); + + // when + int purged = ledger().purgeCompletedBefore(RETENTION_CUTOFF, 10); + + // then + assertThat(purged).isEqualTo(2); + assertThat(ledger().find(ID)).isEmpty(); + assertThat(ledger().find(SECOND_ID)).isEmpty(); + } + + @Test + @DisplayName("IN_PROGRESS 행은 아무리 오래되어도 정리하지 않는다") + void anInProgressClaimIsNeverPurgedHoweverOld() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + int purged = ledger().purgeCompletedBefore(RETENTION_CUTOFF, 10); + + // then + assertThat(purged).isZero(); + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::state) + .isEqualTo(GrpcOperationLedgerState.IN_PROGRESS); + } + + @Test + @DisplayName("cutoff 이후에 끝난 행은 남긴다") + void finishedRecordsCompletedAtOrAfterTheCutoffAreKept() { + // given + committedClaim(ID, RETENTION_CUTOFF.minusSeconds(1)); + failedClaim(SECOND_ID, RETENTION_CUTOFF.minusSeconds(1)); + + // when + int purged = ledger().purgeCompletedBefore(RETENTION_CUTOFF, 10); + + // then + assertThat(purged).isZero(); + assertThat(ledger().find(ID)).isPresent(); + assertThat(ledger().find(SECOND_ID)).isPresent(); + } + + @Test + @DisplayName("한 번의 정리는 limit 개수까지만 지운다") + void onePurgeDeletesAtMostTheLimit() { + // given + committedClaim(ID, NOW); + committedClaim(SECOND_ID, NOW.plusSeconds(1)); + committedClaim(THIRD_ID, NOW.plusSeconds(2)); + + // when + int purged = ledger().purgeCompletedBefore(RETENTION_CUTOFF, 2); + + // then + assertThat(purged).isEqualTo(2); + assertThat( + Stream.of(ID, SECOND_ID, THIRD_ID) + .filter(identity -> ledger().find(identity).isPresent()) + .count()) + .isEqualTo(1); + } + + @Test + @DisplayName("limit이 양수가 아니면 정리를 거부한다") + void aNonPositivePurgeLimitIsRefused() { + // when + Throwable refused = catchThrowable(() -> ledger().purgeCompletedBefore(RETENTION_CUTOFF, 0)); + + // then + assertThat(refused).isInstanceOf(IllegalArgumentException.class).hasMessageContaining("limit"); + } + + private void failedClaim() { + failedClaim(ID, NOW); + } + + // Claimed at `claimedAt`, finished one second later, so the completion moment is what a purge + // cutoff is compared with. + private void failedClaim(GrpcOperationIdentity identity, Instant claimedAt) { + ledger().claim(identity, FINGERPRINT, claimedAt); + ledger().markFailed(identity, GrpcClaimToken.FIRST, claimedAt.plusSeconds(1)); + } + + private void committedClaim(GrpcOperationIdentity identity, Instant claimedAt) { + ledger().claim(identity, FINGERPRINT, claimedAt); + ledger().markCommitted(identity, GrpcClaimToken.FIRST, OUTCOME, claimedAt.plusSeconds(1)); + } + + private static GrpcOperationIdentity identityWithKeyHash(char hexDigit) { + return new GrpcOperationIdentity( + ID.callerFingerprint(), ID.method(), "sha256:" + String.valueOf(hexDigit).repeat(64)); } /** Runs {@code attempt} on two threads released together and returns both results. */ diff --git a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java index f08ddac..7aa3866 100644 --- a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java +++ b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java @@ -1,18 +1,23 @@ package dev.caskeleton.grpc.api.ledger; import java.time.Instant; +import java.util.Comparator; +import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; /** * A hand-rolled ledger for tests of code that depends on {@link GrpcOperationLedger}. * *

{@code putIfAbsent} on a concurrent map is the in-memory equivalent of the unique constraint - * the real adapter relies on, and {@code replace(key, expected, next)} of its conditional update, - * so claim and re-claim are atomic here for the same reason they are there. The transitions match - * the JPA adapter's because {@link GrpcOperationLedgerContract} runs against both; a fake that - * allowed more would let the policy tests pass for code the database refuses. + * the real adapter relies on, and {@code replace(key, expected, next)} and {@code computeIfPresent} + * of its conditional updates, so claim, re-claim, stale takeover, commit, failure and purge are + * atomic here for the same reason they are there. A commit or failure is recorded only for the + * attempt whose {@link GrpcClaimToken} still holds the claim. The transitions match the JPA + * adapter's because {@link GrpcOperationLedgerContract} runs against both; a fake that allowed more + * would let the policy tests pass for code the database refuses. */ public final class InMemoryOperationLedger implements GrpcOperationLedger { @@ -44,16 +49,19 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { } @Override - public void markCommitted(GrpcOperationIdentity identity, String outcomeReference, Instant now) { + public void markCommitted( + GrpcOperationIdentity identity, + GrpcClaimToken claimToken, + String outcomeReference, + Instant now) { GrpcOperationLedgerRecord committed = records.computeIfPresent( identity.storageKey(), - (key, existing) -> - finish( - existing, - GrpcOperationLedgerState.COMMITTED, - Optional.of(outcomeReference), - now)); + (key, existing) -> { + requireHeldBy(existing, claimToken); + return finish( + existing, GrpcOperationLedgerState.COMMITTED, Optional.of(outcomeReference), now); + }); if (committed == null) { throw new IllegalStateException( "cannot commit operation '" + identity.storageKey() + "': no claim exists"); @@ -61,29 +69,114 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { } @Override - public void markFailed(GrpcOperationIdentity identity, Instant now) { + public boolean markFailed( + GrpcOperationIdentity identity, GrpcClaimToken claimToken, Instant now) { + AtomicBoolean recorded = new AtomicBoolean(); records.computeIfPresent( identity.storageKey(), - (key, existing) -> - finish(existing, GrpcOperationLedgerState.FAILED_TERMINAL, Optional.empty(), now)); + (key, existing) -> { + if (existing.state() != GrpcOperationLedgerState.IN_PROGRESS + || !existing.claimToken().equals(claimToken)) { + return existing; + } + recorded.set(true); + return finish(existing, GrpcOperationLedgerState.FAILED_TERMINAL, Optional.empty(), now); + }); + return recorded.get(); } @Override - public boolean reclaimFailed( + public Optional reclaimFailed( GrpcOperationIdentity identity, String requestFingerprint, Instant now) { GrpcOperationLedgerRecord current = records.get(identity.storageKey()); - return current != null - && current.state() == GrpcOperationLedgerState.FAILED_TERMINAL - && current.sameRequestAs(requestFingerprint) - && records.replace( - identity.storageKey(), - current, - GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now)); + if (current == null + || current.state() != GrpcOperationLedgerState.FAILED_TERMINAL + || !current.sameRequestAs(requestFingerprint)) { + return Optional.empty(); + } + return handOver(identity, requestFingerprint, current, now); } @Override - public void release(GrpcOperationIdentity identity) { - records.remove(identity.storageKey()); + public Optional reclaimStale( + GrpcOperationIdentity identity, + String requestFingerprint, + Instant claimedBefore, + Instant now) { + if (claimedBefore.isAfter(now)) { + throw new IllegalArgumentException( + "a stale-claim cutoff after now would leave the new claim stale as well"); + } + GrpcOperationLedgerRecord current = records.get(identity.storageKey()); + if (current == null + || current.state() != GrpcOperationLedgerState.IN_PROGRESS + || !current.claimedAt().isBefore(claimedBefore) + || !current.sameRequestAs(requestFingerprint)) { + return Optional.empty(); + } + return handOver(identity, requestFingerprint, current, now); + } + + // The conditional replace is the compare-and-set: of two hand-overs of the same record, one wins. + private Optional handOver( + GrpcOperationIdentity identity, + String requestFingerprint, + GrpcOperationLedgerRecord current, + Instant now) { + GrpcClaimToken next = current.claimToken().next(); + return records.replace( + identity.storageKey(), + current, + GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now, next)) + ? Optional.of(next) + : Optional.empty(); + } + + @Override + public int purgeCompletedBefore(Instant completedBefore, int limit) { + if (limit < 1) { + throw new IllegalArgumentException("a purge limit must be positive"); + } + List expired = + records.values().stream() + .filter(record -> record.state().terminal()) + .filter(record -> record.completedAt().orElseThrow().isBefore(completedBefore)) + .sorted(Comparator.comparing(record -> record.completedAt().orElseThrow())) + .limit(limit) + .toList(); + int purged = 0; + for (GrpcOperationLedgerRecord record : expired) { + // Conditional, like the database's re-checked DELETE: a row re-claimed since it was read is + // IN_PROGRESS again and stays. + if (records.remove(record.identity().storageKey(), record)) { + purged++; + } + } + return purged; + } + + @Override + public boolean release(GrpcOperationIdentity identity, GrpcClaimToken claimToken) { + GrpcOperationLedgerRecord existing = records.get(identity.storageKey()); + return existing != null + && existing.state() == GrpcOperationLedgerState.IN_PROGRESS + && existing.claimToken().equals(claimToken) + && records.remove(identity.storageKey(), existing); + } + + // An attempt that lost the claim must not commit beside the attempt that holds it; checked + // before the state, so a lost owner hears that it lost rather than that the row finished. + private static void requireHeldBy(GrpcOperationLedgerRecord existing, GrpcClaimToken claimToken) { + if (!existing.claimToken().equals(claimToken)) { + throw new IllegalStateException( + "operation '" + + existing.identity().storageKey() + + "' is no longer held by attempt " + + claimToken.attempt() + + "; attempt " + + existing.claimToken().attempt() + + " holds it"); + } } // Only an IN_PROGRESS claim finishes; a terminal row is re-opened by reclaimFailed alone. @@ -106,6 +199,7 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { existing.requestFingerprint(), outcomeReference, existing.claimedAt(), - Optional.of(now)); + Optional.of(now), + existing.claimToken()); } } diff --git a/grpc/grpc-operation-ledger-jpa/build.gradle b/grpc/grpc-operation-ledger-jpa/build.gradle index 9ecb4dd..05d90ae 100644 --- a/grpc/grpc-operation-ledger-jpa/build.gradle +++ b/grpc/grpc-operation-ledger-jpa/build.gradle @@ -16,6 +16,30 @@ configurations { integrationTestRuntimeOnly.extendsFrom testRuntimeOnly } +// One definition of a versioned migration file and of Flyway's order for both builds (re-review +// README 12, r6-b; r9). This build cannot depend on persistence-jpa's testkit, so the one file the +// ledger lanes need from it — VersionedMigrationFiles, JDK only — is copied from there into this +// lane's sources at build time instead of being kept here as a second copy that could drift. +// Rejected: that testkit directory as a source root of this lane (Spotless refuses a target outside +// this project's directory, and the include filter would cover this lane's own sources too), a +// dependency on the main build (it would pull the whole main build into this one), and a test that +// compares two copies (two definitions stay, and the test would have to parse Java source text). +def sharedMigrationFileSources = tasks.register('sharedMigrationFileSources', Sync) { + from(file('../../adapter/outbound/persistence-jpa/src/testFixtures/java')) { + include 'dev/caskeleton/adapter/outbound/persistence/testkit/migration/VersionedMigrationFiles.java' + } + into(layout.buildDirectory.dir('generated/sources/sharedMigrationFiles/java')) +} +sourceSets.integrationTest.java.srcDir(sharedMigrationFileSources) +// The copy is not this build's to format: Spotless here would name the generated file, and its +// spotlessApply would rewrite only that copy, which the next Sync puts back (re-review 2026-11-03 +// round 6, RR6-03-04). The main build's persistence-jpa spotlessCheck formats the original. +spotless { + java { + targetExclude('build/generated/sources/sharedMigrationFiles/**') + } +} + // Durable mutation idempotency: the operation ledger entity, its state machine, the vendor-neutral // repository port and the Spring Data JPA binding, plus the migration that owns the unique // constraint the whole contract rests on. diff --git a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrationLockWaitIT.java b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrationLockWaitIT.java new file mode 100644 index 0000000..723fe1d --- /dev/null +++ b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrationLockWaitIT.java @@ -0,0 +1,185 @@ +package dev.caskeleton.grpc.ledger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.postgresql.ds.PGSimpleDataSource; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.postgresql.PostgreSQLContainer; + +/** + * The ledger migrations after V001 lock a table already in use and give up within their bound + * instead of stopping it (re-review 2026-11-03 round 2, RR-02-09: the rule V21 follows). + * + *

A transaction that has written the ledger stays open for fifteen seconds — a claim in flight, + * an operator's {@code BEGIN}. A write is the holder both migrations wait for: V003's ALTER TABLE + * would wait for a read too, V002's plain CREATE INDEX only for a write. The migration runs in one + * transaction, which is how Flyway runs the files this tree has: a {@code SET LOCAL lock_timeout} + * and statements that run in a transaction. Unbounded it waits until the writer leaves, with every + * claim queued behind it, and then applies; bounded it fails with {@code lock_not_available} first + * and its rollback leaves the ledger as it was. + * + *

A file Flyway runs outside a transaction — one holding {@code CREATE INDEX CONCURRENTLY} or + * another statement on Flyway's list, or with {@code executeInTransaction=false} beside it — is not + * one this test can judge: run here in a transaction, PostgreSQL refuses it ({@code 25001}), and no + * {@code SET LOCAL} would bound it in a deploy anyway. Name such a file as an exception in this + * test, and keep it alone in its migration ({@code docs/jpa/migration-guide.md}, "Concurrent index + * builds"; re-review 2026-11-03 round 6, RR6-03-05). + * + *

Every migration after V001 is held to this, read from the tree itself ({@link + * GrpcOperationLedgerMigrations#inOrder()}), so a V004 is checked without anyone naming it here. + * This is the ledger tree's lock-wait rule: the optional build cannot depend on the persistence-jpa + * testkit whose text check the other trees run (it compiles only the testkit's file list, {@code + * VersionedMigrationFiles}), and a copy of that check would drift (re-review 2026-11-03 round 3, + * RR3-02-04). A later migration that does not touch the ledger at all fails here too, on purpose: + * say so in this test rather than let the rule pass it unseen. + */ +@Testcontainers(disabledWithoutDocker = true) +class GrpcOperationLedgerMigrationLockWaitIT { + + private static final String LOCK_NOT_AVAILABLE = "55P03"; + private static final String ACTIVE_SQL_TRANSACTION = "25001"; + private static final Duration WRITER_STAYS = Duration.ofSeconds(15); + private static final Duration GIVES_UP_WITHIN = Duration.ofSeconds(10); + + @SuppressWarnings("resource") + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer(System.getProperty("ca.test.image.postgres-16")) + .withDatabaseName("grpc") + .withUsername("grpc") + .withPassword("grpc"); + + /** Every ledger migration after V001, the one that creates the ledger. */ + static Stream migrationsAfterTheFirst() throws IOException { + return GrpcOperationLedgerMigrations.inOrder().stream().skip(1); + } + + @ParameterizedTest + @MethodSource("migrationsAfterTheFirst") + @DisplayName("ledger에 쓴 트랜잭션이 열려 있으면 ledger를 잠그는 마이그레이션은 상한 안에 실패하고 아무것도 바꾸지 않는다") + void aLedgerMigrationGivesUpOnAnOpenWriterAndChangesNothing(String migration) throws Exception { + PGSimpleDataSource dataSource = dataSource(); + try (Connection setup = dataSource.getConnection(); + Statement statement = setup.createStatement()) { + // given + statement.execute("DROP TABLE IF EXISTS grpc_operation_ledger"); + statement.execute(GrpcOperationLedgerMigrations.before(migration)); + String schemaBefore = schema(statement); + + // when + Attempt attempt = + whileAWriterHolds(dataSource, GrpcOperationLedgerMigrations.named(migration)); + + // then + assertThat( + List.of( + String.valueOf(attempt.sqlState()), + attempt.took().compareTo(GIVES_UP_WITHIN) < 0, + schema(statement))) + .as( + "%s must put SET LOCAL lock_timeout = '5s'; before it locks the ledger" + + " (docs/jpa/migration-guide.md, \"Bound the lock wait\")%s", + migration, + ACTIVE_SQL_TRANSACTION.equals(attempt.sqlState()) + ? ". PostgreSQL refused it inside a transaction (25001): Flyway runs such a file" + + " outside one, where SET LOCAL bounds nothing, so it is no case for this" + + " test. Keep it alone in its migration and name it as an exception here" + + " (\"Concurrent index builds\")" + : "") + .containsExactly(LOCK_NOT_AVAILABLE, true, schemaBefore); + } + } + + /** Runs the migration in one transaction while another one holds a write to the ledger. */ + private static Attempt whileAWriterHolds(PGSimpleDataSource dataSource, String migration) + throws SQLException { + try (Connection writer = dataSource.getConnection(); + Connection migrator = dataSource.getConnection()) { + writer.setAutoCommit(false); + try (Statement write = writer.createStatement()) { + write.execute("LOCK TABLE grpc_operation_ledger IN ROW EXCLUSIVE MODE"); + } + ScheduledExecutorService release = Executors.newSingleThreadScheduledExecutor(); + ScheduledFuture released = + release.schedule( + () -> { + writer.rollback(); + return null; + }, + WRITER_STAYS.toSeconds(), + TimeUnit.SECONDS); + try { + migrator.setAutoCommit(false); + long started = System.nanoTime(); + Throwable thrown = + catchThrowable( + () -> { + try (Statement statement = migrator.createStatement()) { + statement.execute(migration); + } + migrator.commit(); + }); + Duration took = Duration.ofNanos(System.nanoTime() - started); + migrator.rollback(); + return new Attempt(thrown, took); + } finally { + released.cancel(true); + release.shutdownNow(); + } + } + } + + /** The ledger's columns, indexes and constraints, enough to see whether a migration landed. */ + private static String schema(Statement statement) throws SQLException { + try (ResultSet row = + statement.executeQuery( + "SELECT (SELECT string_agg(column_name, ',' ORDER BY column_name)" + + " FROM information_schema.columns" + + " WHERE table_name = 'grpc_operation_ledger')" + + " || '|' || (SELECT string_agg(indexname, ',' ORDER BY indexname)" + + " FROM pg_indexes WHERE tablename = 'grpc_operation_ledger')" + + " || '|' || (SELECT string_agg(conname, ',' ORDER BY conname) FROM pg_constraint" + + " WHERE conrelid = 'grpc_operation_ledger'::regclass)")) { + row.next(); + return row.getString(1); + } + } + + private static PGSimpleDataSource dataSource() { + PGSimpleDataSource dataSource = new PGSimpleDataSource(); + dataSource.setUrl(POSTGRES.getJdbcUrl()); + dataSource.setUser(POSTGRES.getUsername()); + dataSource.setPassword(POSTGRES.getPassword()); + return dataSource; + } + + /** What the migration did while the writer held the ledger. */ + private record Attempt(Throwable thrown, Duration took) { + + String sqlState() { + for (Throwable cause = thrown; cause != null; cause = cause.getCause()) { + if (cause instanceof SQLException sql) { + return sql.getSQLState(); + } + } + return null; + } + } +} diff --git a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java new file mode 100644 index 0000000..f8832fc --- /dev/null +++ b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java @@ -0,0 +1,76 @@ +package dev.caskeleton.grpc.ledger; + +import dev.caskeleton.adapter.outbound.persistence.testkit.migration.VersionedMigrationFiles; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; + +/** + * The ledger's Flyway scripts in version order, as one SQL text for a test database. + * + *

One list for every PostgreSQL lane, so a new migration is applied by all of them or by none. + * The list is the tree on disk, not a copy of it: a hand-kept list let a V004 that nobody added to + * it go unapplied and unchecked by every lane (re-review 2026-11-03 round 3, RR3-02-04). Which file + * is a versioned migration, and in what order Flyway applies them, is the persistence-jpa testkit's + * {@link VersionedMigrationFiles}, the definition the main build's trees use; this lane compiles + * that file from there ({@code sharedMigrationFileSources} in this leaf's {@code build.gradle}) + * rather than keep a second regular expression of its own (re-review README 12, r6-b; r9). + */ +final class GrpcOperationLedgerMigrations { + + /** The ledger's tree from this leaf's directory, where the lanes run. */ + private static final Path TREE = Path.of("src/main/resources/db/migration/grpc"); + + /** The same tree on the classpath, where the scripts are read from. */ + private static final String LOCATION = "db/migration/grpc/"; + + private GrpcOperationLedgerMigrations() {} + + /** + * The file name of every versioned migration in the tree ({@code V__.sql}), in + * Flyway's order: by version, its parts compared as numbers. + */ + static List inOrder() throws IOException { + return VersionedMigrationFiles.inVersionOrder(TREE).stream() + .map(file -> file.getFileName().toString()) + .toList(); + } + + /** Every migration's SQL, concatenated in version order. */ + static String all() throws IOException { + return before(null); + } + + /** The SQL of every migration before the one named, concatenated in version order. */ + static String before(String fileName) throws IOException { + StringBuilder sql = new StringBuilder(); + for (String name : inOrder()) { + if (name.equals(fileName)) { + break; + } + sql.append(read(name)).append('\n'); + } + return sql.toString(); + } + + /** One migration's SQL, by its file name. */ + static String named(String fileName) throws IOException { + if (!inOrder().contains(fileName)) { + throw new IOException("no gRPC operation-ledger migration is named " + fileName); + } + return read(fileName); + } + + private static String read(String fileName) throws IOException { + String path = LOCATION + fileName; + try (InputStream stream = + GrpcOperationLedgerMigrations.class.getClassLoader().getResourceAsStream(path)) { + if (stream == null) { + throw new IOException("gRPC operation-ledger migration is missing: " + path); + } + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + } +} diff --git a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerPostgresIT.java b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerPostgresIT.java index 98fd2f2..df1b00b 100644 --- a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerPostgresIT.java +++ b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerPostgresIT.java @@ -3,8 +3,6 @@ package dev.caskeleton.grpc.ledger; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -41,7 +39,7 @@ class GrpcOperationLedgerPostgresIT { @SuppressWarnings("resource") @Container private static final PostgreSQLContainer POSTGRES = - new PostgreSQLContainer("postgres:16-alpine") + new PostgreSQLContainer(System.getProperty("ca.test.image.postgres-16")) .withDatabaseName("grpc") .withUsername("grpc") .withPassword("grpc"); @@ -146,14 +144,6 @@ class GrpcOperationLedgerPostgresIT { } private static String migration() throws IOException { - try (InputStream stream = - GrpcOperationLedgerPostgresIT.class - .getClassLoader() - .getResourceAsStream("db/migration/grpc/V001__create_grpc_operation_ledger.sql")) { - if (stream == null) { - throw new IOException("gRPC operation-ledger migration is missing"); - } - return new String(stream.readAllBytes(), StandardCharsets.UTF_8); - } + return GrpcOperationLedgerMigrations.all(); } } diff --git a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java index 2d1e6b2..b4b645d 100644 --- a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java +++ b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java @@ -2,6 +2,7 @@ package dev.caskeleton.grpc.ledger; import static org.assertj.core.api.Assertions.assertThat; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerContract; @@ -11,8 +12,6 @@ import dev.caskeleton.grpc.core.GrpcMethodName; import jakarta.persistence.EntityManagerFactory; import jakarta.persistence.PersistenceConfiguration; import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -22,6 +21,7 @@ import java.util.Optional; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.data.jpa.repository.support.JpaRepositoryFactory; import org.springframework.orm.jpa.JpaTransactionManager; @@ -43,7 +43,7 @@ class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract { @SuppressWarnings("resource") @Container private static final PostgreSQLContainer POSTGRES = - new PostgreSQLContainer("postgres:16-alpine") + new PostgreSQLContainer(System.getProperty("ca.test.image.postgres-16")) .withDatabaseName("grpc") .withUsername("grpc") .withPassword("grpc"); @@ -103,14 +103,15 @@ class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract { String fingerprint = "sha256:0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0"; Instant now = Instant.parse("2026-09-25T10:00:00Z"); ledger.claim(id, fingerprint, now); - ledger.markFailed(id, now.plusSeconds(1)); + ledger.markFailed(id, GrpcClaimToken.FIRST, now.plusSeconds(1)); // when transactions.executeWithoutResult( status -> { adapter.claim(id, fingerprint, now.plusSeconds(2)); - adapter.reclaimFailed(id, fingerprint, now.plusSeconds(2)); - adapter.markCommitted(id, "outcome://document/42", now.plusSeconds(3)); + GrpcClaimToken reclaimed = + adapter.reclaimFailed(id, fingerprint, now.plusSeconds(2)).orElseThrow(); + adapter.markCommitted(id, reclaimed, "outcome://document/42", now.plusSeconds(3)); }); // then @@ -120,6 +121,47 @@ class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract { .containsExactly(GrpcOperationLedgerState.COMMITTED, Optional.of("outcome://document/42")); } + /** + * The takeover's conditional update clears the persistence context for the same reason the failed + * re-claim does: the claim's read-on-conflict loaded the stale row, and a commit reading that + * copy would keep the abandoned claim moment. + */ + @Test + @DisplayName("stale claim을 넘겨받아 같은 트랜잭션에서 commit하면 commit된다") + void aRetryThatTakesOverAStaleClaimAndCommitsInOneTransactionCommits() { + // given + GrpcOperationIdentity id = + new GrpcOperationIdentity( + "tenant-1.actor-1", + GrpcMethodName.parse("caskeleton.document.v1.DocumentService/CreateDocument"), + "sha256:aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899"); + String fingerprint = "sha256:0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0"; + Instant abandonedAt = Instant.parse("2026-09-25T10:00:00Z"); + Instant retryAt = abandonedAt.plusSeconds(301); + ledger.claim(id, fingerprint, abandonedAt); + + // when + transactions.executeWithoutResult( + status -> { + adapter.claim(id, fingerprint, retryAt); + GrpcClaimToken takenOver = + adapter + .reclaimStale(id, fingerprint, retryAt.minusSeconds(300), retryAt) + .orElseThrow(); + adapter.markCommitted(id, takenOver, "outcome://document/42", retryAt.plusSeconds(1)); + }); + + // then + assertThat(ledger.find(id)) + .get() + .extracting( + GrpcOperationLedgerRecord::state, + GrpcOperationLedgerRecord::claimedAt, + GrpcOperationLedgerRecord::outcomeReference) + .containsExactly( + GrpcOperationLedgerState.COMMITTED, retryAt, Optional.of("outcome://document/42")); + } + private static void execute(String sql) throws SQLException { try (Connection connection = DriverManager.getConnection( @@ -130,15 +172,7 @@ class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract { } private static String migration() throws IOException { - try (InputStream stream = - JpaGrpcOperationLedgerContractIT.class - .getClassLoader() - .getResourceAsStream("db/migration/grpc/V001__create_grpc_operation_ledger.sql")) { - if (stream == null) { - throw new IOException("gRPC operation-ledger migration is missing"); - } - return new String(stream.readAllBytes(), StandardCharsets.UTF_8); - } + return GrpcOperationLedgerMigrations.all(); } /** Runs each call in its own transaction, as a caller's business transaction would. */ @@ -159,27 +193,49 @@ class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract { @Override public void markCommitted( - GrpcOperationIdentity identity, String outcomeReference, Instant now) { + GrpcOperationIdentity identity, + GrpcClaimToken claimToken, + String outcomeReference, + Instant now) { transactions.executeWithoutResult( - status -> delegate.markCommitted(identity, outcomeReference, now)); + status -> delegate.markCommitted(identity, claimToken, outcomeReference, now)); } @Override - public void markFailed(GrpcOperationIdentity identity, Instant now) { - transactions.executeWithoutResult(status -> delegate.markFailed(identity, now)); - } - - @Override - public boolean reclaimFailed( - GrpcOperationIdentity identity, String requestFingerprint, Instant now) { + public boolean markFailed( + GrpcOperationIdentity identity, GrpcClaimToken claimToken, Instant now) { return Boolean.TRUE.equals( - transactions.execute( - status -> delegate.reclaimFailed(identity, requestFingerprint, now))); + transactions.execute(status -> delegate.markFailed(identity, claimToken, now))); } @Override - public void release(GrpcOperationIdentity identity) { - transactions.executeWithoutResult(status -> delegate.release(identity)); + public Optional reclaimFailed( + GrpcOperationIdentity identity, String requestFingerprint, Instant now) { + return transactions.execute( + status -> delegate.reclaimFailed(identity, requestFingerprint, now)); + } + + @Override + public Optional reclaimStale( + GrpcOperationIdentity identity, + String requestFingerprint, + Instant claimedBefore, + Instant now) { + return transactions.execute( + status -> delegate.reclaimStale(identity, requestFingerprint, claimedBefore, now)); + } + + @Override + public int purgeCompletedBefore(Instant completedBefore, int limit) { + Integer purged = + transactions.execute(status -> delegate.purgeCompletedBefore(completedBefore, limit)); + return purged == null ? 0 : purged; + } + + @Override + public boolean release(GrpcOperationIdentity identity, GrpcClaimToken claimToken) { + return Boolean.TRUE.equals( + transactions.execute(status -> delegate.release(identity, claimToken))); } } } diff --git a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerEntity.java b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerEntity.java index 54e26ab..571f36e 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerEntity.java +++ b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerEntity.java @@ -1,5 +1,6 @@ package dev.caskeleton.grpc.ledger; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerState; @@ -56,6 +57,9 @@ public class GrpcOperationLedgerEntity { @Column(name = "completed_at") private Instant completedAt; + @Column(name = "claim_attempt", nullable = false) + private long claimAttempt; + /** JPA requires a no-argument constructor. */ protected GrpcOperationLedgerEntity() { // for JPA @@ -70,6 +74,7 @@ public class GrpcOperationLedgerEntity { this.requestFingerprint = requestFingerprint; this.state = GrpcOperationLedgerState.IN_PROGRESS; this.claimedAt = claimedAt; + this.claimAttempt = GrpcClaimToken.FIRST.attempt(); } /** A new claim, in progress. */ @@ -114,13 +119,26 @@ public class GrpcOperationLedgerEntity { return Optional.ofNullable(completedAt); } + /** The attempt that holds, or last held, the claim. */ + public GrpcClaimToken getClaimToken() { + return new GrpcClaimToken(claimAttempt); + } + /** - * Records a commit with its replayable outcome. + * Records a commit with its replayable outcome, on a row this code already holds. * - * @throws IllegalStateException when the claim already finished; a second terminal transition - * would overwrite the outcome a retry is entitled to receive + *

The row-level statement of the rule. {@link JpaGrpcOperationLedger} applies the same rule as + * one conditional {@code UPDATE} ({@link GrpcOperationLedgerRepository#tryMarkCommitted}), so + * that the database, not a value read earlier, decides between concurrent transitions. + * + * @param claimToken the attempt committing + * @throws IllegalStateException when another attempt holds the claim, or the claim already + * finished; a second terminal transition would overwrite the outcome a retry is entitled to + * receive */ - public void markCommitted(String outcomeReference, Instant completedAt) { + public void markCommitted( + GrpcClaimToken claimToken, String outcomeReference, Instant completedAt) { + requireHeldBy(claimToken); requireInProgress(); if (outcomeReference == null || outcomeReference.isBlank()) { throw new IllegalArgumentException( @@ -131,8 +149,16 @@ public class GrpcOperationLedgerEntity { this.completedAt = completedAt; } - /** Records a terminal failure. */ - public void markFailed(Instant completedAt) { + /** + * Records a terminal failure, on a row this code already holds; the row-level statement of {@link + * GrpcOperationLedgerRepository#tryMarkFailed}. + * + * @param claimToken the attempt that failed + * @throws IllegalStateException when another attempt holds the claim, or the claim already + * finished + */ + public void markFailed(GrpcClaimToken claimToken, Instant completedAt) { + requireHeldBy(claimToken); requireInProgress(); this.state = GrpcOperationLedgerState.FAILED_TERMINAL; this.completedAt = completedAt; @@ -146,7 +172,21 @@ public class GrpcOperationLedgerEntity { requestFingerprint, Optional.ofNullable(outcomeReference), claimedAt, - Optional.ofNullable(completedAt)); + Optional.ofNullable(completedAt), + getClaimToken()); + } + + private void requireHeldBy(GrpcClaimToken claimToken) { + if (claimAttempt != claimToken.attempt()) { + throw new IllegalStateException( + "operation '" + + storageKey + + "' is no longer held by attempt " + + claimToken.attempt() + + "; attempt " + + claimAttempt + + " holds it"); + } } private void requireInProgress() { diff --git a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java index f1823b8..c24507f 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java +++ b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java @@ -11,10 +11,12 @@ import org.springframework.transaction.annotation.Transactional; /** * The narrow persistence interface the ledger adapter needs. * - *

The claim path is deliberately not {@code save}. A claim entity already has a non-null string - * id, so Spring Data JPA treats it as detached and routes {@code save} through {@code merge}; merge - * can overwrite a committed row back to {@code IN_PROGRESS}. Claiming therefore uses one native - * insert with PostgreSQL conflict handling and decides ownership from the affected-row count. + *

No path here is {@code save}. A claim entity already has a non-null string id, so Spring Data + * JPA treats it as detached and routes {@code save} through {@code merge}; merge can overwrite a + * committed row back to {@code IN_PROGRESS}, and a load-check-save commit lets two attempts that + * both read {@code IN_PROGRESS} both write. Claiming is therefore one native insert with PostgreSQL + * conflict handling, and every transition is one conditional update; ownership and every race are + * decided by the affected-row count. */ public interface GrpcOperationLedgerRepository extends Repository { @@ -31,7 +33,8 @@ public interface GrpcOperationLedgerRepository state, outcome_reference, claimed_at, - completed_at + completed_at, + claim_attempt ) VALUES ( :storageKey, :callerFingerprint, @@ -41,7 +44,8 @@ public interface GrpcOperationLedgerRepository 'IN_PROGRESS', NULL, :claimedAt, - NULL + NULL, + 1 ) ON CONFLICT DO NOTHING """; @@ -71,7 +75,11 @@ public interface GrpcOperationLedgerRepository * claim's read-on-conflict does) would otherwise still read {@code FAILED_TERMINAL} and refuse * the commit. * - * @return one when this caller re-claimed the row, zero otherwise + *

Conditional on the attempt the caller read as well, and the new attempt is that one plus + * one: the caller then knows its own fencing token without reading the row again. + * + * @return one when this caller re-claimed the row as attempt {@code readAttempt + 1}, zero + * otherwise */ @Modifying(flushAutomatically = true, clearAutomatically = true) @Transactional @@ -81,25 +89,164 @@ public interface GrpcOperationLedgerRepository UPDATE grpc_operation_ledger SET state = 'IN_PROGRESS', claimed_at = :claimedAt, - completed_at = NULL + completed_at = NULL, + claim_attempt = claim_attempt + 1 WHERE storage_key = :storageKey AND state = 'FAILED_TERMINAL' AND request_fingerprint = :requestFingerprint + AND claim_attempt = :readAttempt """, nativeQuery = true) int tryReclaimFailed( @Param("storageKey") String storageKey, @Param("requestFingerprint") String requestFingerprint, + @Param("readAttempt") long readAttempt, @Param("claimedAt") Instant claimedAt); - /** Updates an existing claim after it has been loaded. */ - GrpcOperationLedgerEntity save(GrpcOperationLedgerEntity entity); + /** + * Takes over an {@code IN_PROGRESS} row claimed before {@code claimedBefore}, for the same + * request. + * + *

Same shape as {@link #tryReclaimFailed}: one conditional update decided by PostgreSQL's row + * lock. A concurrent second takeover waits for the winner, re-evaluates the condition against the + * winner's row — whose {@code claimed_at} is now {@code :claimedAt}, not before the cutoff — and + * updates nothing. It clears the persistence context for the same reason. The attempt grows by + * one, as in {@link #tryReclaimFailed}, and that is what fences the owner taken over. + * + * @return one when this caller took the row over as attempt {@code readAttempt + 1}, zero + * otherwise + */ + @Modifying(flushAutomatically = true, clearAutomatically = true) + @Transactional + @Query( + value = + """ + UPDATE grpc_operation_ledger + SET claimed_at = :claimedAt, + claim_attempt = claim_attempt + 1 + WHERE storage_key = :storageKey + AND state = 'IN_PROGRESS' + AND request_fingerprint = :requestFingerprint + AND claimed_at < :claimedBefore + AND claim_attempt = :readAttempt + """, + nativeQuery = true) + int tryReclaimStale( + @Param("storageKey") String storageKey, + @Param("requestFingerprint") String requestFingerprint, + @Param("readAttempt") long readAttempt, + @Param("claimedBefore") Instant claimedBefore, + @Param("claimedAt") Instant claimedAt); + + /** + * Commits the claim if attempt {@code claimAttempt} still holds it. + * + *

One conditional update rather than load-check-save: two commits of the same claim both read + * {@code IN_PROGRESS} under load-check-save and both wrote; here PostgreSQL's row lock makes the + * second re-evaluate the condition against the first's {@code COMMITTED} row and update nothing. + * An attempt whose claim was taken over or re-claimed matches nothing for the same reason. + * + * @return one when the claim was committed, zero otherwise + */ + @Modifying(flushAutomatically = true, clearAutomatically = true) + @Transactional + @Query( + value = + """ + UPDATE grpc_operation_ledger + SET state = 'COMMITTED', + outcome_reference = :outcomeReference, + completed_at = :completedAt + WHERE storage_key = :storageKey + AND state = 'IN_PROGRESS' + AND claim_attempt = :claimAttempt + """, + nativeQuery = true) + int tryMarkCommitted( + @Param("storageKey") String storageKey, + @Param("claimAttempt") long claimAttempt, + @Param("outcomeReference") String outcomeReference, + @Param("completedAt") Instant completedAt); + + /** + * Marks the claim terminally failed if attempt {@code claimAttempt} still holds it; the same + * conditional shape as {@link #tryMarkCommitted}. + * + * @return one when the failure was recorded, zero otherwise + */ + @Modifying(flushAutomatically = true, clearAutomatically = true) + @Transactional + @Query( + value = + """ + UPDATE grpc_operation_ledger + SET state = 'FAILED_TERMINAL', + completed_at = :completedAt + WHERE storage_key = :storageKey + AND state = 'IN_PROGRESS' + AND claim_attempt = :claimAttempt + """, + nativeQuery = true) + int tryMarkFailed( + @Param("storageKey") String storageKey, + @Param("claimAttempt") long claimAttempt, + @Param("completedAt") Instant completedAt); + + /** + * Deletes up to {@code limit} finished rows that completed before {@code completedBefore}, oldest + * first. Never an {@code IN_PROGRESS} row. + * + *

{@code FOR UPDATE SKIP LOCKED} in the bounded subquery keeps the purge out of the way of a + * concurrent {@link #tryReclaimFailed}: a row that re-claim holds is skipped, and one it already + * turned back into {@code IN_PROGRESS} no longer matches. The outer state condition says the same + * thing again so the statement cannot delete a live claim even if the subquery is edited. The + * partial index {@code ix_grpc_operation_ledger_finished_completed_at} (V002) serves the + * subquery's filter and order. + * + * @return how many rows were deleted + */ + @Modifying(flushAutomatically = true, clearAutomatically = true) + @Transactional + @Query( + value = + """ + DELETE FROM grpc_operation_ledger + WHERE storage_key IN ( + SELECT storage_key + FROM grpc_operation_ledger + WHERE state IN ('COMMITTED', 'FAILED_TERMINAL') + AND completed_at < :completedBefore + ORDER BY completed_at + LIMIT :limit + FOR UPDATE SKIP LOCKED) + AND state <> 'IN_PROGRESS' + """, + nativeQuery = true) + int deleteFinishedBefore( + @Param("completedBefore") Instant completedBefore, @Param("limit") int limit); /** Reads a claim by its derived storage key. */ Optional findById(String storageKey); - /** Removes a claim. Only for one whose owner is known to be gone. */ - void deleteById(String storageKey); + /** + * Deletes the claim if attempt {@code claimAttempt} still holds it and it has not finished; the + * same fencing as {@link #tryMarkFailed}. Only for a claim whose owner is known to be gone + * (review 2026-11-03 RR-01-15). + * + * @return one when the claim was deleted, zero otherwise + */ + @Modifying(flushAutomatically = true, clearAutomatically = true) + @Transactional + @Query( + value = + """ + DELETE FROM grpc_operation_ledger + WHERE storage_key = :storageKey + AND state = 'IN_PROGRESS' + AND claim_attempt = :claimAttempt + """, + nativeQuery = true) + int tryRelease(@Param("storageKey") String storageKey, @Param("claimAttempt") long claimAttempt); /** Whether a claim exists. */ boolean existsById(String storageKey); diff --git a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java index 7cd1afc..5901103 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java +++ b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java @@ -1,5 +1,6 @@ package dev.caskeleton.grpc.ledger; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; @@ -16,6 +17,11 @@ import java.util.Optional; * {@code save}: this entity has a non-null application-assigned id, so save routes through merge * and can overwrite an existing terminal row. * + *

Every transition — re-claim, takeover, commit, failure — is one conditional {@code UPDATE} + * that names the claim attempt ({@link GrpcClaimToken}). An attempt whose claim was handed over + * matches nothing: its commit is refused and its failure ignored, which is what keeps a takeover of + * a slow owner from becoming two concurrent runs (re-review 2026-11-03, RR-02-04). + * *

No transaction annotations here. The commit is meant to happen inside the caller's business * transaction, and a {@code REQUIRES_NEW} on this adapter would put the ledger in its own * transaction — reintroducing the window in which the mutation is durable and the claim is not. @@ -36,6 +42,28 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { public Optional claim( GrpcOperationIdentity identity, String requestFingerprint, Instant now) { requireIdentity(identity); + // Twice at most. Zero rows means another claim holds the key; an empty read after that means + // the row went in between (a purge of an expired record, or a release). Treating that as + // "ours" would run the mutation without a claim row to guard it, so the insert is tried once + // more and decides again. + for (int round = 0; round < 2; round++) { + if (insertClaim(identity, requestFingerprint, now)) { + return Optional.empty(); + } + Optional existing = repository.findById(identity.storageKey()); + if (existing.isPresent()) { + return Optional.of(existing.get().toRecord(identity)); + } + } + throw new IllegalStateException( + "claim for '" + + identity.storageKey() + + "' conflicted twice, but there was no row to read either time; another owner keeps" + + " releasing it. Retry the claim."); + } + + private boolean insertClaim( + GrpcOperationIdentity identity, String requestFingerprint, Instant now) { int inserted = repository.tryInsertClaim( identity.storageKey(), @@ -44,25 +72,10 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { identity.idempotencyKeyHash(), requestFingerprint, now); - if (inserted == 1) { - return Optional.empty(); - } - if (inserted != 0) { + if (inserted != 0 && inserted != 1) { throw new IllegalStateException("claim insert affected an unexpected row count: " + inserted); } - // Zero rows means another claim holds the key. An empty read now means that owner released it - // in between; treating that as "ours" would run the mutation without a claim row to guard it. - return Optional.of( - repository - .findById(identity.storageKey()) - .orElseThrow( - () -> - new IllegalStateException( - "claim for '" - + identity.storageKey() - + "' conflicted, but there was no row to read; the other owner released" - + " it in between. Retry the claim.")) - .toRecord(identity)); + return inserted == 1; } @Override @@ -71,10 +84,30 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { return repository.findById(identity.storageKey()).map(entity -> entity.toRecord(identity)); } + /** + * Commits the claim if {@code claimToken} still holds it, as one conditional update. + * + *

When nothing matched, the row is read only to say why; the refusal itself was the + * database's, and the exception is what rolls back the caller's business transaction. + */ @Override - public void markCommitted(GrpcOperationIdentity identity, String outcomeReference, Instant now) { + public void markCommitted( + GrpcOperationIdentity identity, + GrpcClaimToken claimToken, + String outcomeReference, + Instant now) { requireIdentity(identity); - GrpcOperationLedgerEntity entity = + requireToken(claimToken); + if (outcomeReference == null || outcomeReference.isBlank()) { + throw new IllegalArgumentException( + "a commit records the outcome to replay; without it the row cannot answer a retry"); + } + if (repository.tryMarkCommitted( + identity.storageKey(), claimToken.attempt(), outcomeReference, now) + == 1) { + return; + } + GrpcOperationLedgerEntity current = repository .findById(identity.storageKey()) .orElseThrow( @@ -84,33 +117,100 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { + identity.storageKey() + "': no claim exists. The claim is what a retry reads; committing " + "without one leaves the mutation durable and unguarded.")); - entity.markCommitted(outcomeReference, now); - repository.save(entity); + if (!current.getClaimToken().equals(claimToken)) { + throw new IllegalStateException( + "operation '" + + identity.storageKey() + + "' is no longer held by attempt " + + claimToken.attempt() + + "; attempt " + + current.getClaimToken().attempt() + + " holds it, so this attempt's transaction must roll back"); + } + throw new IllegalStateException( + "operation '" + + identity.storageKey() + + "' is already " + + current.getState() + + " and cannot transition again"); } @Override - public void markFailed(GrpcOperationIdentity identity, Instant now) { + public boolean markFailed( + GrpcOperationIdentity identity, GrpcClaimToken claimToken, Instant now) { requireIdentity(identity); - repository - .findById(identity.storageKey()) - .ifPresent( - entity -> { - entity.markFailed(now); - repository.save(entity); - }); + requireToken(claimToken); + return repository.tryMarkFailed(identity.storageKey(), claimToken.attempt(), now) == 1; } + /** + * Re-claims a failed record. Conditional on the attempt read just before, so the new token is + * known exactly: that attempt plus one. A record that changed in between matches nothing, and + * this caller has lost the race it would have lost anyway. + */ @Override - public boolean reclaimFailed( + public Optional reclaimFailed( GrpcOperationIdentity identity, String requestFingerprint, Instant now) { requireIdentity(identity); - return repository.tryReclaimFailed(identity.storageKey(), requestFingerprint, now) == 1; + Optional read = currentToken(identity); + if (read.isEmpty()) { + return Optional.empty(); + } + return repository.tryReclaimFailed( + identity.storageKey(), requestFingerprint, read.get().attempt(), now) + == 1 + ? Optional.of(read.get().next()) + : Optional.empty(); } @Override - public void release(GrpcOperationIdentity identity) { + public Optional reclaimStale( + GrpcOperationIdentity identity, + String requestFingerprint, + Instant claimedBefore, + Instant now) { requireIdentity(identity); - repository.deleteById(identity.storageKey()); + if (claimedBefore.isAfter(now)) { + throw new IllegalArgumentException( + "a stale-claim cutoff after now would leave the new claim stale as well"); + } + Optional read = currentToken(identity); + if (read.isEmpty()) { + return Optional.empty(); + } + return repository.tryReclaimStale( + identity.storageKey(), requestFingerprint, read.get().attempt(), claimedBefore, now) + == 1 + ? Optional.of(read.get().next()) + : Optional.empty(); + } + + private Optional currentToken(GrpcOperationIdentity identity) { + return repository.findById(identity.storageKey()).map(GrpcOperationLedgerEntity::getClaimToken); + } + + @Override + public int purgeCompletedBefore(Instant completedBefore, int limit) { + if (completedBefore == null) { + throw new IllegalArgumentException("a purge needs a cutoff"); + } + if (limit < 1) { + throw new IllegalArgumentException("a purge limit must be positive"); + } + return repository.deleteFinishedBefore(completedBefore, limit); + } + + @Override + public boolean release(GrpcOperationIdentity identity, GrpcClaimToken claimToken) { + requireIdentity(identity); + requireToken(claimToken); + return repository.tryRelease(identity.storageKey(), claimToken.attempt()) == 1; + } + + private static void requireToken(GrpcClaimToken claimToken) { + if (claimToken == null) { + throw new IllegalArgumentException("a transition names the attempt that holds the claim"); + } } private static void requireIdentity(GrpcOperationIdentity identity) { diff --git a/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V002__index_grpc_operation_ledger_finished_rows.sql b/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V002__index_grpc_operation_ledger_finished_rows.sql new file mode 100644 index 0000000..3c6adef --- /dev/null +++ b/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V002__index_grpc_operation_ledger_finished_rows.sql @@ -0,0 +1,29 @@ +-- Retention of finished ledger rows (review 2026-11-01, 03-10 follow-up). +-- +-- GrpcOperationLedgerRepository.deleteFinishedBefore deletes COMMITTED and FAILED_TERMINAL rows +-- whose completed_at is past the retention, oldest first and a bounded batch at a time. V001's +-- (state, claimed_at) index orders by claim time, not completion time, so without this index every +-- purge batch scans and sorts all finished rows. +-- +-- Partial: IN_PROGRESS rows have no completed_at and are never purged, so they stay out of the +-- index. The predicate repeats the query's state list literally so the planner can match it. +-- +-- A plain CREATE INDEX, not CONCURRENTLY: Flyway runs each migration in a transaction, and the +-- platform is not adopted anywhere yet, so the table is empty when this runs. An adopter that +-- applies it to a large existing ledger should run the CONCURRENTLY form by hand first. +-- +-- This file was edited after it was first written, before any deployment applied it (the branch +-- that added it was not merged, and the ledger is adopted nowhere yet), to add the bound below. A +-- development database that applied the earlier text fails Flyway validation: recreate it. + +-- Bound the wait for the lock this migration takes (SHARE on grpc_operation_ledger, held until it +-- commits; it stops writes, not reads). Unbounded, it queues behind any open transaction that has +-- written to the table, and every write of the table after it queues behind the migration — every +-- claim writes it. Five seconds, then the migration fails, rolls back whole and the database stays +-- at V001: deploy again once that transaction has ended (re-review 2026-11-03 round 2, RR-02-09; +-- the rule is docs/jpa/migration-guide.md "Bound the lock wait"). +SET LOCAL lock_timeout = '5s'; + +CREATE INDEX ix_grpc_operation_ledger_finished_completed_at + ON grpc_operation_ledger (completed_at) + WHERE state IN ('COMMITTED', 'FAILED_TERMINAL'); diff --git a/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql b/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql new file mode 100644 index 0000000..89007ad --- /dev/null +++ b/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql @@ -0,0 +1,33 @@ +-- Claim attempts: the fencing token of the operation ledger (re-review 2026-11-03, RR-02-04). +-- +-- A claim changes hands when a stale one is taken over after the claim TTL or a failed one is +-- re-claimed. The owner that lost it may still be running, and when it finishes it records an +-- outcome. Without knowing which attempt holds the claim, that record landed on the new attempt's +-- claim: an old owner's failure turned a running attempt FAILED_TERMINAL, the next retry re-claimed +-- it, and two attempts ran the mutation at once. +-- +-- claim_attempt is 1 for the first claim and grows by one with every hand-over. A commit or a +-- failure is one conditional UPDATE that names the attempt (WHERE state = 'IN_PROGRESS' AND +-- claim_attempt = :attempt), so a lost owner's record changes nothing and two commits of the same +-- attempt have one winner. +-- +-- Existing rows are their first attempt: no row has been handed over under a token yet. The table is +-- small and the platform is not adopted anywhere, so a plain ALTER with a default is enough. +-- +-- This file was edited after it was first written, before any deployment applied it (the branch +-- that added it was not merged, and the ledger is adopted nowhere yet), to add the bound below. A +-- development database that applied the earlier text fails Flyway validation: recreate it. + +-- Bound the wait for the lock this migration takes (ACCESS EXCLUSIVE on grpc_operation_ledger, held +-- until it commits). Unbounded, it queues behind any open transaction that has touched the table, +-- and every read and write of the table after it queues behind the migration — every claim reads +-- and writes it. Five seconds, then the migration fails, rolls back whole and the database stays at +-- V002: deploy again once that transaction has ended (re-review 2026-11-03 round 2, RR-02-09; the +-- rule is docs/jpa/migration-guide.md "Bound the lock wait"). +SET LOCAL lock_timeout = '5s'; + +ALTER TABLE grpc_operation_ledger + ADD COLUMN claim_attempt BIGINT NOT NULL DEFAULT 1; + +ALTER TABLE grpc_operation_ledger + ADD CONSTRAINT ck_grpc_operation_ledger_claim_attempt_positive CHECK (claim_attempt > 0); diff --git a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java index 8da7c64..37c0478 100644 --- a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java +++ b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; @@ -15,6 +16,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; +import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -34,12 +36,12 @@ class GrpcOperationLedgerRepositoryTest { "sha256:aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899"; /** - * An in-memory stand-in for Spring Data's non-null-id save semantics. + * An in-memory stand-in for the repository's native statements. * - *

A newly constructed ledger entity already has its string id, so a real {@code - * SimpleJpaRepository.save} treats it as detached and merges it. The old fake used {@code - * putIfAbsent} and therefore tested the database constraint we wished save() would hit rather - * than the merge behavior it actually uses. + *

The insert is {@code putIfAbsent}, the stand-in for the unique constraint. The commit and + * failure are the entity's own transition rules, which state the conditions the native {@code + * UPDATE}s carry (state {@code IN_PROGRESS} and the same claim attempt); a refused transition + * updates zero rows. The re-claims and the purge are certified against PostgreSQL only. */ private static class InMemoryRepository implements GrpcOperationLedgerRepository { private final ConcurrentMap rows = new ConcurrentHashMap<>(); @@ -61,15 +63,56 @@ class GrpcOperationLedgerRepositoryTest { } @Override - public int tryReclaimFailed(String storageKey, String requestFingerprint, Instant claimedAt) { + public int tryReclaimFailed( + String storageKey, String requestFingerprint, long readAttempt, Instant claimedAt) { throw new UnsupportedOperationException( "the re-claim is certified against PostgreSQL by JpaGrpcOperationLedgerContractIT"); } @Override - public GrpcOperationLedgerEntity save(GrpcOperationLedgerEntity entity) { - rows.put(entity.getStorageKey(), entity); - return entity; + public int tryReclaimStale( + String storageKey, + String requestFingerprint, + long readAttempt, + Instant claimedBefore, + Instant claimedAt) { + throw new UnsupportedOperationException( + "the stale takeover is certified against PostgreSQL by JpaGrpcOperationLedgerContractIT"); + } + + @Override + public int deleteFinishedBefore(Instant completedBefore, int limit) { + throw new UnsupportedOperationException( + "the bounded purge is certified against PostgreSQL by JpaGrpcOperationLedgerContractIT"); + } + + @Override + public int tryMarkCommitted( + String storageKey, long claimAttempt, String outcomeReference, Instant completedAt) { + GrpcOperationLedgerEntity row = rows.get(storageKey); + if (row == null) { + return 0; + } + try { + row.markCommitted(new GrpcClaimToken(claimAttempt), outcomeReference, completedAt); + return 1; + } catch (IllegalStateException refused) { + return 0; + } + } + + @Override + public int tryMarkFailed(String storageKey, long claimAttempt, Instant completedAt) { + GrpcOperationLedgerEntity row = rows.get(storageKey); + if (row == null) { + return 0; + } + try { + row.markFailed(new GrpcClaimToken(claimAttempt), completedAt); + return 1; + } catch (IllegalStateException refused) { + return 0; + } } @Override @@ -78,8 +121,13 @@ class GrpcOperationLedgerRepositoryTest { } @Override - public void deleteById(String storageKey) { - rows.remove(storageKey); + public int tryRelease(String storageKey, long claimAttempt) { + GrpcOperationLedgerEntity row = rows.get(storageKey); + boolean held = + row != null + && row.getState() == GrpcOperationLedgerState.IN_PROGRESS + && row.getClaimToken().equals(new GrpcClaimToken(claimAttempt)); + return held && rows.remove(storageKey, row) ? 1 : 0; } @Override @@ -126,6 +174,7 @@ class GrpcOperationLedgerRepositoryTest { } @Test + @DisplayName("충돌한 행이 다시 시도해도 계속 사라지면 소유로 보지 않고 거부한다") void aConflictWhoseRowVanishedBeforeTheReadIsNotTakenAsOwnership() { // given InMemoryRepository conflictThenReleased = @@ -153,6 +202,54 @@ class GrpcOperationLedgerRepositoryTest { .hasMessageContaining("no row to read"); } + /** + * A purge (or release) can delete the conflicting row between the insert and the read. The read + * then finds nothing, and the claim tries the insert once more instead of failing the call + * (re-review 2026-11-03, RR-02-04). + */ + @Test + @DisplayName("충돌한 행이 읽기 전에 사라지면 claim 은 insert 를 한 번 더 시도해 소유한다") + void aConflictWhoseRowVanishedOnceIsClaimedByTheRetriedInsert() { + // given + InMemoryRepository conflictOnceThenFree = + new InMemoryRepository() { + private boolean conflicted; + + @Override + public int tryInsertClaim( + String storageKey, + String callerFingerprint, + String fullMethodName, + String idempotencyKeyHash, + String requestFingerprint, + Instant claimedAt) { + if (!conflicted) { + conflicted = true; + return 0; // a row held the key and was purged before this read + } + return super.tryInsertClaim( + storageKey, + callerFingerprint, + fullMethodName, + idempotencyKeyHash, + requestFingerprint, + claimedAt); + } + }; + GrpcOperationLedger racing = new JpaGrpcOperationLedger(conflictOnceThenFree); + + // when + Optional existing = + racing.claim(identity("tenant-1.actor-1"), FINGERPRINT, NOW); + + // then + assertThat(existing).isEmpty(); + assertThat(racing.find(identity("tenant-1.actor-1"))) + .get() + .extracting(GrpcOperationLedgerRecord::state, GrpcOperationLedgerRecord::claimToken) + .containsExactly(GrpcOperationLedgerState.IN_PROGRESS, GrpcClaimToken.FIRST); + } + @Test @DisplayName("one caller's key does not collide with another's") void callersDoNotCollide() { @@ -166,7 +263,7 @@ class GrpcOperationLedgerRepositoryTest { GrpcOperationIdentity id = identity("tenant-1.actor-1"); ledger.claim(id, FINGERPRINT, NOW); - ledger.markCommitted(id, "outcome://document/42", NOW.plusSeconds(1)); + ledger.markCommitted(id, GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); GrpcOperationLedgerRecord record = ledger.find(id).orElseThrow(); assertThat(record.state()).isEqualTo(GrpcOperationLedgerState.COMMITTED); @@ -179,7 +276,7 @@ class GrpcOperationLedgerRepositoryTest { void retryingACommittedClaimPreservesTheWinner() { GrpcOperationIdentity id = identity("tenant-1.actor-1"); assertThat(ledger.claim(id, FINGERPRINT, NOW)).isEmpty(); - ledger.markCommitted(id, "outcome://document/42", NOW.plusSeconds(1)); + ledger.markCommitted(id, GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); GrpcOperationLedgerRecord retry = ledger.claim(id, OTHER_FINGERPRINT, NOW.plusSeconds(2)).orElseThrow(); @@ -193,7 +290,12 @@ class GrpcOperationLedgerRepositoryTest { @DisplayName("a commit without a claim is refused rather than silently inserted") void committingWithoutAClaimIsRefused() { assertThatThrownBy( - () -> ledger.markCommitted(identity("tenant-1.actor-1"), "outcome://document/42", NOW)) + () -> + ledger.markCommitted( + identity("tenant-1.actor-1"), + GrpcClaimToken.FIRST, + "outcome://document/42", + NOW)) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("durable and unguarded"); } @@ -203,9 +305,12 @@ class GrpcOperationLedgerRepositoryTest { void aTerminalRowDoesNotTransitionAgain() { GrpcOperationIdentity id = identity("tenant-1.actor-1"); ledger.claim(id, FINGERPRINT, NOW); - ledger.markCommitted(id, "outcome://document/42", NOW.plusSeconds(1)); + ledger.markCommitted(id, GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); - assertThatThrownBy(() -> ledger.markCommitted(id, "outcome://document/99", NOW.plusSeconds(2))) + assertThatThrownBy( + () -> + ledger.markCommitted( + id, GrpcClaimToken.FIRST, "outcome://document/99", NOW.plusSeconds(2))) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("cannot transition again"); } @@ -221,18 +326,24 @@ class GrpcOperationLedgerRepositoryTest { assertThat(stored.sameRequestAs(OTHER_FINGERPRINT)).isFalse(); } + /** + * A failed record is finished: a retry re-claims it with a new token ({@code reclaimFailed}), so + * release, which deletes and resets the token, leaves it (review 2026-11-03 RR-01-15). + */ @Test - @DisplayName("a failed claim is terminal, and releasing removes it") - void failureIsTerminalAndReleaseRemoves() { + @DisplayName("실패한 claim 은 끝난 기록이라 release 로 지워지지 않는다") + void aFailedClaimIsTerminalAndReleaseLeavesIt() { + // given GrpcOperationIdentity id = identity("tenant-1.actor-1"); ledger.claim(id, FINGERPRINT, NOW); + ledger.markFailed(id, GrpcClaimToken.FIRST, NOW.plusSeconds(1)); - ledger.markFailed(id, NOW.plusSeconds(1)); - assertThat(ledger.find(id).orElseThrow().state()) - .isEqualTo(GrpcOperationLedgerState.FAILED_TERMINAL); + // when + boolean released = ledger.release(id, GrpcClaimToken.FIRST); - ledger.release(id); - assertThat(ledger.find(id)).isEmpty(); + // then + assertThat(List.of(released, ledger.find(id).orElseThrow().state())) + .containsExactly(false, GrpcOperationLedgerState.FAILED_TERMINAL); } @Test @@ -246,7 +357,8 @@ class GrpcOperationLedgerRepositoryTest { FINGERPRINT, Optional.empty(), NOW, - Optional.of(NOW))) + Optional.of(NOW), + GrpcClaimToken.FIRST)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("cannot be replayed"); } diff --git a/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyDecision.java b/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyDecision.java index ecc9f40..ca7ebef 100644 --- a/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyDecision.java +++ b/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyDecision.java @@ -1,5 +1,6 @@ package dev.caskeleton.grpc.idempotency; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import java.time.Duration; import java.util.Optional; @@ -10,9 +11,18 @@ import java.util.Optional; * duplicate". A duplicate of a committed operation should be answered with the original result; a * duplicate of a running one is a race that has to wait or be refused; and a key reused for a * different request is a caller bug that must be reported rather than absorbed. + * + *

A {@code PROCEED} that won a claim carries the claim's {@link GrpcClaimToken}. The caller + * hands it back to {@code recordCommit} or {@code recordFailure}: the ledger records an outcome + * only for the attempt that still holds the claim, so a caller whose claim was taken over cannot + * write over the attempt that took it (re-review 2026-11-03, RR-02-04). */ public record GrpcIdempotencyDecision( - Outcome outcome, Optional outcomeReference, Duration retryAfter, String reason) { + Outcome outcome, + Optional outcomeReference, + Duration retryAfter, + String reason, + Optional claimToken) { /** The five ways an idempotency check can end. */ public enum Outcome { @@ -48,11 +58,31 @@ public record GrpcIdempotencyDecision( if (outcome == Outcome.WAIT_AND_POLL && retryAfter.isZero()) { throw new IllegalArgumentException("a WAIT_AND_POLL decision says how long to wait"); } + if (claimToken == null) { + throw new IllegalArgumentException("the claim token Optional must be present"); + } + if (outcome != Outcome.PROCEED && claimToken.isPresent()) { + throw new IllegalArgumentException("only a PROCEED decision can hold a claim"); + } } - /** Proceed with the operation. */ + /** Proceed with an operation that is not claimed: the method takes no idempotency key. */ public static GrpcIdempotencyDecision proceed(String reason) { - return new GrpcIdempotencyDecision(Outcome.PROCEED, Optional.empty(), Duration.ZERO, reason); + return new GrpcIdempotencyDecision( + Outcome.PROCEED, Optional.empty(), Duration.ZERO, reason, Optional.empty()); + } + + /** + * Proceed as the attempt that holds the claim. + * + * @param claimToken the token to hand back when recording the commit or the failure + */ + public static GrpcIdempotencyDecision proceedHolding(GrpcClaimToken claimToken, String reason) { + if (claimToken == null) { + throw new IllegalArgumentException("a claimed PROCEED carries its claim token"); + } + return new GrpcIdempotencyDecision( + Outcome.PROCEED, Optional.empty(), Duration.ZERO, reason, Optional.of(claimToken)); } /** Replay a stored outcome. */ @@ -61,13 +91,18 @@ public record GrpcIdempotencyDecision( Outcome.REPLAY, Optional.of(outcomeReference), Duration.ZERO, - "an identical request already committed; returning the stored outcome"); + "an identical request already committed; returning the stored outcome", + Optional.empty()); } /** Wait and poll for an in-progress claim. */ public static GrpcIdempotencyDecision waitAndPoll(Duration retryAfter) { return new GrpcIdempotencyDecision( - Outcome.WAIT_AND_POLL, Optional.empty(), retryAfter, "an identical request is in progress"); + Outcome.WAIT_AND_POLL, + Optional.empty(), + retryAfter, + "an identical request is in progress", + Optional.empty()); } /** Refuse a duplicate of an in-progress claim. */ @@ -76,7 +111,8 @@ public record GrpcIdempotencyDecision( Outcome.REJECT_IN_PROGRESS, Optional.empty(), Duration.ZERO, - "an identical request is in progress and this method does not wait for it"); + "an identical request is in progress and this method does not wait for it", + Optional.empty()); } /** Report a key reused for a different request. */ @@ -85,7 +121,8 @@ public record GrpcIdempotencyDecision( Outcome.FINGERPRINT_MISMATCH, Optional.empty(), Duration.ZERO, - "the idempotency key was already used for a different request"); + "the idempotency key was already used for a different request", + Optional.empty()); } /** Whether the use case should run. */ diff --git a/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java b/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java index e232669..5387ffa 100644 --- a/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java +++ b/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java @@ -1,5 +1,6 @@ package dev.caskeleton.grpc.idempotency; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; @@ -19,6 +20,12 @@ import java.util.Optional; * client takes the same one before deciding whether a retry is safe. An interceptor that owns the * logic leaves the client re-deriving it. * + *

A claim does not block its key forever. One still {@code IN_PROGRESS} after the claim TTL is + * treated as abandoned by a dead owner and taken over by the next retry of the same request — one + * winner, like the re-claim of a terminal failure. The owner taken over may only have been slow; it + * still holds the old {@link GrpcClaimToken}, so its later commit is refused and its later failure + * ignored instead of landing on the attempt that took the claim. + * *

The ordering matters and is easy to get subtly wrong. The claim is attempted first, * atomically; only then is the existing record examined. Reading first and claiming second leaves a * window in which two attempts both read nothing. @@ -29,18 +36,24 @@ public final class GrpcIdempotencyInterceptor { private final GrpcMethodPolicyCatalog catalog; private final Duration inProgressPollInterval; private final boolean waitForInProgress; + private final Duration claimTtl; /** * @param inProgressPollInterval how long a duplicate is told to wait before asking again * @param waitForInProgress whether a duplicate of a running operation waits or is refused. A * method whose caller is a user-facing request usually wants to be refused quickly; a worker * usually wants to wait. + * @param claimTtl how long an {@code IN_PROGRESS} claim is trusted to have a live owner. After + * it, a retry of the same request takes the claim over. It must be longer than every keyed + * method's deadline, and longer than the slowest such method can actually run: a claim taken + * over from an owner that is merely slow runs the mutation twice. */ public GrpcIdempotencyInterceptor( GrpcOperationLedger ledger, GrpcMethodPolicyCatalog catalog, Duration inProgressPollInterval, - boolean waitForInProgress) { + boolean waitForInProgress, + Duration claimTtl) { if (ledger == null || catalog == null) { throw new IllegalArgumentException("an idempotency interceptor needs a ledger and a catalog"); } @@ -54,6 +67,32 @@ public final class GrpcIdempotencyInterceptor { this.catalog = catalog; this.inProgressPollInterval = inProgressPollInterval; this.waitForInProgress = waitForInProgress; + this.claimTtl = requireClaimTtlOutlivesKeyedDeadlines(claimTtl, catalog); + } + + // A TTL inside a keyed method's deadline lets a retry take over a claim whose owner is still + // within its budget. The deadline is the only per-method bound on owner lifetime the catalog + // knows; it does not bound a server that keeps working after the deadline fires. + private static Duration requireClaimTtlOutlivesKeyedDeadlines( + Duration claimTtl, GrpcMethodPolicyCatalog catalog) { + if (claimTtl == null || claimTtl.isZero() || claimTtl.isNegative()) { + throw new IllegalArgumentException("a claim TTL must be present and positive"); + } + for (GrpcMethodName method : catalog.methods()) { + GrpcMethodPolicy policy = catalog.require(method); + if (policy.idempotency().idempotencyKeyRequired() + && claimTtl.compareTo(policy.deadline().total()) <= 0) { + throw new IllegalArgumentException( + "claim TTL " + + claimTtl + + " is not longer than the " + + policy.deadline().total() + + " deadline of keyed method '" + + method.canonical() + + "'; a retry would take over a claim whose owner is still running"); + } + } + return claimTtl; } /** @@ -88,7 +127,8 @@ public final class GrpcIdempotencyInterceptor { callerFingerprint, method, GrpcRequestFingerprint.hashIdempotencyKey(idempotencyKey)); Optional existing = ledger.claim(identity, requestFingerprint, now); if (existing.isEmpty()) { - return GrpcIdempotencyDecision.proceed("claimed; this caller owns the operation"); + return GrpcIdempotencyDecision.proceedHolding( + GrpcClaimToken.FIRST, "claimed; this caller owns the operation"); } GrpcOperationLedgerRecord record = existing.get(); @@ -106,16 +146,39 @@ public final class GrpcIdempotencyInterceptor { + identity.storageKey() + "' has no outcome reference to replay"))); } - if (record.state() == GrpcOperationLedgerState.FAILED_TERMINAL - && ledger.reclaimFailed(identity, requestFingerprint, now)) { + if (record.state() == GrpcOperationLedgerState.FAILED_TERMINAL) { // A terminal failure is not replayed: the caller asked for the operation, it did not happen, // and repeating it is exactly what the key makes safe — but only for the one retry that // re-claims the row. Proceeding unclaimed runs concurrent retries twice and leaves the // commit without an owner. - return GrpcIdempotencyDecision.proceed( - "the previous attempt failed terminally; this caller re-claimed it for a new attempt"); + Optional reclaimed = ledger.reclaimFailed(identity, requestFingerprint, now); + if (reclaimed.isPresent()) { + return GrpcIdempotencyDecision.proceedHolding( + reclaimed.get(), + "the previous attempt failed terminally; this caller re-claimed it for a new attempt"); + } } - // Running, or a failed attempt another retry re-claimed first. + if (record.state() == GrpcOperationLedgerState.IN_PROGRESS) { + Instant cutoff = now.minus(claimTtl); + // The owner outlived the TTL without committing or failing, so it is presumed dead. Its + // mutation did not commit if the commit was recorded in the mutation's own transaction, + // which the ledger contract requires; when an adopter commits them separately, the + // abandoned attempt's outcome is unknown and this re-run can duplicate it. An owner that was + // only slow is fenced by the new claim token: its commit is refused, its failure ignored. + Optional takenOver = + record.claimedAt().isBefore(cutoff) + ? ledger.reclaimStale(identity, requestFingerprint, cutoff, now) + : Optional.empty(); + if (takenOver.isPresent()) { + return GrpcIdempotencyDecision.proceedHolding( + takenOver.get(), + "the previous claim was abandoned (older than the " + + claimTtl + + " claim TTL); this caller took it over for a new attempt"); + } + } + // Running, a failed attempt another retry re-claimed first, or an abandoned claim another + // retry took over first. return waitForInProgress ? GrpcIdempotencyDecision.waitAndPoll(inProgressPollInterval) : GrpcIdempotencyDecision.rejectInProgress(); @@ -127,26 +190,42 @@ public final class GrpcIdempotencyInterceptor { *

Called from inside the business transaction wherever the datastore permits it. A ledger * committed after the mutation leaves a window in which the write is durable and the claim is * not. + * + * @param claimToken the token of the {@code PROCEED} decision this attempt ran under + * @throws IllegalStateException when this attempt no longer holds the claim — another attempt + * took it over — so the business transaction rolls back instead of committing a second run */ public void recordCommit( GrpcMethodName method, String callerFingerprint, String idempotencyKey, + GrpcClaimToken claimToken, String outcomeReference, Instant now) { ledger.markCommitted( new GrpcOperationIdentity( callerFingerprint, method, GrpcRequestFingerprint.hashIdempotencyKey(idempotencyKey)), + claimToken, outcomeReference, now); } - /** Records a terminal failure against the claim. */ - public void recordFailure( - GrpcMethodName method, String callerFingerprint, String idempotencyKey, Instant now) { - ledger.markFailed( + /** + * Records a terminal failure against the claim, if this attempt still holds it. + * + * @param claimToken the token of the {@code PROCEED} decision this attempt ran under + * @return false when another attempt holds the claim now; its run is not failed by this one + */ + public boolean recordFailure( + GrpcMethodName method, + String callerFingerprint, + String idempotencyKey, + GrpcClaimToken claimToken, + Instant now) { + return ledger.markFailed( new GrpcOperationIdentity( callerFingerprint, method, GrpcRequestFingerprint.hashIdempotencyKey(idempotencyKey)), + claimToken, now); } } diff --git a/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetention.java b/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetention.java new file mode 100644 index 0000000..ac4e22c --- /dev/null +++ b/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetention.java @@ -0,0 +1,80 @@ +package dev.caskeleton.grpc.idempotency; + +import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; +import java.time.Duration; +import java.time.Instant; + +/** + * Deletes finished operation-ledger records once they are older than the retention. + * + *

Without it the ledger keeps every {@code COMMITTED} and {@code FAILED_TERMINAL} row forever. A + * finished row exists only to answer a retry, so once it is older than any retry a caller still + * sends, it is dead weight on the table every keyed call writes to. + * + *

This class decides what to delete; it does not decide when. Nothing here + * schedules itself: the platform starter does not own a scheduler, does not start the server, and + * cannot know which replica should run the purge, so the adopter calls {@link #purgeExpired} from + * its own scheduler (one replica, or every replica — the bounded delete is safe either way). + */ +public final class GrpcOperationLedgerRetention { + + /** + * Rows deleted per statement. Small enough that one delete holds its row locks briefly on the + * hottest table of the idempotency path, large enough that a day's backlog drains in few calls. + */ + public static final int DEFAULT_BATCH_SIZE = 500; + + private final GrpcOperationLedger ledger; + private final Duration retention; + private final int batchSize; + + /** + * @param retention how long a finished record is kept after it completed. It must exceed the + * longest window in which a caller still retries the same key: a retry after the purge finds + * no record, claims afresh and runs the mutation again. + * @param batchSize the most rows one delete statement removes + */ + public GrpcOperationLedgerRetention( + GrpcOperationLedger ledger, Duration retention, int batchSize) { + if (ledger == null) { + throw new IllegalArgumentException("a ledger retention needs a ledger"); + } + if (retention == null || retention.isZero() || retention.isNegative()) { + throw new IllegalArgumentException("a ledger retention must be present and positive"); + } + if (batchSize < 1) { + throw new IllegalArgumentException("a ledger purge batch size must be positive"); + } + this.ledger = ledger; + this.retention = retention; + this.batchSize = batchSize; + } + + /** How long a finished record is kept. */ + public Duration retention() { + return retention; + } + + /** + * Deletes every finished record that completed more than the retention before {@code now}, in + * bounded batches, and never an {@code IN_PROGRESS} one. + * + *

The cutoff is fixed at the start, so the loop ends: records finishing while it runs are + * newer than the cutoff and are left for the next call. + * + * @return how many records were deleted + */ + public int purgeExpired(Instant now) { + if (now == null) { + throw new IllegalArgumentException("a purge needs the current moment"); + } + Instant cutoff = now.minus(retention); + int total = 0; + int purged; + do { + purged = ledger.purgeCompletedBefore(cutoff, batchSize); + total += purged; + } while (purged == batchSize); + return total; + } +} diff --git a/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/server/GrpcAdmissionController.java b/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/server/GrpcAdmissionController.java index a6b346c..b0aa8e3 100644 --- a/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/server/GrpcAdmissionController.java +++ b/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/server/GrpcAdmissionController.java @@ -14,7 +14,10 @@ public final class GrpcAdmissionController { private final int maxQueuedCalls; // One lock for both counters: every decision reads one and changes the other, and a check on an - // atomic followed by an increment lets racing callers all pass the same check. + // atomic followed by an increment lets racing callers all pass the same check. A private object + // rather than `synchronized` methods: the controller is a shared bean, and a method that locks + // `this` waits on a monitor any holder of the reference can take, stalling every admission. + private final Object lock = new Object(); private int inFlight; private int queued; private int rejected; @@ -49,54 +52,66 @@ public final class GrpcAdmissionController { } /** Admits a call, queues it, or refuses it. */ - public synchronized Decision tryAdmit() { - if (inFlight < maxConcurrentCalls) { - inFlight++; - return new Decision(true, inFlight, queued, "within the concurrency bound"); + public Decision tryAdmit() { + synchronized (lock) { + if (inFlight < maxConcurrentCalls) { + inFlight++; + return new Decision(true, inFlight, queued, "within the concurrency bound"); + } + if (queued < maxQueuedCalls) { + queued++; + return new Decision(true, inFlight, queued, "queued within the admission bound"); + } + rejected++; + return new Decision( + false, + inFlight, + queued, + "at capacity: " + + inFlight + + " in flight and " + + queued + + " queued. Refusing is better than accepting work whose caller will have given up."); } - if (queued < maxQueuedCalls) { - queued++; - return new Decision(true, inFlight, queued, "queued within the admission bound"); - } - rejected++; - return new Decision( - false, - inFlight, - queued, - "at capacity: " - + inFlight - + " in flight and " - + queued - + " queued. Refusing is better than accepting work whose caller will have given up."); } /** Moves a queued call into execution. */ - public synchronized void promoteFromQueue() { - if (queued > 0) { - queued--; - inFlight++; + public void promoteFromQueue() { + synchronized (lock) { + if (queued > 0) { + queued--; + inFlight++; + } } } /** Releases a finished call. */ - public synchronized void release() { - if (inFlight > 0) { - inFlight--; + public void release() { + synchronized (lock) { + if (inFlight > 0) { + inFlight--; + } } } /** How many calls are executing. */ - public synchronized int inFlight() { - return inFlight; + public int inFlight() { + synchronized (lock) { + return inFlight; + } } /** How many calls are waiting. */ - public synchronized int queued() { - return queued; + public int queued() { + synchronized (lock) { + return queued; + } } /** How many calls have been refused. */ - public synchronized int rejected() { - return rejected; + public int rejected() { + synchronized (lock) { + return rejected; + } } } diff --git a/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java index 972630e..dd50082 100644 --- a/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java +++ b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java @@ -2,8 +2,11 @@ package dev.caskeleton.grpc.idempotency; import static org.assertj.core.api.Assertions.assertThat; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; +import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger; import dev.caskeleton.grpc.core.GrpcMethodName; +import java.time.Duration; import java.time.Instant; import java.util.Optional; import org.junit.jupiter.api.DisplayName; @@ -29,6 +32,31 @@ class GrpcCompletionReconcilerTest { NOW); } + @Test + @DisplayName("버려진 claim으로 대기열에 오른 건은 넘겨받은 재시도가 commit하면 다음 회차에 COMMITTED로 풀린다") + void anAbandonedClaimStaysPendingUntilATakeoverCommits() { + // given + GrpcOperationIdentity identity = + new GrpcOperationIdentity(CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY)); + String fingerprint = GrpcRequestFingerprint.of("body"); + Instant afterTtl = NOW.plus(Duration.ofMinutes(5)).plusSeconds(1); + GrpcCompletionReconciler reconciler = new GrpcCompletionReconciler(query); + claim(); + reconciler.reconcile(CREATE, CALLER, KEY, null); + GrpcClaimToken takenOver = + ledger + .reclaimStale(identity, fingerprint, afterTtl.minus(Duration.ofMinutes(5)), afterTtl) + .orElseThrow(); + ledger.markCommitted(identity, takenOver, "outcome://document/42", afterTtl.plusSeconds(1)); + + // when + GrpcCompletionResolution nextPass = reconciler.reconcile(CREATE, CALLER, KEY, null); + + // then + assertThat(nextPass.status()).isEqualTo(GrpcOperationStatus.COMMITTED); + assertThat(nextPass.outcomeReference()).contains("outcome://document/42"); + } + @Test @DisplayName("a status query returns the five documented answers") void theQueryReturnsTheFiveDocumentedAnswers() { @@ -69,6 +97,7 @@ class GrpcCompletionReconcilerTest { ledger.markCommitted( new dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity( CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY)), + GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); @@ -92,6 +121,7 @@ class GrpcCompletionReconcilerTest { ledger.markCommitted( new dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity( CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY)), + GrpcClaimToken.FIRST, "outcome://from-ledger", NOW.plusSeconds(1)); GrpcCompletionReconciler reconciler = new GrpcCompletionReconciler(query); @@ -141,6 +171,5 @@ class GrpcCompletionReconcilerTest { GrpcCompletionResolution resolution = reconciler.reconcile(CREATE, CALLER, KEY, null); assertThat(resolution.status()).isEqualTo(GrpcOperationStatus.IN_PROGRESS); - assertThat(reconciler.pendingCases()).hasSize(1); } } diff --git a/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java index 6ed3d16..99fa068 100644 --- a/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java +++ b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java @@ -2,8 +2,10 @@ package dev.caskeleton.grpc.idempotency; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.grpc.api.deadline.GrpcDeadlineProfile; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerState; @@ -36,6 +38,8 @@ class GrpcIdempotencyInterceptorTest { private static final Instant NOW = Instant.parse("2026-08-30T10:00:00Z"); private static final String CALLER = "tenant-1|actor-1"; private static final String KEY = "order-2026-0001"; + private static final Duration CLAIM_TTL = Duration.ofMinutes(5); + private static final Instant AFTER_TTL = NOW.plus(CLAIM_TTL).plusSeconds(1); private final InMemoryOperationLedger ledger = new InMemoryOperationLedger(); @@ -57,7 +61,7 @@ class GrpcIdempotencyInterceptorTest { private GrpcIdempotencyInterceptor interceptor(boolean waitForInProgress) { return new GrpcIdempotencyInterceptor( - ledger, catalog(), Duration.ofMillis(250), waitForInProgress); + ledger, catalog(), Duration.ofMillis(250), waitForInProgress, CLAIM_TTL); } @Test @@ -95,8 +99,10 @@ class GrpcIdempotencyInterceptorTest { void aDuplicateOfACommittedOperationReplays() { GrpcIdempotencyInterceptor interceptor = interceptor(false); String fingerprint = GrpcRequestFingerprint.of("body"); - interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); - interceptor.recordCommit(CREATE, CALLER, KEY, "outcome://document/42", NOW.plusSeconds(1)); + GrpcClaimToken claim = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.recordCommit( + CREATE, CALLER, KEY, claim, "outcome://document/42", NOW.plusSeconds(1)); GrpcIdempotencyDecision replayed = interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2)); @@ -136,8 +142,9 @@ class GrpcIdempotencyInterceptorTest { void aTerminallyFailedOperationMayBeRetried() { GrpcIdempotencyInterceptor interceptor = interceptor(false); String fingerprint = GrpcRequestFingerprint.of("body"); - interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); - interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1)); + GrpcClaimToken claim = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.recordFailure(CREATE, CALLER, KEY, claim, NOW.plusSeconds(1)); assertThat(interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2)).outcome()) .isEqualTo(GrpcIdempotencyDecision.Outcome.PROCEED); @@ -148,8 +155,9 @@ class GrpcIdempotencyInterceptorTest { // given GrpcIdempotencyInterceptor interceptor = interceptor(false); String fingerprint = GrpcRequestFingerprint.of("body"); - interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); - interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1)); + GrpcClaimToken claim = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.recordFailure(CREATE, CALLER, KEY, claim, NOW.plusSeconds(1)); // when interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2)); @@ -169,8 +177,9 @@ class GrpcIdempotencyInterceptorTest { // given GrpcIdempotencyInterceptor interceptor = interceptor(false); String fingerprint = GrpcRequestFingerprint.of("body"); - interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); - interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1)); + GrpcClaimToken claim = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.recordFailure(CREATE, CALLER, KEY, claim, NOW.plusSeconds(1)); CountDownLatch start = new CountDownLatch(1); Callable retry = () -> { @@ -194,6 +203,171 @@ class GrpcIdempotencyInterceptorTest { GrpcIdempotencyDecision.Outcome.REJECT_IN_PROGRESS); } + @Test + @DisplayName("소유자가 죽어 TTL이 지난 claim은 같은 요청의 재시도가 넘겨받아 진행한다") + void aRetryAfterTheClaimTtlTakesOverAnAbandonedClaimAndProceeds() { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); + + // when + GrpcIdempotencyDecision decision = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL); + + // then + assertThat(decision.outcome()).isEqualTo(GrpcIdempotencyDecision.Outcome.PROCEED); + assertThat(decision.reason()).contains("abandoned"); + } + + @Test + @DisplayName("넘겨받은 claim은 재시도 시각에 다시 잡힌 진행 중 claim이다") + void aTakenOverClaimIsHeldInProgressFromTheRetry() { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); + + // when + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL); + + // then + assertThat( + ledger + .find( + new GrpcOperationIdentity( + CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY))) + .map(GrpcOperationLedgerRecord::claimedAt)) + .contains(AFTER_TTL); + } + + @Test + @DisplayName("claim 을 이긴 PROCEED 는 그 시도의 claim token 을 들고 있다") + void aProceedThatWonAClaimHoldsItsClaimToken() { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); + + // when + GrpcIdempotencyDecision takenOver = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL); + + // then + assertThat(takenOver.claimToken()).contains(GrpcClaimToken.FIRST.next()); + } + + /** + * The re-review probe's sequence through the interceptor (2026-11-03, RR-02-04): A is slow, B + * takes over after the TTL, A then fails. A's failure must not free the key for C while B runs. + */ + @Test + @DisplayName("TTL 뒤 넘겨받힌 옛 owner 의 실패 기록은 무시되어 다음 재시도는 진행 중으로 거부된다") + void aFailureRecordedByTheOwnerThatLostTheClaimDoesNotLetTheNextRetryRun() { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + GrpcClaimToken first = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL); + boolean recorded = + interceptor.recordFailure(CREATE, CALLER, KEY, first, AFTER_TTL.plusSeconds(1)); + + // when + GrpcIdempotencyDecision third = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL.plusSeconds(2)); + + // then + assertThat(recorded).isFalse(); + assertThat(third.outcome()).isEqualTo(GrpcIdempotencyDecision.Outcome.REJECT_IN_PROGRESS); + } + + @Test + @DisplayName("TTL 뒤 넘겨받힌 옛 owner 의 commit 은 거부되어 그 트랜잭션이 롤백된다") + void aCommitRecordedByTheOwnerThatLostTheClaimIsRefused() { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + GrpcClaimToken first = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL); + + // when + Throwable refused = + org.assertj.core.api.Assertions.catchThrowable( + () -> + interceptor.recordCommit( + CREATE, CALLER, KEY, first, "outcome://document/42", AFTER_TTL.plusSeconds(1))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("no longer held"); + } + + @Test + @DisplayName("TTL이 지나기 직전의 중복 요청은 여전히 대기하거나 거부된다") + void aDuplicateAtTheTtlBoundaryIsStillTreatedAsRunning() { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); + + // when + GrpcIdempotencyDecision decision = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plus(CLAIM_TTL)); + + // then + assertThat(decision.outcome()).isEqualTo(GrpcIdempotencyDecision.Outcome.REJECT_IN_PROGRESS); + } + + @Test + @DisplayName("버려진 claim에 동시에 온 재시도 중 하나만 진행한다") + void concurrentRetriesOfAnAbandonedClaimProceedOnlyOnce() throws Exception { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); + CountDownLatch start = new CountDownLatch(1); + Callable retry = + () -> { + start.await(); + return interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL).outcome(); + }; + + // when + List outcomes; + try (ExecutorService executor = Executors.newFixedThreadPool(2)) { + Future first = executor.submit(retry); + Future second = executor.submit(retry); + start.countDown(); + outcomes = List.of(first.get(), second.get()); + } + + // then + assertThat(outcomes) + .containsExactlyInAnyOrder( + GrpcIdempotencyDecision.Outcome.PROCEED, + GrpcIdempotencyDecision.Outcome.REJECT_IN_PROGRESS); + } + + @Test + @DisplayName("claim TTL이 key 필수 method의 deadline보다 짧으면 생성 시 거부한다") + void aClaimTtlNoLongerThanAKeyedMethodsDeadlineIsRefused() { + // when + Throwable refused = + catchThrowable( + () -> + new GrpcIdempotencyInterceptor( + ledger, catalog(), Duration.ofMillis(250), false, Duration.ofSeconds(10))); + + // then + assertThat(refused) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(CREATE.canonical()) + .hasMessageContaining("claim TTL"); + } + @Test @DisplayName("one caller's key does not suppress another caller's write") void oneCallersKeyDoesNotSuppressAnother() { diff --git a/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetentionTest.java b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetentionTest.java new file mode 100644 index 0000000..817514a --- /dev/null +++ b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetentionTest.java @@ -0,0 +1,85 @@ +package dev.caskeleton.grpc.idempotency; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; +import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; +import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger; +import dev.caskeleton.grpc.core.GrpcMethodName; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +class GrpcOperationLedgerRetentionTest { + + private static final GrpcMethodName CREATE = + GrpcMethodName.parse("caskeleton.document.v1.DocumentService/CreateDocument"); + private static final Instant NOW = Instant.parse("2026-11-01T10:00:00Z"); + private static final Duration RETENTION = Duration.ofHours(24); + private static final String BODY = GrpcRequestFingerprint.of("body"); + + private final InMemoryOperationLedger ledger = new InMemoryOperationLedger(); + + @Test + @DisplayName("보존 기간이 지난 종결 행을 batch 크기와 무관하게 모두 지운다") + void everyFinishedRecordPastTheRetentionIsPurgedAcrossBatches() { + // given + List old = List.of(identity("k1"), identity("k2"), identity("k3")); + old.forEach(identity -> committed(identity, NOW.minus(RETENTION).minusSeconds(60))); + GrpcOperationLedgerRetention retention = new GrpcOperationLedgerRetention(ledger, RETENTION, 2); + + // when + int purged = retention.purgeExpired(NOW); + + // then + assertThat(purged).isEqualTo(3); + assertThat(old).allSatisfy(identity -> assertThat(ledger.find(identity)).isEmpty()); + } + + @Test + @DisplayName("보존 기간 안에 끝난 행과 진행 중인 행은 남긴다") + void recentFinishedRecordsAndInProgressClaimsAreKept() { + // given + GrpcOperationIdentity recent = identity("recent"); + GrpcOperationIdentity running = identity("running"); + committed(recent, NOW.minus(RETENTION).plusSeconds(60)); + ledger.claim(running, BODY, NOW.minus(Duration.ofDays(30))); + GrpcOperationLedgerRetention retention = new GrpcOperationLedgerRetention(ledger, RETENTION, 2); + + // when + int purged = retention.purgeExpired(NOW); + + // then + assertThat(purged).isZero(); + assertThat(ledger.find(recent)).isPresent(); + assertThat(ledger.find(running)).isPresent(); + } + + @Test + @DisplayName("보존 기간이 양수가 아니면 만들 수 없다") + void aNonPositiveRetentionIsRefused() { + // when + Throwable refused = + catchThrowable(() -> new GrpcOperationLedgerRetention(ledger, Duration.ZERO, 2)); + + // then + assertThat(refused) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("retention must be present and positive"); + } + + private static GrpcOperationIdentity identity(String key) { + return new GrpcOperationIdentity( + "tenant-1|actor-1", CREATE, GrpcRequestFingerprint.hashIdempotencyKey(key)); + } + + // Finished at the claim moment; the completion moment is what the retention measures. + private void committed(GrpcOperationIdentity identity, Instant claimedAt) { + ledger.claim(identity, BODY, claimedAt); + ledger.markCommitted( + identity, GrpcClaimToken.FIRST, "outcome://document/" + identity.storageKey(), claimedAt); + } +} diff --git a/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/server/GrpcAdmissionControllerTest.java b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/server/GrpcAdmissionControllerTest.java index 895fdf5..34b9260 100644 --- a/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/server/GrpcAdmissionControllerTest.java +++ b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/server/GrpcAdmissionControllerTest.java @@ -8,6 +8,7 @@ import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; class GrpcAdmissionControllerTest { @@ -24,6 +25,27 @@ class GrpcAdmissionControllerTest { assertThat(mostAdmitted).isEqualTo(2); } + @Test + void codeHoldingTheControllersOwnMonitorDoesNotStallAdmission() throws Exception { + // given + GrpcAdmissionController controller = new GrpcAdmissionController(1, 1); + ExecutorService caller = Executors.newSingleThreadExecutor(); + Future admitted; + try { + // The controller is a shared bean: anything that holds a reference can lock it. A decision + // that waits on that monitor stalls every call the server is trying to admit. + synchronized (controller) { + // when + admitted = caller.submit(() -> controller.tryAdmit().admitted()); + + // then + assertThat(admitted.get(5, TimeUnit.SECONDS)).isTrue(); + } + } finally { + caller.shutdownNow(); + } + } + /** Races {@link #CALLERS} callers at a 1-running, 1-queued controller, round after round. */ private static int mostAdmittedOverRounds() throws Exception { ExecutorService pool = Executors.newFixedThreadPool(CALLERS); diff --git a/grpc/grpc-spring-boot-starter/build.gradle b/grpc/grpc-spring-boot-starter/build.gradle index ad114e6..2a8fbfa 100644 --- a/grpc/grpc-spring-boot-starter/build.gradle +++ b/grpc/grpc-spring-boot-starter/build.gradle @@ -18,4 +18,6 @@ dependencies { implementation 'org.springframework.boot:spring-boot-autoconfigure' + // The shared operation-ledger fake stands in for the adopter's ledger bean in context tests. + testImplementation testFixtures(project(':grpc:grpc-core-api')) } diff --git a/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfiguration.java b/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfiguration.java index 8516939..02eddca 100644 --- a/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfiguration.java +++ b/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfiguration.java @@ -5,15 +5,20 @@ import dev.caskeleton.grpc.admin.GrpcDrainPolicy; import dev.caskeleton.grpc.admin.GrpcHealthPolicy; import dev.caskeleton.grpc.admin.GrpcReflectionPolicy; import dev.caskeleton.grpc.admin.GrpcServiceHealthRegistry; +import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.policy.GrpcMethodPolicyCatalog; import dev.caskeleton.grpc.client.GrpcNamedChannelProfile; import dev.caskeleton.grpc.context.GrpcContextBinder; import dev.caskeleton.grpc.context.GrpcContextPropagationPolicy; import dev.caskeleton.grpc.error.GrpcErrorMapper; +import dev.caskeleton.grpc.idempotency.GrpcIdempotencyInterceptor; +import dev.caskeleton.grpc.idempotency.GrpcOperationLedgerRetention; import dev.caskeleton.grpc.security.GrpcTlsProfile; import dev.caskeleton.grpc.server.GrpcAdmissionController; import dev.caskeleton.grpc.server.GrpcExecutorProfile; import dev.caskeleton.grpc.server.GrpcServerProfile; +import java.time.Duration; +import java.util.List; import java.util.UUID; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.SmartInitializingSingleton; @@ -131,6 +136,89 @@ public class GrpcPlatformAutoConfiguration { serverProfile); } + /** + * The keyed-idempotency decision, assembled when {@code operation-ledger-enabled=true}. + * + *

Built here rather than left to the adopter (review 2026-11-01, 03-11, option (b)): a flag + * that only satisfied the startup check while nothing honoured the key was a promise the platform + * did not keep. The ledger itself stays the deployment's bean — the starter does not depend on + * {@code grpc-operation-ledger-jpa} (11-M9) — and a missing one refuses startup. + * + *

A duplicate of a running operation is refused rather than told to wait: keyed callers are + * usually user-facing requests that already retry with backoff. There is deliberately no + * {@code @ConditionalOnMissingBean}: a replacement bean would silently ignore {@code + * operation-ledger-claim-ttl}, which is the same false promise this bean exists to remove. + * + *

The starter does not build a server, so nothing calls this yet: the adopter's {@code + * IDEMPOTENCY} stage interceptor ({@code GrpcServerInterceptorChain}) or service adapter calls + * {@code decide} before the mutation and {@code recordCommit} inside the mutation's transaction. + */ + @Bean + @ConditionalOnProperty( + prefix = "app.grpc.platform", + name = "operation-ledger-enabled", + havingValue = "true") + public GrpcIdempotencyInterceptor grpcIdempotencyInterceptor( + GrpcPlatformProperties properties, + ObjectProvider ledgers, + ObjectProvider catalogs) { + GrpcOperationLedger ledger = requireLedger(ledgers); + try { + return new GrpcIdempotencyInterceptor( + ledger, + catalogs.getIfAvailable(() -> GrpcMethodPolicyCatalog.builder().build()), + Duration.ZERO, + false, + properties.getOperationLedgerClaimTtl()); + } catch (IllegalArgumentException invalidTtl) { + // The ledger and catalog are present and the poll interval is fixed, so the claim TTL is the + // only input left for the constructor to refuse. + throw new GrpcPlatformConfigurationException( + List.of("app.grpc.platform.operation-ledger-claim-ttl: " + invalidTtl.getMessage())); + } + } + + /** + * Retention of finished ledger records, assembled with the interceptor. + * + *

Only the policy: nothing here schedules it. The starter owns no scheduler and does not know + * which replica should run the purge, so the adopter calls {@code purgeExpired(now)} from its own + * scheduler. A scheduled bean here was rejected because it would force a task scheduler onto + * every deployment and run on every replica whether or not that deployment wants it. + */ + @Bean + @ConditionalOnProperty( + prefix = "app.grpc.platform", + name = "operation-ledger-enabled", + havingValue = "true") + public GrpcOperationLedgerRetention grpcOperationLedgerRetention( + GrpcPlatformProperties properties, ObjectProvider ledgers) { + GrpcOperationLedger ledger = requireLedger(ledgers); + try { + return new GrpcOperationLedgerRetention( + ledger, + properties.getOperationLedgerRetention(), + GrpcOperationLedgerRetention.DEFAULT_BATCH_SIZE); + } catch (IllegalArgumentException invalidRetention) { + throw new GrpcPlatformConfigurationException( + List.of( + "app.grpc.platform.operation-ledger-retention: " + invalidRetention.getMessage())); + } + } + + private static GrpcOperationLedger requireLedger(ObjectProvider ledgers) { + GrpcOperationLedger ledger = ledgers.getIfAvailable(); + if (ledger == null) { + throw new GrpcPlatformConfigurationException( + List.of( + "app.grpc.platform.operation-ledger-enabled=true but there is no GrpcOperationLedger" + + " bean; the starter does not bring one. Add :grpc:grpc-operation-ledger-jpa and" + + " define a JpaGrpcOperationLedger bean (with the db/migration/grpc Flyway" + + " location), or turn the flag off")); + } + return ledger; + } + /** The shared error mapper. Its execution ids are random and carry no derived content. */ @Bean @ConditionalOnMissingBean diff --git a/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformProperties.java b/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformProperties.java index c480573..3d3a73b 100644 --- a/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformProperties.java +++ b/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformProperties.java @@ -3,6 +3,7 @@ package dev.caskeleton.grpc.boot; import dev.caskeleton.grpc.admin.GrpcReflectionMode; import dev.caskeleton.grpc.security.GrpcTlsProfile; import dev.caskeleton.grpc.server.GrpcServerTransport; +import java.time.Duration; import org.springframework.boot.context.properties.ConfigurationProperties; /** @@ -49,9 +50,32 @@ public class GrpcPlatformProperties { /** The bounded executor's maximum size. */ private int executorMaxPoolSize = 32; - /** Whether the operation ledger is available, which gates keyed idempotency. */ + /** + * Whether keyed idempotency runs. When true the starter assembles {@code + * GrpcIdempotencyInterceptor} and {@code GrpcOperationLedgerRetention} over the deployment's + * {@code GrpcOperationLedger} bean, and refuses to start without one; the starter never brings a + * ledger implementation itself. When false, a method that requires an idempotency key is refused + * at startup, because the key would be accepted and never honoured. + */ private boolean operationLedgerEnabled; + /** + * How long an {@code IN_PROGRESS} claim is trusted to have a live owner before a retry of the + * same request takes it over. Five minutes: far beyond a unary deadline (the interceptor refuses + * a TTL not longer than every keyed method's deadline), so a slow but live owner is not + * overtaken, and short enough that a key whose owner died unblocks within a client's normal retry + * patience. It is also the value the ledger contract and interceptor tests certify. + */ + private Duration operationLedgerClaimTtl = Duration.ofMinutes(5); + + /** + * How long a finished ({@code COMMITTED} or {@code FAILED_TERMINAL}) ledger record is kept before + * {@code GrpcOperationLedgerRetention} may delete it. A retry of the same key after that is a new + * operation, so this must exceed the callers' retry window. 24 hours, matching the HTTP + * idempotency replay TTL ({@code app.capabilities.idempotency.replay-ttl}). + */ + private Duration operationLedgerRetention = Duration.ofHours(24); + public boolean isEnabled() { return enabled; } @@ -124,6 +148,22 @@ public class GrpcPlatformProperties { this.operationLedgerEnabled = operationLedgerEnabled; } + public Duration getOperationLedgerClaimTtl() { + return operationLedgerClaimTtl; + } + + public void setOperationLedgerClaimTtl(Duration operationLedgerClaimTtl) { + this.operationLedgerClaimTtl = operationLedgerClaimTtl; + } + + public Duration getOperationLedgerRetention() { + return operationLedgerRetention; + } + + public void setOperationLedgerRetention(Duration operationLedgerRetention) { + this.operationLedgerRetention = operationLedgerRetention; + } + /** * The reflection mode this deployment runs, defaulting to the environment's floor, and to * disabled while the environment is unset (the startup check refuses that state). diff --git a/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidator.java b/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidator.java index 1f1a4de..85a3ed8 100644 --- a/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidator.java +++ b/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidator.java @@ -24,6 +24,11 @@ import java.util.Set; * reflection in production publishes the schema, and a keyed method without a ledger accepts * idempotency keys it cannot honour. None of them fails a smoke test. * + *

The keyed-method rule has a limit it states in its own message: with the ledger flag on, the + * starter assembles the decision function, and the server stage that calls it is the adopter's. A + * running process cannot see whether that stage exists, so passing this check with the flag on does + * not mean keys are honoured (re-review 2026-11-03, RR-02-07; ADR-GRPC-004). + * *

Fails once with every violation, so a deployment learns the whole list in one restart. */ public final class GrpcPlatformStartupValidator { @@ -151,8 +156,11 @@ public final class GrpcPlatformStartupValidator { violations.add( "method '" + method.canonical() - + "' requires an idempotency key but no operation ledger is enabled; the key would " - + "be accepted and never honoured"); + + "' requires an idempotency key but app.grpc.platform.operation-ledger-enabled is" + + " false; the key would be accepted and never honoured. Turning the flag on" + + " assembles the decision function only: the key is honoured once the server's" + + " IDEMPOTENCY stage interceptor or the service adapter calls decide/recordCommit," + + " which this starter does not wire"); } if (!policy.rpcType().stable()) { violations.add( diff --git a/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfigurationContextTest.java b/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfigurationContextTest.java index 6957096..b3fcbd2 100644 --- a/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfigurationContextTest.java +++ b/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfigurationContextTest.java @@ -4,13 +4,42 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; +import dev.caskeleton.grpc.api.deadline.GrpcDeadlineProfile; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; +import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; +import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; +import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger; +import dev.caskeleton.grpc.api.policy.GrpcMethodPolicy; +import dev.caskeleton.grpc.api.policy.GrpcMethodPolicyCatalog; +import dev.caskeleton.grpc.api.policy.RpcIdempotencyProfile; +import dev.caskeleton.grpc.api.policy.WaitForReadyPolicy; +import dev.caskeleton.grpc.core.GrpcMethodName; +import dev.caskeleton.grpc.core.RpcType; +import dev.caskeleton.grpc.idempotency.GrpcIdempotencyDecision; +import dev.caskeleton.grpc.idempotency.GrpcIdempotencyInterceptor; +import dev.caskeleton.grpc.idempotency.GrpcOperationLedgerRetention; +import dev.caskeleton.grpc.idempotency.GrpcRequestFingerprint; +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; import java.util.Map; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; import org.springframework.core.env.MapPropertySource; class GrpcPlatformAutoConfigurationContextTest { + private static final GrpcMethodName CREATE = + GrpcMethodName.parse("caskeleton.document.v1.DocumentService/CreateDocument"); + private static final Duration KEYED_DEADLINE = Duration.ofSeconds(10); + private static final String CALLER = "tenant-1|actor-1"; + private static final String KEY = "order-2026-0001"; + private static final Instant NOW = Instant.parse("2026-11-01T10:00:00Z"); + @Test void invalidProductionSecurityFailsDuringActualAutoConfigurationStartup() { assertThatThrownBy( @@ -77,12 +106,170 @@ class GrpcPlatformAutoConfigurationContextTest { } } - private static AnnotationConfigApplicationContext start(Map properties) { + @Test + @DisplayName("ledger 플래그가 켜져 있고 ledger bean이 있으면 멱등성 interceptor가 조립된다") + void anEnabledLedgerWithALedgerBeanAssemblesTheIdempotencyInterceptor() { + // when + try (AnnotationConfigApplicationContext context = + start(ledgerEnabled(Map.of()), KeyedMethodWithLedger.class)) { + + // then + assertThat(context.getBeansOfType(GrpcIdempotencyInterceptor.class)).hasSize(1); + } + } + + @Test + @DisplayName("ledger 플래그가 켜져 있는데 ledger bean이 없으면 기동을 거부한다") + void anEnabledLedgerWithoutALedgerBeanRefusesToStart() { + // when + Throwable failure = catchThrowable(() -> start(ledgerEnabled(Map.of()), KeyedMethod.class)); + + // then + assertThat(failure) + .rootCause() + .isInstanceOf(GrpcPlatformConfigurationException.class) + .hasMessageContaining("app.grpc.platform.operation-ledger-enabled=true") + .hasMessageContaining("no GrpcOperationLedger bean"); + } + + @Test + @DisplayName("ledger 플래그가 꺼져 있으면 멱등성 interceptor와 보존 정리를 만들지 않는다") + void aDisabledLedgerAssemblesNoIdempotencyInterceptorOrRetention() { + // when + try (AnnotationConfigApplicationContext context = + start(localPlatform(Map.of()), LedgerOnly.class)) { + + // then + assertThat(context.getBeansOfType(GrpcIdempotencyInterceptor.class)).isEmpty(); + assertThat(context.getBeansOfType(GrpcOperationLedgerRetention.class)).isEmpty(); + } + } + + @Test + @DisplayName("claim-ttl 설정이 interceptor의 stale claim 판단에 쓰인다") + void theClaimTtlKeyDecidesWhenAnInProgressClaimIsTakenOver() { + // given + try (AnnotationConfigApplicationContext context = + start( + ledgerEnabled(Map.of("app.grpc.platform.operation-ledger-claim-ttl", "1m")), + KeyedMethodWithLedger.class)) { + GrpcIdempotencyInterceptor interceptor = context.getBean(GrpcIdempotencyInterceptor.class); + String body = GrpcRequestFingerprint.of("body"); + interceptor.decide(CREATE, CALLER, KEY, body, NOW); + + // when + GrpcIdempotencyDecision retry = + interceptor.decide(CREATE, CALLER, KEY, body, NOW.plus(Duration.ofSeconds(61))); + + // then + assertThat(retry.outcome()).isEqualTo(GrpcIdempotencyDecision.Outcome.PROCEED); + } + } + + @Test + @DisplayName("claim-ttl이 keyed method의 deadline보다 길지 않으면 기동을 거부한다") + void aClaimTtlWithinAKeyedDeadlineRefusesToStart() { + // when + Throwable failure = + catchThrowable( + () -> + start( + ledgerEnabled(Map.of("app.grpc.platform.operation-ledger-claim-ttl", "10s")), + KeyedMethodWithLedger.class)); + + // then + assertThat(failure) + .rootCause() + .isInstanceOf(GrpcPlatformConfigurationException.class) + .hasMessageContaining("app.grpc.platform.operation-ledger-claim-ttl") + .hasMessageContaining("CreateDocument"); + } + + @Test + @DisplayName("retention 설정이 지난 종결 행을 보존 정리가 지운다") + void theRetentionKeyDecidesWhichFinishedRecordsArePurged() { + // given + try (AnnotationConfigApplicationContext context = + start( + ledgerEnabled(Map.of("app.grpc.platform.operation-ledger-retention", "1h")), + KeyedMethodWithLedger.class)) { + GrpcOperationLedger ledger = context.getBean(GrpcOperationLedger.class); + GrpcOperationIdentity identity = + new GrpcOperationIdentity(CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY)); + ledger.claim(identity, GrpcRequestFingerprint.of("body"), NOW); + ledger.markCommitted( + identity, GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); + + // when + int purged = + context + .getBean(GrpcOperationLedgerRetention.class) + .purgeExpired(NOW.plus(Duration.ofHours(2))); + + // then + assertThat(purged).isEqualTo(1); + assertThat(ledger.find(identity)).isEmpty(); + } + } + + private static Map localPlatform(Map extra) { + Map properties = new HashMap<>(); + properties.put("app.grpc.platform.enabled", "true"); + properties.put("app.grpc.platform.environment", "LOCAL"); + properties.put("app.grpc.platform.transport", "NETTY_SHADED"); + properties.putAll(extra); + return properties; + } + + private static Map ledgerEnabled(Map extra) { + Map properties = new HashMap<>(extra); + properties.put("app.grpc.platform.operation-ledger-enabled", "true"); + return localPlatform(properties); + } + + /** One keyed method with a 10-second deadline, the shape the claim TTL is measured against. */ + @Configuration(proxyBeanMethods = false) + static class KeyedMethod { + @Bean + GrpcMethodPolicyCatalog keyedCatalog() { + return GrpcMethodPolicyCatalog.builder() + .register( + new GrpcMethodPolicy( + CREATE, + RpcType.UNARY, + RpcIdempotencyProfile.IDEMPOTENCY_KEY_REQUIRED, + GrpcDeadlineProfile.of(KEYED_DEADLINE), + WaitForReadyPolicy.DISABLED, + true, + 1024, + 1024)) + .build(); + } + } + + /** The adopter's ledger, here the shared in-memory fake. */ + @Configuration(proxyBeanMethods = false) + static class LedgerOnly { + @Bean + GrpcOperationLedger operationLedger() { + return new InMemoryOperationLedger(); + } + } + + @Configuration(proxyBeanMethods = false) + @Import({KeyedMethod.class, LedgerOnly.class}) + static class KeyedMethodWithLedger {} + + private static AnnotationConfigApplicationContext start( + Map properties, Class... adopterConfigurations) { AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); context .getEnvironment() .getPropertySources() .addFirst(new MapPropertySource("grpc-platform-test", properties)); + if (adopterConfigurations.length > 0) { + context.register(adopterConfigurations); + } context.register(GrpcPlatformAutoConfiguration.class); try { context.refresh(); diff --git a/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidatorTest.java b/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidatorTest.java index dcffee9..d7307b1 100644 --- a/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidatorTest.java +++ b/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidatorTest.java @@ -200,6 +200,44 @@ class GrpcPlatformStartupValidatorTest { .anySatisfy(violation -> assertThat(violation).contains("never honoured")); } + /** + * The flag assembles the decision function only; the starter builds no server, so nothing calls + * it until the adopter's IDEMPOTENCY stage or service adapter does. The refusal must not read as + * "turn the flag on and keys are honoured" (re-review 2026-11-03, RR-02-07). + */ + @Test + @DisplayName("키 필수 메서드 거부 문구는 플래그가 결정 함수만 조립하고 서버 단계 연결은 따로라고 말한다") + void theKeyedMethodRefusalSaysTheFlagAssemblesTheDecisionOnly() { + // given + GrpcPlatformProperties properties = productionProperties(); + properties.setOperationLedgerEnabled(false); + GrpcMethodPolicyCatalog catalog = + GrpcMethodPolicyCatalog.builder() + .register( + new GrpcMethodPolicy( + CREATE, + RpcType.UNARY, + RpcIdempotencyProfile.IDEMPOTENCY_KEY_REQUIRED, + GrpcDeadlineProfile.of(Duration.ofSeconds(2)), + WaitForReadyPolicy.DISABLED, + false, + 1024, + 1024)) + .build(); + + // when + var violations = + GrpcPlatformStartupValidator.violations(properties, catalog, oneChannel(), stableServer()); + + // then + assertThat(violations) + .anySatisfy( + violation -> + assertThat(violation) + .contains("assembles the decision function only") + .contains("IDEMPOTENCY stage")); + } + @Test @DisplayName("a client-streaming method is refused as an Advanced capability") void clientStreamingIsRefusedAsAdvanced() { diff --git a/grpc/grpc-testkit/build.gradle b/grpc/grpc-testkit/build.gradle index 3387707..7dad319 100644 --- a/grpc/grpc-testkit/build.gradle +++ b/grpc/grpc-testkit/build.gradle @@ -5,8 +5,9 @@ plugins { // Certification. The Stable plan splits this across four modules (core / in-process / netty / // fault); this repository already expresses "these two runs are not the same kind of evidence" with // strict test lanes rather than with module boundaries, so the four become four lanes over one -// leaf (adaptation design §2). A lane that discovers nothing fails, and none of them can serve an -// up-to-date result — which is the property the split was protecting. +// leaf (adaptation design §2). A lane that discovers nothing fails, and the three hermetic lanes +// reuse a result only when their classes and their lane policy are unchanged — the performance lane +// never does. That is the property the split was protecting. strictTestLanes { lane('grpcInProcessContractTest') { @@ -33,8 +34,8 @@ strictTestLanes { } // The performance lane is excluded from the default `test` task. It measures a running server -// under load, and a measurement in the release gate is a flaky test on a shared CI runner; it runs -// when somebody asks for it, by name. +// under load, so it is not in the pull-request or release gate; it runs in the optional build's +// `performanceCheck`, which integration-main.yml's performance-check job runs on main and nightly. tasks.named('test') { useJUnitPlatform { excludeTags 'grpc-performance' diff --git a/settings.gradle b/settings.gradle index cf08c10..a34cfb8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -19,3 +19,18 @@ dependencyResolutionManagement { } rootProject.name = 'optional-platforms' + +// gradle.properties here must say what src/gradle.properties says (review 2026-11-01 08-06): Gradle +// reads only this build's own file, and a copy that drifted would make the optional build quietly +// slower or differently configured than the main one. +def readBuildProperties = { File file -> + def properties = new Properties() + file.withInputStream { properties.load(it) } + properties +} +def mainBuildProperties = readBuildProperties(new File(settingsDir, '../gradle.properties')) +def ownBuildProperties = readBuildProperties(new File(settingsDir, 'gradle.properties')) +if (mainBuildProperties != ownBuildProperties) { + throw new GradleException("optional-platforms/gradle.properties differs from src/gradle.properties: " + + "${ownBuildProperties} vs ${mainBuildProperties}. Copy the main build's values.") +}