diff --git a/build.gradle b/build.gradle index 59808a9..1c5ff1b 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,9 @@ plugins { id 'ca.architecture' + // Every strict lane of this build belongs to a stage a workflow executes, checked here exactly + // as in the main build: the main build's check cannot see this build's lanes, and while only it + // applied the plugin, grpcPerformanceTest ran nowhere (review 2026-11-03 RR-06-03). + id 'ca.strict-lane-stages' } // No `ca.release-provenance` any more: each gRPC leaf's `ca.java-conventions` derives its version @@ -47,3 +51,33 @@ tasks.register('integrationCheck') { description = 'Runs every optional-platform leaf integration test lane. Needs a container runtime.' dependsOn subprojects.findAll { it.childProjects.isEmpty() }.collect { "${it.path}:integrationTestLaneCheck" } } + +// The performance stage of this build: grpcPerformanceTest. It asserts the shape of a measurement +// (ordered percentiles, a gate that reads them) with a generous budget, not a number, so it can run +// on a shared runner. integration-main.yml's performance-check job runs it next to the main build's +// `:performanceCheck`; before this aggregate it ran when somebody asked for it by name, which was +// nobody (review 2026-11-03 RR-06-03). +tasks.register('performanceCheck') { + group = 'verification' + description = 'Runs every optional-platform leaf performance test lane.' + dependsOn subprojects.findAll { it.childProjects.isEmpty() }.collect { "${it.path}:performanceTestLaneCheck" } +} + +// The tag-only stage of this build: the hermetic strict lanes (the three gRPC certification lanes). +// Their tests already run inside each leaf's `test`; the lanes add the one thing `test` cannot say — +// that a tag still selects something — which is worth asserting once per release tag. release.yml +// runs this aggregate instead of naming the lanes, so a lane added here is in the release gate +// without a workflow edit. +tasks.register('releaseCheck') { + group = 'verification' + description = 'Runs every optional-platform leaf hermetic strict test lane (the release gate).' + dependsOn subprojects.findAll { it.childProjects.isEmpty() }.collect { "${it.path}:testLaneCheck" } +} + +// The stages this build's strict lanes may belong to, executed by workflows as +// `./gradlew -p optional-platforms ` from `src`. +strictLaneStages { + stage ':ci', ':integrationCheck', ':performanceCheck', ':releaseCheck' + workflowDirectory = layout.projectDirectory.dir('../../.github/workflows') + workflowProjectDirectory = 'optional-platforms' +} diff --git a/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md b/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md index 495d348..7307c65 100644 --- a/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md +++ b/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md @@ -69,6 +69,36 @@ method can run longer than any fixed TTL; revisit when one exists. A sweeper job rows — it needs a scheduler this family does not have, and deleting a claim makes the next retry a fresh claim with no record that an attempt was abandoned. +**A taken-over owner is fenced by a claim token** (re-review 2026-11-03, RR-02-04). The owner a +takeover presumes dead may only be slow. Without knowing which attempt held the claim, its late +`markFailed` turned the new attempt's claim `FAILED_TERMINAL`, the next retry re-claimed it, and two +attempts ran the mutation at once; its late `markCommitted` committed beside the new attempt. Every +claim now carries a `GrpcClaimToken` (`claim_attempt`, migration V003): 1 for the first claim, one more +for each `reclaimFailed` or `reclaimStale`. `decide` returns it on `PROCEED`, and `recordCommit` / +`recordFailure` hand it back. A commit or a failure is one conditional +`UPDATE … WHERE state = 'IN_PROGRESS' AND claim_attempt = :attempt`: a commit by an attempt that lost +the claim is refused with an exception, so its business transaction rolls back; a failure by one is +ignored (`markFailed` returns false). The same conditional update closes the JPA commit race — +load-check-save let two commits that both read `IN_PROGRESS` both write; now exactly one matches. +A claim whose conflicting row is purged between insert and read retries the insert once instead of +failing the call. + +Chosen over the claim moment as the token (`WHERE claimed_at = :claimedAt`), which needs no column: +PostgreSQL stores microseconds while `Instant` carries nanoseconds, so the token a caller holds and +the one stored can differ by rounding, and two processes' clocks can give two attempts the same +moment. The counter has neither problem. The heartbeat lease above stays rejected for the same reason +as before; the token fences without one. + +**What `operation-ledger-enabled=true` promises (03-11 (b), recorded 2026-11-03, RR-02-07).** The +2026-11-01 review offered (a) delete the flag and judge by whether a ledger bean exists, or (b) keep the +flag and have the starter assemble the decision function from it. (b) is kept: the assembly and the +TTL validation live in one place instead of in every adopter. (a) was rejected because it removes the +false promise only by removing the assembly with it. The limit stays and is stated where an operator +meets it: the starter builds no server, so the flag assembles the decision function only, and keys are +honoured once the adopter's `IDEMPOTENCY` stage interceptor or service adapter calls `decide` and +`recordCommit`. `GrpcPlatformStartupValidator`'s refusal for a keyed method says exactly that; a running +process cannot see whether the stage exists, so the validator does not claim to check it. + **Finished rows are purged after a retention; in-progress rows never are** (03-10 follow-up). `GrpcOperationLedger.purgeCompletedBefore(cutoff, limit)` deletes at most `limit` `COMMITTED` or `FAILED_TERMINAL` rows that completed before the cutoff, oldest first — on PostgreSQL one bounded diff --git a/docs/runbooks/grpc-platform-operations.md b/docs/runbooks/grpc-platform-operations.md index 0b63212..aa896ec 100644 --- a/docs/runbooks/grpc-platform-operations.md +++ b/docs/runbooks/grpc-platform-operations.md @@ -44,7 +44,9 @@ the time, and which half is not knowable from the status. and a later reconciler pass sees that attempt's `COMMITTED` or `FAILED_TERMINAL`. Until a retry arrives, the case stays pending. The re-run is safe only when the ledger commit is recorded in the mutation's own transaction; if this service commits them separately, check the business resource - before letting the retry through. + before letting the retry through. An owner that was only slow cannot spoil the takeover: its claim + attempt (`claim_attempt`) is behind, so its late commit is refused and its late failure ignored. + A log line saying an operation "is no longer held by attempt N" is that refusal, not an outage. **Escalate when** the reconciler's pending list grows across passes. That means the ledger is unreachable rather than slow. A case pending as `IN_PROGRESS` for longer than the claim TTL with no diff --git a/grpc/CLAUDE.md b/grpc/CLAUDE.md index 19e78b3..5705281 100644 --- a/grpc/CLAUDE.md +++ b/grpc/CLAUDE.md @@ -145,7 +145,7 @@ the repository's `.seconds` / `.total` unit naming and the corresponding validat `grpc-spring-boot-starter` does not pull the JPA ledger adapter transitively (11-M9). A deployment that needs durable operation idempotency adds `:grpc:grpc-operation-ledger-jpa` explicitly, defines a `GrpcOperationLedger` bean (`new JpaGrpcOperationLedger(repository)`), includes that adapter's Flyway -location `db/migration/grpc` (V001 table, V002 retention index), and sets the flag below. A deployment +location `db/migration/grpc` (V001 table, V002 retention index, V003 claim attempt), and sets the flag below. A deployment that does not adopt the ledger pays no Spring Data JPA classpath cost. | Key (`app.grpc.platform.*`) | Default | What it does | @@ -165,9 +165,9 @@ optional build is not part of the shipped runtime. They join the registry when a adopts the platform. **Why the starter assembles the interceptor (03-11 (b)), and not (a).** The review proposed (a): -delete the flag and judge by whether a ledger bean exists. The owner chose (b): the flag stays and -means what it says — when true, the decision function is assembled and its TTL comes from -configuration. (a) would have removed the false promise but left every adopter to rebuild the same +delete the flag and judge by whether a ledger bean exists. (b) was chosen and is recorded in +ADR-GRPC-004 (re-review 2026-11-03, RR-02-07): the flag stays and means what it says — when true, the +decision function is assembled and its TTL comes from configuration. (a) would have removed the false promise but left every adopter to rebuild the same assembly and TTL validation by hand; (b) keeps that in one place. The interceptor bean has no `@ConditionalOnMissingBean` on purpose: a replacement would silently ignore `operation-ledger-claim-ttl`. Duplicates of a running operation are refused (`REJECT_IN_PROGRESS`), not told to wait; a @@ -179,7 +179,10 @@ server, so assembling the decision does not wire the platform into `adapter:inbo The adopter injects the bean by type and calls it from its own `IDEMPOTENCY` stage interceptor (`GrpcServerInterceptorChain.builder().stage(GrpcServerInterceptorStage.IDEMPOTENCY, ...)`) or from its typed service adapter: `decide(...)` before the mutation, `recordCommit(...)` inside the mutation's -transaction, `recordFailure(...)` on a terminal failure. +transaction, `recordFailure(...)` on a terminal failure. Both record calls take the `GrpcClaimToken` +the `PROCEED` decision carried: a commit by an attempt whose claim was taken over is refused (its +transaction rolls back) and its failure is ignored, so a slow owner cannot write over the attempt that +took its claim (re-review RR-02-04, ADR-GRPC-004). **Purge scheduling is the adopter's.** `GrpcOperationLedgerRetention.purgeExpired(now)` deletes expired finished rows in bounded batches (500 per statement) until a batch comes back short. The diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcClaimToken.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcClaimToken.java new file mode 100644 index 0000000..7b5fe35 --- /dev/null +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcClaimToken.java @@ -0,0 +1,35 @@ +package dev.caskeleton.grpc.api.ledger; + +/** + * Which attempt holds a claim: the fencing token a commit or a failure record must present. + * + *

A claim can change hands. A stale one is taken over after the claim TTL, and a failed one is + * re-claimed by the next retry; both make a new attempt of the same operation. The owner that lost + * the claim may still be running, and when it finishes it records its outcome. Without a token that + * record lands on the new attempt's claim: an old owner's failure turned the new attempt's claim + * {@code FAILED_TERMINAL}, a third retry re-claimed it, and two attempts ran the mutation at once + * (re-review 2026-11-03, RR-02-04). With it, the ledger records an outcome only for the attempt + * that still holds the claim. + * + *

A counter rather than the claim moment: it increases by one with every hand-over, so two + * attempts never share a value, whatever the clocks of the processes that claimed it. + * + * @param attempt one for the first claim of an operation, one more for each hand-over + */ +public record GrpcClaimToken(long attempt) { + + /** The token of the claim that created the record. */ + public static final GrpcClaimToken FIRST = new GrpcClaimToken(1); + + /** Requires a positive attempt number. */ + public GrpcClaimToken { + if (attempt < 1) { + throw new IllegalArgumentException("a claim attempt is numbered from one, was " + attempt); + } + } + + /** The token of the attempt that takes this claim over. */ + public GrpcClaimToken next() { + return new GrpcClaimToken(Math.addExact(attempt, 1)); + } +} diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java index 2c27515..381979a 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java @@ -20,7 +20,8 @@ public interface GrpcOperationLedger { * implementation admits both concurrent attempts, which is the exact failure the ledger exists to * prevent, and it does so only under the load where it matters. * - * @return the existing record when one was already claimed, empty when this call created it + * @return the existing record when one was already claimed, empty when this call created it — the + * caller then holds the claim as {@link GrpcClaimToken#FIRST} */ Optional claim( GrpcOperationIdentity identity, String requestFingerprint, Instant now); @@ -34,11 +35,34 @@ public interface GrpcOperationLedger { *

Called inside the business transaction wherever the datastore allows it. When the ledger and * the mutation commit separately, there is a window in which the mutation is durable and the * claim is not, and a retry in that window runs the mutation twice. + * + *

Only the attempt that holds the claim commits it. An attempt whose claim was taken over or + * re-claimed since is refused with an exception, so its business transaction rolls back rather + * than committing beside the attempt that now owns the operation. Of two concurrent commits of + * one claim, exactly one succeeds. + * + * @param claimToken the token of the attempt committing, from the claim it won + * @throws IllegalStateException when there is no claim, it already finished, or it is no longer + * held by {@code claimToken} */ - void markCommitted(GrpcOperationIdentity identity, String outcomeReference, Instant now); + void markCommitted( + GrpcOperationIdentity identity, + GrpcClaimToken claimToken, + String outcomeReference, + Instant now); - /** Marks a claim terminally failed. */ - void markFailed(GrpcOperationIdentity identity, Instant now); + /** + * Marks a claim terminally failed, if {@code claimToken} still holds it. + * + *

An attempt that lost its claim reports a failure the operation no longer has: another + * attempt is running it. That record is ignored rather than written over the running attempt's + * claim. + * + * @param claimToken the token of the attempt that failed, from the claim it won + * @return true when the failure was recorded; false when the claim is missing, already finished, + * or held by another attempt, in which case nothing changed + */ + boolean markFailed(GrpcOperationIdentity identity, GrpcClaimToken claimToken, Instant now); /** * Re-claims a {@code FAILED_TERMINAL} record for a new attempt of the same request. @@ -48,9 +72,11 @@ public interface GrpcOperationLedger { * request, is left untouched. A retry that proceeds without winning this leaves its commit with * no owner and runs concurrently with every other retry of the same key. * - * @return true when this call now owns the record as a fresh attempt claimed at {@code now} + * @return the new attempt's token when this call now owns the record as a fresh attempt claimed + * at {@code now}; empty otherwise */ - boolean reclaimFailed(GrpcOperationIdentity identity, String requestFingerprint, Instant now); + Optional reclaimFailed( + GrpcOperationIdentity identity, String requestFingerprint, Instant now); /** * Takes over an {@code IN_PROGRESS} claim whose owner is presumed dead, for a new attempt of the @@ -73,10 +99,15 @@ public interface GrpcOperationLedger { * When an adopter commits them separately, the abandoned attempt's outcome is unknown and a * takeover can run the mutation twice. * - * @return true when this call now owns the record as a fresh attempt claimed at {@code now} + *

The owner taken over may be slow rather than dead. Its later commit is refused and its later + * failure ignored, because it no longer holds the claim token; that is what keeps the takeover + * from becoming a second concurrent run. + * + * @return the new attempt's token when this call now owns the record as a fresh attempt claimed + * at {@code now}; empty otherwise * @throws IllegalArgumentException when {@code claimedBefore} is after {@code now} */ - boolean reclaimStale( + Optional reclaimStale( GrpcOperationIdentity identity, String requestFingerprint, Instant claimedBefore, diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerRecord.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerRecord.java index 2bc0942..22f8632 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerRecord.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerRecord.java @@ -10,6 +10,9 @@ import java.util.Optional; * A caller that reuses a key for a different request has made a mistake, and returning the first * request's answer would hide it; comparing fingerprints turns that into a {@code * FAILED_PRECONDITION} the caller can act on. + * + *

{@code claimToken} names the attempt that holds (or last held) the claim; see {@link + * GrpcClaimToken}. */ public record GrpcOperationLedgerRecord( GrpcOperationIdentity identity, @@ -17,9 +20,10 @@ public record GrpcOperationLedgerRecord( String requestFingerprint, Optional outcomeReference, Instant claimedAt, - Optional completedAt) { + Optional completedAt, + GrpcClaimToken claimToken) { - /** Requires an identity, a state, a fingerprint and a claim moment. */ + /** Requires an identity, a state, a fingerprint, a claim moment and a claim token. */ public GrpcOperationLedgerRecord { if (identity == null || state == null) { throw new IllegalArgumentException("a ledger record needs an identity and a state"); @@ -30,6 +34,9 @@ public record GrpcOperationLedgerRecord( if (claimedAt == null || outcomeReference == null || completedAt == null) { throw new IllegalArgumentException("a ledger record needs its timestamps and Optionals"); } + if (claimToken == null) { + throw new IllegalArgumentException("a ledger record names the attempt that holds it"); + } if (state.terminal() && completedAt.isEmpty()) { throw new IllegalArgumentException("a terminal ledger record records when it finished"); } @@ -40,16 +47,26 @@ public record GrpcOperationLedgerRecord( } } - /** A fresh claim. */ + /** A fresh claim, held by the first attempt. */ public static GrpcOperationLedgerRecord claim( GrpcOperationIdentity identity, String requestFingerprint, Instant claimedAt) { + return claim(identity, requestFingerprint, claimedAt, GrpcClaimToken.FIRST); + } + + /** A claim in progress, held by the attempt {@code claimToken} names. */ + public static GrpcOperationLedgerRecord claim( + GrpcOperationIdentity identity, + String requestFingerprint, + Instant claimedAt, + GrpcClaimToken claimToken) { return new GrpcOperationLedgerRecord( identity, GrpcOperationLedgerState.IN_PROGRESS, requestFingerprint, Optional.empty(), claimedAt, - Optional.empty()); + Optional.empty(), + claimToken); } /** Whether {@code candidateFingerprint} is the same request this claim was made for. */ diff --git a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java index 0c625fc..d91f8de 100644 --- a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java +++ b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java @@ -88,7 +88,7 @@ public abstract class GrpcOperationLedgerContract { ledger().claim(ID, FINGERPRINT, NOW); // when - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); // then assertThat(ledger().find(ID)) @@ -106,7 +106,8 @@ public abstract class GrpcOperationLedgerContract { @Test void committingWithoutAClaimIsRefused() { // when - Throwable refused = catchThrowable(() -> ledger().markCommitted(ID, OUTCOME, NOW)); + Throwable refused = + catchThrowable(() -> ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW)); // then assertThat(refused) @@ -118,12 +119,15 @@ public abstract class GrpcOperationLedgerContract { void aCommittedClaimCannotBeCommittedAgain() { // given ledger().claim(ID, FINGERPRINT, NOW); - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); // when Throwable refused = catchThrowable( - () -> ledger().markCommitted(ID, "outcome://document/99", NOW.plusSeconds(2))); + () -> + ledger() + .markCommitted( + ID, GrpcClaimToken.FIRST, "outcome://document/99", NOW.plusSeconds(2))); // then assertThat(refused) @@ -137,7 +141,7 @@ public abstract class GrpcOperationLedgerContract { ledger().claim(ID, FINGERPRINT, NOW); // when - ledger().markFailed(ID, NOW.plusSeconds(1)); + ledger().markFailed(ID, GrpcClaimToken.FIRST, NOW.plusSeconds(1)); // then assertThat(ledger().find(ID)) @@ -153,7 +157,8 @@ public abstract class GrpcOperationLedgerContract { // when Throwable refused = - catchThrowable(() -> ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(2))); + catchThrowable( + () -> ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(2))); // then assertThat(refused) @@ -167,10 +172,11 @@ public abstract class GrpcOperationLedgerContract { failedClaim(); // when - boolean reclaimed = ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + Optional reclaimed = + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); // then - assertThat(reclaimed).isTrue(); + assertThat(reclaimed).isPresent(); } @Test @@ -196,10 +202,11 @@ public abstract class GrpcOperationLedgerContract { void aReclaimedClaimCanBeCommitted() { // given failedClaim(); - ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + GrpcClaimToken reclaimed = + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)).orElseThrow(); // when - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(3)); + ledger().markCommitted(ID, reclaimed, OUTCOME, NOW.plusSeconds(3)); // then assertThat(ledger().find(ID)) @@ -212,27 +219,28 @@ public abstract class GrpcOperationLedgerContract { void aFailedClaimIsNotReclaimedForADifferentRequest() { failedClaim(); - assertThat(ledger().reclaimFailed(ID, OTHER_FINGERPRINT, NOW.plusSeconds(2))).isFalse(); + assertThat(ledger().reclaimFailed(ID, OTHER_FINGERPRINT, NOW.plusSeconds(2))).isEmpty(); } @Test void anInProgressClaimIsNotReclaimed() { ledger().claim(ID, FINGERPRINT, NOW); - assertThat(ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(1))).isFalse(); + assertThat(ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(1))).isEmpty(); } @Test void aCommittedClaimIsNotReclaimed() { // given ledger().claim(ID, FINGERPRINT, NOW); - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); // when - boolean reclaimed = ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + Optional reclaimed = + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); // then - assertThat(reclaimed).isFalse(); + assertThat(reclaimed).isEmpty(); } @Test @@ -242,7 +250,7 @@ public abstract class GrpcOperationLedgerContract { // when List reclaimed = - concurrently(() -> ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2))); + concurrently(() -> ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)).isPresent()); // then assertThat(reclaimed).containsExactlyInAnyOrder(true, false); @@ -255,10 +263,10 @@ public abstract class GrpcOperationLedgerContract { ledger().claim(ID, FINGERPRINT, NOW); // when - boolean reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + Optional reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); // then - assertThat(reclaimed).isTrue(); + assertThat(reclaimed).isPresent(); } @Test @@ -285,10 +293,10 @@ public abstract class GrpcOperationLedgerContract { void aTakenOverStaleClaimCanBeCommitted() { // given ledger().claim(ID, FINGERPRINT, NOW); - ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + GrpcClaimToken takenOver = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER).orElseThrow(); // when - ledger().markCommitted(ID, OUTCOME, LATER.plusSeconds(1)); + ledger().markCommitted(ID, takenOver, OUTCOME, LATER.plusSeconds(1)); // then assertThat(ledger().find(ID)) @@ -304,10 +312,11 @@ public abstract class GrpcOperationLedgerContract { ledger().claim(ID, FINGERPRINT, NOW); // when - boolean reclaimed = ledger().reclaimStale(ID, FINGERPRINT, NOW, NOW.plus(CLAIM_TTL)); + Optional reclaimed = + ledger().reclaimStale(ID, FINGERPRINT, NOW, NOW.plus(CLAIM_TTL)); // then - assertThat(reclaimed).isFalse(); + assertThat(reclaimed).isEmpty(); } @Test @@ -317,10 +326,11 @@ public abstract class GrpcOperationLedgerContract { ledger().claim(ID, FINGERPRINT, NOW); // when - boolean reclaimed = ledger().reclaimStale(ID, OTHER_FINGERPRINT, CUTOFF, LATER); + Optional reclaimed = + ledger().reclaimStale(ID, OTHER_FINGERPRINT, CUTOFF, LATER); // then - assertThat(reclaimed).isFalse(); + assertThat(reclaimed).isEmpty(); } @Test @@ -328,13 +338,13 @@ public abstract class GrpcOperationLedgerContract { void aCommittedClaimIsNotTakenOverAsStale() { // given ledger().claim(ID, FINGERPRINT, NOW); - ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); // when - boolean reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + Optional reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); // then - assertThat(reclaimed).isFalse(); + assertThat(reclaimed).isEmpty(); } @Test @@ -344,20 +354,20 @@ public abstract class GrpcOperationLedgerContract { failedClaim(); // when - boolean reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + Optional reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); // then - assertThat(reclaimed).isFalse(); + assertThat(reclaimed).isEmpty(); } @Test @DisplayName("claim이 없으면 stale 회수는 아무것도 만들지 않는다") void takingOverAMissingClaimCreatesNothing() { // when - boolean reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + Optional reclaimed = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); // then - assertThat(reclaimed).isFalse(); + assertThat(reclaimed).isEmpty(); assertThat(ledger().find(ID)).isEmpty(); } @@ -369,10 +379,10 @@ public abstract class GrpcOperationLedgerContract { ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); // when - boolean again = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + Optional again = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); // then - assertThat(again).isFalse(); + assertThat(again).isEmpty(); } @Test @@ -383,7 +393,7 @@ public abstract class GrpcOperationLedgerContract { // when List reclaimed = - concurrently(() -> ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER)); + concurrently(() -> ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER).isPresent()); // then assertThat(reclaimed).containsExactlyInAnyOrder(true, false); @@ -402,6 +412,126 @@ public abstract class GrpcOperationLedgerContract { assertThat(refused).isInstanceOf(IllegalArgumentException.class).hasMessageContaining("cutoff"); } + @Test + @DisplayName("claim 을 넘겨받을 때마다 claim token 은 다음 시도 번호가 된다") + void eachHandOverIssuesTheNextClaimToken() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + GrpcClaimToken takenOver = ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER).orElseThrow(); + + // then + assertThat(takenOver).isEqualTo(GrpcClaimToken.FIRST.next()); + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::claimToken) + .isEqualTo(takenOver); + } + + /** + * The re-review probe's sequence (2026-11-03, RR-02-04): A claims and is slow, B takes the claim + * over after the TTL, then A reports a failure. Recording it would mark B's running attempt + * failed, and the next retry would re-claim it and run beside B. + */ + @Test + @DisplayName("TTL 뒤 넘겨받은 claim 에 옛 owner 가 실패를 기록하면 무시되고 새 시도는 진행 중으로 남는다") + void aFailureReportedByTheOwnerThatLostTheClaimIsIgnored() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // when + boolean recorded = ledger().markFailed(ID, GrpcClaimToken.FIRST, LATER.plusSeconds(1)); + + // then + assertThat(recorded).isFalse(); + assertThat(ledger().find(ID)) + .get() + .extracting( + GrpcOperationLedgerRecord::state, + GrpcOperationLedgerRecord::claimedAt, + GrpcOperationLedgerRecord::completedAt) + .containsExactly(GrpcOperationLedgerState.IN_PROGRESS, LATER, Optional.empty()); + } + + @Test + @DisplayName("옛 owner 의 실패가 무시되므로 다음 재시도는 진행 중인 새 시도를 다시 claim 하지 못한다") + void aRetryCannotReclaimTheRunningAttemptAfterTheLostOwnerFailed() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + ledger().markFailed(ID, GrpcClaimToken.FIRST, LATER.plusSeconds(1)); + + // when + Optional third = ledger().reclaimFailed(ID, FINGERPRINT, LATER.plusSeconds(2)); + + // then + assertThat(third).isEmpty(); + } + + @Test + @DisplayName("TTL 뒤 넘겨받은 claim 에 옛 owner 가 commit 하면 거부되고 새 시도는 진행 중으로 남는다") + void aCommitByTheOwnerThatLostTheClaimIsRefused() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // when + Throwable refused = + catchThrowable( + () -> ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, LATER.plusSeconds(1))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("no longer held by attempt 1"); + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::state) + .isEqualTo(GrpcOperationLedgerState.IN_PROGRESS); + } + + @Test + @DisplayName("실패 뒤 다시 claim 된 기록에 이전 시도가 commit 하면 거부된다") + void aCommitByTheAttemptBeforeAFailedReclaimIsRefused() { + // given + failedClaim(); + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + + // when + Throwable refused = + catchThrowable( + () -> ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(3))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("no longer held by attempt 1"); + } + + @Test + @DisplayName("한 claim 을 동시에 commit 하면 하나만 성공한다") + void concurrentCommitsOfOneClaimHaveExactlyOneWinner() throws Exception { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + List committed = + concurrently( + () -> { + try { + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); + return true; + } catch (IllegalStateException refused) { + return false; + } + }); + + // then + assertThat(committed).containsExactlyInAnyOrder(true, false); + } + @Test @DisplayName("보존 기간이 지난 COMMITTED·FAILED_TERMINAL 행은 정리된다") void finishedRecordsCompletedBeforeTheCutoffArePurged() { @@ -489,12 +619,12 @@ public abstract class GrpcOperationLedgerContract { // cutoff is compared with. private void failedClaim(GrpcOperationIdentity identity, Instant claimedAt) { ledger().claim(identity, FINGERPRINT, claimedAt); - ledger().markFailed(identity, claimedAt.plusSeconds(1)); + ledger().markFailed(identity, GrpcClaimToken.FIRST, claimedAt.plusSeconds(1)); } private void committedClaim(GrpcOperationIdentity identity, Instant claimedAt) { ledger().claim(identity, FINGERPRINT, claimedAt); - ledger().markCommitted(identity, OUTCOME, claimedAt.plusSeconds(1)); + ledger().markCommitted(identity, GrpcClaimToken.FIRST, OUTCOME, claimedAt.plusSeconds(1)); } private static GrpcOperationIdentity identityWithKeyHash(char hexDigit) { diff --git a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java index 5547914..bc26942 100644 --- a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java +++ b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java @@ -6,15 +6,18 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; /** * A hand-rolled ledger for tests of code that depends on {@link GrpcOperationLedger}. * *

{@code putIfAbsent} on a concurrent map is the in-memory equivalent of the unique constraint - * the real adapter relies on, and {@code replace(key, expected, next)} of its conditional update, - * so claim, re-claim, stale takeover and purge are atomic here for the same reason they are there. - * The transitions match the JPA adapter's because {@link GrpcOperationLedgerContract} runs against - * both; a fake that allowed more would let the policy tests pass for code the database refuses. + * the real adapter relies on, and {@code replace(key, expected, next)} and {@code computeIfPresent} + * of its conditional updates, so claim, re-claim, stale takeover, commit, failure and purge are + * atomic here for the same reason they are there. A commit or failure is recorded only for the + * attempt whose {@link GrpcClaimToken} still holds the claim. The transitions match the JPA + * adapter's because {@link GrpcOperationLedgerContract} runs against both; a fake that allowed more + * would let the policy tests pass for code the database refuses. */ public final class InMemoryOperationLedger implements GrpcOperationLedger { @@ -46,16 +49,19 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { } @Override - public void markCommitted(GrpcOperationIdentity identity, String outcomeReference, Instant now) { + public void markCommitted( + GrpcOperationIdentity identity, + GrpcClaimToken claimToken, + String outcomeReference, + Instant now) { GrpcOperationLedgerRecord committed = records.computeIfPresent( identity.storageKey(), - (key, existing) -> - finish( - existing, - GrpcOperationLedgerState.COMMITTED, - Optional.of(outcomeReference), - now)); + (key, existing) -> { + requireHeldBy(existing, claimToken); + return finish( + existing, GrpcOperationLedgerState.COMMITTED, Optional.of(outcomeReference), now); + }); if (committed == null) { throw new IllegalStateException( "cannot commit operation '" + identity.storageKey() + "': no claim exists"); @@ -63,28 +69,36 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { } @Override - public void markFailed(GrpcOperationIdentity identity, Instant now) { + public boolean markFailed( + GrpcOperationIdentity identity, GrpcClaimToken claimToken, Instant now) { + AtomicBoolean recorded = new AtomicBoolean(); records.computeIfPresent( identity.storageKey(), - (key, existing) -> - finish(existing, GrpcOperationLedgerState.FAILED_TERMINAL, Optional.empty(), now)); + (key, existing) -> { + if (existing.state() != GrpcOperationLedgerState.IN_PROGRESS + || !existing.claimToken().equals(claimToken)) { + return existing; + } + recorded.set(true); + return finish(existing, GrpcOperationLedgerState.FAILED_TERMINAL, Optional.empty(), now); + }); + return recorded.get(); } @Override - public boolean reclaimFailed( + public Optional reclaimFailed( GrpcOperationIdentity identity, String requestFingerprint, Instant now) { GrpcOperationLedgerRecord current = records.get(identity.storageKey()); - return current != null - && current.state() == GrpcOperationLedgerState.FAILED_TERMINAL - && current.sameRequestAs(requestFingerprint) - && records.replace( - identity.storageKey(), - current, - GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now)); + if (current == null + || current.state() != GrpcOperationLedgerState.FAILED_TERMINAL + || !current.sameRequestAs(requestFingerprint)) { + return Optional.empty(); + } + return handOver(identity, requestFingerprint, current, now); } @Override - public boolean reclaimStale( + public Optional reclaimStale( GrpcOperationIdentity identity, String requestFingerprint, Instant claimedBefore, @@ -94,14 +108,28 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { "a stale-claim cutoff after now would leave the new claim stale as well"); } GrpcOperationLedgerRecord current = records.get(identity.storageKey()); - return current != null - && current.state() == GrpcOperationLedgerState.IN_PROGRESS - && current.claimedAt().isBefore(claimedBefore) - && current.sameRequestAs(requestFingerprint) - && records.replace( + if (current == null + || current.state() != GrpcOperationLedgerState.IN_PROGRESS + || !current.claimedAt().isBefore(claimedBefore) + || !current.sameRequestAs(requestFingerprint)) { + return Optional.empty(); + } + return handOver(identity, requestFingerprint, current, now); + } + + // The conditional replace is the compare-and-set: of two hand-overs of the same record, one wins. + private Optional handOver( + GrpcOperationIdentity identity, + String requestFingerprint, + GrpcOperationLedgerRecord current, + Instant now) { + GrpcClaimToken next = current.claimToken().next(); + return records.replace( identity.storageKey(), current, - GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now)); + GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now, next)) + ? Optional.of(next) + : Optional.empty(); } @Override @@ -132,6 +160,21 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { records.remove(identity.storageKey()); } + // An attempt that lost the claim must not commit beside the attempt that holds it; checked + // before the state, so a lost owner hears that it lost rather than that the row finished. + private static void requireHeldBy(GrpcOperationLedgerRecord existing, GrpcClaimToken claimToken) { + if (!existing.claimToken().equals(claimToken)) { + throw new IllegalStateException( + "operation '" + + existing.identity().storageKey() + + "' is no longer held by attempt " + + claimToken.attempt() + + "; attempt " + + existing.claimToken().attempt() + + " holds it"); + } + } + // Only an IN_PROGRESS claim finishes; a terminal row is re-opened by reclaimFailed alone. private static GrpcOperationLedgerRecord finish( GrpcOperationLedgerRecord existing, @@ -152,6 +195,7 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { existing.requestFingerprint(), outcomeReference, existing.claimedAt(), - Optional.of(now)); + Optional.of(now), + existing.claimToken()); } } diff --git a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java index 6e28dfa..d84695d 100644 --- a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java +++ b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java @@ -16,7 +16,8 @@ final class GrpcOperationLedgerMigrations { private static final List IN_ORDER = List.of( "db/migration/grpc/V001__create_grpc_operation_ledger.sql", - "db/migration/grpc/V002__index_grpc_operation_ledger_finished_rows.sql"); + "db/migration/grpc/V002__index_grpc_operation_ledger_finished_rows.sql", + "db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql"); private GrpcOperationLedgerMigrations() {} diff --git a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java index 7888f7e..ec357db 100644 --- a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java +++ b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java @@ -2,6 +2,7 @@ package dev.caskeleton.grpc.ledger; import static org.assertj.core.api.Assertions.assertThat; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerContract; @@ -102,14 +103,15 @@ class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract { String fingerprint = "sha256:0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0"; Instant now = Instant.parse("2026-09-25T10:00:00Z"); ledger.claim(id, fingerprint, now); - ledger.markFailed(id, now.plusSeconds(1)); + ledger.markFailed(id, GrpcClaimToken.FIRST, now.plusSeconds(1)); // when transactions.executeWithoutResult( status -> { adapter.claim(id, fingerprint, now.plusSeconds(2)); - adapter.reclaimFailed(id, fingerprint, now.plusSeconds(2)); - adapter.markCommitted(id, "outcome://document/42", now.plusSeconds(3)); + GrpcClaimToken reclaimed = + adapter.reclaimFailed(id, fingerprint, now.plusSeconds(2)).orElseThrow(); + adapter.markCommitted(id, reclaimed, "outcome://document/42", now.plusSeconds(3)); }); // then @@ -142,8 +144,11 @@ class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract { transactions.executeWithoutResult( status -> { adapter.claim(id, fingerprint, retryAt); - adapter.reclaimStale(id, fingerprint, retryAt.minusSeconds(300), retryAt); - adapter.markCommitted(id, "outcome://document/42", retryAt.plusSeconds(1)); + GrpcClaimToken takenOver = + adapter + .reclaimStale(id, fingerprint, retryAt.minusSeconds(300), retryAt) + .orElseThrow(); + adapter.markCommitted(id, takenOver, "outcome://document/42", retryAt.plusSeconds(1)); }); // then @@ -188,33 +193,36 @@ class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract { @Override public void markCommitted( - GrpcOperationIdentity identity, String outcomeReference, Instant now) { + GrpcOperationIdentity identity, + GrpcClaimToken claimToken, + String outcomeReference, + Instant now) { transactions.executeWithoutResult( - status -> delegate.markCommitted(identity, outcomeReference, now)); + status -> delegate.markCommitted(identity, claimToken, outcomeReference, now)); } @Override - public void markFailed(GrpcOperationIdentity identity, Instant now) { - transactions.executeWithoutResult(status -> delegate.markFailed(identity, now)); - } - - @Override - public boolean reclaimFailed( - GrpcOperationIdentity identity, String requestFingerprint, Instant now) { + public boolean markFailed( + GrpcOperationIdentity identity, GrpcClaimToken claimToken, Instant now) { return Boolean.TRUE.equals( - transactions.execute( - status -> delegate.reclaimFailed(identity, requestFingerprint, now))); + transactions.execute(status -> delegate.markFailed(identity, claimToken, now))); } @Override - public boolean reclaimStale( + public Optional reclaimFailed( + GrpcOperationIdentity identity, String requestFingerprint, Instant now) { + return transactions.execute( + status -> delegate.reclaimFailed(identity, requestFingerprint, now)); + } + + @Override + public Optional reclaimStale( GrpcOperationIdentity identity, String requestFingerprint, Instant claimedBefore, Instant now) { - return Boolean.TRUE.equals( - transactions.execute( - status -> delegate.reclaimStale(identity, requestFingerprint, claimedBefore, now))); + return transactions.execute( + status -> delegate.reclaimStale(identity, requestFingerprint, claimedBefore, now)); } @Override diff --git a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerEntity.java b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerEntity.java index 54e26ab..571f36e 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerEntity.java +++ b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerEntity.java @@ -1,5 +1,6 @@ package dev.caskeleton.grpc.ledger; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerState; @@ -56,6 +57,9 @@ public class GrpcOperationLedgerEntity { @Column(name = "completed_at") private Instant completedAt; + @Column(name = "claim_attempt", nullable = false) + private long claimAttempt; + /** JPA requires a no-argument constructor. */ protected GrpcOperationLedgerEntity() { // for JPA @@ -70,6 +74,7 @@ public class GrpcOperationLedgerEntity { this.requestFingerprint = requestFingerprint; this.state = GrpcOperationLedgerState.IN_PROGRESS; this.claimedAt = claimedAt; + this.claimAttempt = GrpcClaimToken.FIRST.attempt(); } /** A new claim, in progress. */ @@ -114,13 +119,26 @@ public class GrpcOperationLedgerEntity { return Optional.ofNullable(completedAt); } + /** The attempt that holds, or last held, the claim. */ + public GrpcClaimToken getClaimToken() { + return new GrpcClaimToken(claimAttempt); + } + /** - * Records a commit with its replayable outcome. + * Records a commit with its replayable outcome, on a row this code already holds. * - * @throws IllegalStateException when the claim already finished; a second terminal transition - * would overwrite the outcome a retry is entitled to receive + *

The row-level statement of the rule. {@link JpaGrpcOperationLedger} applies the same rule as + * one conditional {@code UPDATE} ({@link GrpcOperationLedgerRepository#tryMarkCommitted}), so + * that the database, not a value read earlier, decides between concurrent transitions. + * + * @param claimToken the attempt committing + * @throws IllegalStateException when another attempt holds the claim, or the claim already + * finished; a second terminal transition would overwrite the outcome a retry is entitled to + * receive */ - public void markCommitted(String outcomeReference, Instant completedAt) { + public void markCommitted( + GrpcClaimToken claimToken, String outcomeReference, Instant completedAt) { + requireHeldBy(claimToken); requireInProgress(); if (outcomeReference == null || outcomeReference.isBlank()) { throw new IllegalArgumentException( @@ -131,8 +149,16 @@ public class GrpcOperationLedgerEntity { this.completedAt = completedAt; } - /** Records a terminal failure. */ - public void markFailed(Instant completedAt) { + /** + * Records a terminal failure, on a row this code already holds; the row-level statement of {@link + * GrpcOperationLedgerRepository#tryMarkFailed}. + * + * @param claimToken the attempt that failed + * @throws IllegalStateException when another attempt holds the claim, or the claim already + * finished + */ + public void markFailed(GrpcClaimToken claimToken, Instant completedAt) { + requireHeldBy(claimToken); requireInProgress(); this.state = GrpcOperationLedgerState.FAILED_TERMINAL; this.completedAt = completedAt; @@ -146,7 +172,21 @@ public class GrpcOperationLedgerEntity { requestFingerprint, Optional.ofNullable(outcomeReference), claimedAt, - Optional.ofNullable(completedAt)); + Optional.ofNullable(completedAt), + getClaimToken()); + } + + private void requireHeldBy(GrpcClaimToken claimToken) { + if (claimAttempt != claimToken.attempt()) { + throw new IllegalStateException( + "operation '" + + storageKey + + "' is no longer held by attempt " + + claimToken.attempt() + + "; attempt " + + claimAttempt + + " holds it"); + } } private void requireInProgress() { diff --git a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java index d48c61c..9a04367 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java +++ b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java @@ -11,10 +11,12 @@ import org.springframework.transaction.annotation.Transactional; /** * The narrow persistence interface the ledger adapter needs. * - *

The claim path is deliberately not {@code save}. A claim entity already has a non-null string - * id, so Spring Data JPA treats it as detached and routes {@code save} through {@code merge}; merge - * can overwrite a committed row back to {@code IN_PROGRESS}. Claiming therefore uses one native - * insert with PostgreSQL conflict handling and decides ownership from the affected-row count. + *

No path here is {@code save}. A claim entity already has a non-null string id, so Spring Data + * JPA treats it as detached and routes {@code save} through {@code merge}; merge can overwrite a + * committed row back to {@code IN_PROGRESS}, and a load-check-save commit lets two attempts that + * both read {@code IN_PROGRESS} both write. Claiming is therefore one native insert with PostgreSQL + * conflict handling, and every transition is one conditional update; ownership and every race are + * decided by the affected-row count. */ public interface GrpcOperationLedgerRepository extends Repository { @@ -31,7 +33,8 @@ public interface GrpcOperationLedgerRepository state, outcome_reference, claimed_at, - completed_at + completed_at, + claim_attempt ) VALUES ( :storageKey, :callerFingerprint, @@ -41,7 +44,8 @@ public interface GrpcOperationLedgerRepository 'IN_PROGRESS', NULL, :claimedAt, - NULL + NULL, + 1 ) ON CONFLICT DO NOTHING """; @@ -71,7 +75,11 @@ public interface GrpcOperationLedgerRepository * claim's read-on-conflict does) would otherwise still read {@code FAILED_TERMINAL} and refuse * the commit. * - * @return one when this caller re-claimed the row, zero otherwise + *

Conditional on the attempt the caller read as well, and the new attempt is that one plus + * one: the caller then knows its own fencing token without reading the row again. + * + * @return one when this caller re-claimed the row as attempt {@code readAttempt + 1}, zero + * otherwise */ @Modifying(flushAutomatically = true, clearAutomatically = true) @Transactional @@ -81,15 +89,18 @@ public interface GrpcOperationLedgerRepository UPDATE grpc_operation_ledger SET state = 'IN_PROGRESS', claimed_at = :claimedAt, - completed_at = NULL + completed_at = NULL, + claim_attempt = claim_attempt + 1 WHERE storage_key = :storageKey AND state = 'FAILED_TERMINAL' AND request_fingerprint = :requestFingerprint + AND claim_attempt = :readAttempt """, nativeQuery = true) int tryReclaimFailed( @Param("storageKey") String storageKey, @Param("requestFingerprint") String requestFingerprint, + @Param("readAttempt") long readAttempt, @Param("claimedAt") Instant claimedAt); /** @@ -99,9 +110,11 @@ public interface GrpcOperationLedgerRepository *

Same shape as {@link #tryReclaimFailed}: one conditional update decided by PostgreSQL's row * lock. A concurrent second takeover waits for the winner, re-evaluates the condition against the * winner's row — whose {@code claimed_at} is now {@code :claimedAt}, not before the cutoff — and - * updates nothing. It clears the persistence context for the same reason. + * updates nothing. It clears the persistence context for the same reason. The attempt grows by + * one, as in {@link #tryReclaimFailed}, and that is what fences the owner taken over. * - * @return one when this caller took the row over, zero otherwise + * @return one when this caller took the row over as attempt {@code readAttempt + 1}, zero + * otherwise */ @Modifying(flushAutomatically = true, clearAutomatically = true) @Transactional @@ -109,19 +122,76 @@ public interface GrpcOperationLedgerRepository value = """ UPDATE grpc_operation_ledger - SET claimed_at = :claimedAt + SET claimed_at = :claimedAt, + claim_attempt = claim_attempt + 1 WHERE storage_key = :storageKey AND state = 'IN_PROGRESS' AND request_fingerprint = :requestFingerprint AND claimed_at < :claimedBefore + AND claim_attempt = :readAttempt """, nativeQuery = true) int tryReclaimStale( @Param("storageKey") String storageKey, @Param("requestFingerprint") String requestFingerprint, + @Param("readAttempt") long readAttempt, @Param("claimedBefore") Instant claimedBefore, @Param("claimedAt") Instant claimedAt); + /** + * Commits the claim if attempt {@code claimAttempt} still holds it. + * + *

One conditional update rather than load-check-save: two commits of the same claim both read + * {@code IN_PROGRESS} under load-check-save and both wrote; here PostgreSQL's row lock makes the + * second re-evaluate the condition against the first's {@code COMMITTED} row and update nothing. + * An attempt whose claim was taken over or re-claimed matches nothing for the same reason. + * + * @return one when the claim was committed, zero otherwise + */ + @Modifying(flushAutomatically = true, clearAutomatically = true) + @Transactional + @Query( + value = + """ + UPDATE grpc_operation_ledger + SET state = 'COMMITTED', + outcome_reference = :outcomeReference, + completed_at = :completedAt + WHERE storage_key = :storageKey + AND state = 'IN_PROGRESS' + AND claim_attempt = :claimAttempt + """, + nativeQuery = true) + int tryMarkCommitted( + @Param("storageKey") String storageKey, + @Param("claimAttempt") long claimAttempt, + @Param("outcomeReference") String outcomeReference, + @Param("completedAt") Instant completedAt); + + /** + * Marks the claim terminally failed if attempt {@code claimAttempt} still holds it; the same + * conditional shape as {@link #tryMarkCommitted}. + * + * @return one when the failure was recorded, zero otherwise + */ + @Modifying(flushAutomatically = true, clearAutomatically = true) + @Transactional + @Query( + value = + """ + UPDATE grpc_operation_ledger + SET state = 'FAILED_TERMINAL', + completed_at = :completedAt + WHERE storage_key = :storageKey + AND state = 'IN_PROGRESS' + AND claim_attempt = :claimAttempt + """, + nativeQuery = true) + int tryMarkFailed( + @Param("storageKey") String storageKey, + @Param("claimAttempt") long claimAttempt, + @Param("completedAt") Instant completedAt); + /** * Deletes up to {@code limit} finished rows that completed before {@code completedBefore}, oldest * first. Never an {@code IN_PROGRESS} row. @@ -155,9 +225,6 @@ public interface GrpcOperationLedgerRepository int deleteFinishedBefore( @Param("completedBefore") Instant completedBefore, @Param("limit") int limit); - /** Updates an existing claim after it has been loaded. */ - GrpcOperationLedgerEntity save(GrpcOperationLedgerEntity entity); - /** Reads a claim by its derived storage key. */ Optional findById(String storageKey); diff --git a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java index 55d80bc..9db0c3d 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java +++ b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java @@ -1,5 +1,6 @@ package dev.caskeleton.grpc.ledger; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; @@ -16,6 +17,11 @@ import java.util.Optional; * {@code save}: this entity has a non-null application-assigned id, so save routes through merge * and can overwrite an existing terminal row. * + *

Every transition — re-claim, takeover, commit, failure — is one conditional {@code UPDATE} + * that names the claim attempt ({@link GrpcClaimToken}). An attempt whose claim was handed over + * matches nothing: its commit is refused and its failure ignored, which is what keeps a takeover of + * a slow owner from becoming two concurrent runs (re-review 2026-11-03, RR-02-04). + * *

No transaction annotations here. The commit is meant to happen inside the caller's business * transaction, and a {@code REQUIRES_NEW} on this adapter would put the ledger in its own * transaction — reintroducing the window in which the mutation is durable and the claim is not. @@ -36,6 +42,28 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { public Optional claim( GrpcOperationIdentity identity, String requestFingerprint, Instant now) { requireIdentity(identity); + // Twice at most. Zero rows means another claim holds the key; an empty read after that means + // the row went in between (a purge of an expired record, or a release). Treating that as + // "ours" would run the mutation without a claim row to guard it, so the insert is tried once + // more and decides again. + for (int round = 0; round < 2; round++) { + if (insertClaim(identity, requestFingerprint, now)) { + return Optional.empty(); + } + Optional existing = repository.findById(identity.storageKey()); + if (existing.isPresent()) { + return Optional.of(existing.get().toRecord(identity)); + } + } + throw new IllegalStateException( + "claim for '" + + identity.storageKey() + + "' conflicted twice, but there was no row to read either time; another owner keeps" + + " releasing it. Retry the claim."); + } + + private boolean insertClaim( + GrpcOperationIdentity identity, String requestFingerprint, Instant now) { int inserted = repository.tryInsertClaim( identity.storageKey(), @@ -44,25 +72,10 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { identity.idempotencyKeyHash(), requestFingerprint, now); - if (inserted == 1) { - return Optional.empty(); - } - if (inserted != 0) { + if (inserted != 0 && inserted != 1) { throw new IllegalStateException("claim insert affected an unexpected row count: " + inserted); } - // Zero rows means another claim holds the key. An empty read now means that owner released it - // in between; treating that as "ours" would run the mutation without a claim row to guard it. - return Optional.of( - repository - .findById(identity.storageKey()) - .orElseThrow( - () -> - new IllegalStateException( - "claim for '" - + identity.storageKey() - + "' conflicted, but there was no row to read; the other owner released" - + " it in between. Retry the claim.")) - .toRecord(identity)); + return inserted == 1; } @Override @@ -71,10 +84,30 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { return repository.findById(identity.storageKey()).map(entity -> entity.toRecord(identity)); } + /** + * Commits the claim if {@code claimToken} still holds it, as one conditional update. + * + *

When nothing matched, the row is read only to say why; the refusal itself was the + * database's, and the exception is what rolls back the caller's business transaction. + */ @Override - public void markCommitted(GrpcOperationIdentity identity, String outcomeReference, Instant now) { + public void markCommitted( + GrpcOperationIdentity identity, + GrpcClaimToken claimToken, + String outcomeReference, + Instant now) { requireIdentity(identity); - GrpcOperationLedgerEntity entity = + requireToken(claimToken); + if (outcomeReference == null || outcomeReference.isBlank()) { + throw new IllegalArgumentException( + "a commit records the outcome to replay; without it the row cannot answer a retry"); + } + if (repository.tryMarkCommitted( + identity.storageKey(), claimToken.attempt(), outcomeReference, now) + == 1) { + return; + } + GrpcOperationLedgerEntity current = repository .findById(identity.storageKey()) .orElseThrow( @@ -84,31 +117,54 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { + identity.storageKey() + "': no claim exists. The claim is what a retry reads; committing " + "without one leaves the mutation durable and unguarded.")); - entity.markCommitted(outcomeReference, now); - repository.save(entity); + if (!current.getClaimToken().equals(claimToken)) { + throw new IllegalStateException( + "operation '" + + identity.storageKey() + + "' is no longer held by attempt " + + claimToken.attempt() + + "; attempt " + + current.getClaimToken().attempt() + + " holds it, so this attempt's transaction must roll back"); + } + throw new IllegalStateException( + "operation '" + + identity.storageKey() + + "' is already " + + current.getState() + + " and cannot transition again"); } @Override - public void markFailed(GrpcOperationIdentity identity, Instant now) { + public boolean markFailed( + GrpcOperationIdentity identity, GrpcClaimToken claimToken, Instant now) { requireIdentity(identity); - repository - .findById(identity.storageKey()) - .ifPresent( - entity -> { - entity.markFailed(now); - repository.save(entity); - }); + requireToken(claimToken); + return repository.tryMarkFailed(identity.storageKey(), claimToken.attempt(), now) == 1; } + /** + * Re-claims a failed record. Conditional on the attempt read just before, so the new token is + * known exactly: that attempt plus one. A record that changed in between matches nothing, and + * this caller has lost the race it would have lost anyway. + */ @Override - public boolean reclaimFailed( + public Optional reclaimFailed( GrpcOperationIdentity identity, String requestFingerprint, Instant now) { requireIdentity(identity); - return repository.tryReclaimFailed(identity.storageKey(), requestFingerprint, now) == 1; + Optional read = currentToken(identity); + if (read.isEmpty()) { + return Optional.empty(); + } + return repository.tryReclaimFailed( + identity.storageKey(), requestFingerprint, read.get().attempt(), now) + == 1 + ? Optional.of(read.get().next()) + : Optional.empty(); } @Override - public boolean reclaimStale( + public Optional reclaimStale( GrpcOperationIdentity identity, String requestFingerprint, Instant claimedBefore, @@ -118,8 +174,19 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { throw new IllegalArgumentException( "a stale-claim cutoff after now would leave the new claim stale as well"); } - return repository.tryReclaimStale(identity.storageKey(), requestFingerprint, claimedBefore, now) - == 1; + Optional read = currentToken(identity); + if (read.isEmpty()) { + return Optional.empty(); + } + return repository.tryReclaimStale( + identity.storageKey(), requestFingerprint, read.get().attempt(), claimedBefore, now) + == 1 + ? Optional.of(read.get().next()) + : Optional.empty(); + } + + private Optional currentToken(GrpcOperationIdentity identity) { + return repository.findById(identity.storageKey()).map(GrpcOperationLedgerEntity::getClaimToken); } @Override @@ -139,6 +206,12 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { repository.deleteById(identity.storageKey()); } + private static void requireToken(GrpcClaimToken claimToken) { + if (claimToken == null) { + throw new IllegalArgumentException("a transition names the attempt that holds the claim"); + } + } + private static void requireIdentity(GrpcOperationIdentity identity) { if (identity == null) { throw new IllegalArgumentException("an operation identity is required"); diff --git a/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql b/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql new file mode 100644 index 0000000..23b0368 --- /dev/null +++ b/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql @@ -0,0 +1,20 @@ +-- Claim attempts: the fencing token of the operation ledger (re-review 2026-11-03, RR-02-04). +-- +-- A claim changes hands when a stale one is taken over after the claim TTL or a failed one is +-- re-claimed. The owner that lost it may still be running, and when it finishes it records an +-- outcome. Without knowing which attempt holds the claim, that record landed on the new attempt's +-- claim: an old owner's failure turned a running attempt FAILED_TERMINAL, the next retry re-claimed +-- it, and two attempts ran the mutation at once. +-- +-- claim_attempt is 1 for the first claim and grows by one with every hand-over. A commit or a +-- failure is one conditional UPDATE that names the attempt (WHERE state = 'IN_PROGRESS' AND +-- claim_attempt = :attempt), so a lost owner's record changes nothing and two commits of the same +-- attempt have one winner. +-- +-- Existing rows are their first attempt: no row has been handed over under a token yet. The table is +-- small and the platform is not adopted anywhere, so a plain ALTER with a default is enough. +ALTER TABLE grpc_operation_ledger + ADD COLUMN claim_attempt BIGINT NOT NULL DEFAULT 1; + +ALTER TABLE grpc_operation_ledger + ADD CONSTRAINT ck_grpc_operation_ledger_claim_attempt_positive CHECK (claim_attempt > 0); diff --git a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java index e3aada6..f2f9075 100644 --- a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java +++ b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; @@ -34,12 +35,12 @@ class GrpcOperationLedgerRepositoryTest { "sha256:aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899"; /** - * An in-memory stand-in for Spring Data's non-null-id save semantics. + * An in-memory stand-in for the repository's native statements. * - *

A newly constructed ledger entity already has its string id, so a real {@code - * SimpleJpaRepository.save} treats it as detached and merges it. The old fake used {@code - * putIfAbsent} and therefore tested the database constraint we wished save() would hit rather - * than the merge behavior it actually uses. + *

The insert is {@code putIfAbsent}, the stand-in for the unique constraint. The commit and + * failure are the entity's own transition rules, which state the conditions the native {@code + * UPDATE}s carry (state {@code IN_PROGRESS} and the same claim attempt); a refused transition + * updates zero rows. The re-claims and the purge are certified against PostgreSQL only. */ private static class InMemoryRepository implements GrpcOperationLedgerRepository { private final ConcurrentMap rows = new ConcurrentHashMap<>(); @@ -61,14 +62,19 @@ class GrpcOperationLedgerRepositoryTest { } @Override - public int tryReclaimFailed(String storageKey, String requestFingerprint, Instant claimedAt) { + public int tryReclaimFailed( + String storageKey, String requestFingerprint, long readAttempt, Instant claimedAt) { throw new UnsupportedOperationException( "the re-claim is certified against PostgreSQL by JpaGrpcOperationLedgerContractIT"); } @Override public int tryReclaimStale( - String storageKey, String requestFingerprint, Instant claimedBefore, Instant claimedAt) { + String storageKey, + String requestFingerprint, + long readAttempt, + Instant claimedBefore, + Instant claimedAt) { throw new UnsupportedOperationException( "the stale takeover is certified against PostgreSQL by JpaGrpcOperationLedgerContractIT"); } @@ -80,9 +86,32 @@ class GrpcOperationLedgerRepositoryTest { } @Override - public GrpcOperationLedgerEntity save(GrpcOperationLedgerEntity entity) { - rows.put(entity.getStorageKey(), entity); - return entity; + public int tryMarkCommitted( + String storageKey, long claimAttempt, String outcomeReference, Instant completedAt) { + GrpcOperationLedgerEntity row = rows.get(storageKey); + if (row == null) { + return 0; + } + try { + row.markCommitted(new GrpcClaimToken(claimAttempt), outcomeReference, completedAt); + return 1; + } catch (IllegalStateException refused) { + return 0; + } + } + + @Override + public int tryMarkFailed(String storageKey, long claimAttempt, Instant completedAt) { + GrpcOperationLedgerEntity row = rows.get(storageKey); + if (row == null) { + return 0; + } + try { + row.markFailed(new GrpcClaimToken(claimAttempt), completedAt); + return 1; + } catch (IllegalStateException refused) { + return 0; + } } @Override @@ -139,6 +168,7 @@ class GrpcOperationLedgerRepositoryTest { } @Test + @DisplayName("충돌한 행이 다시 시도해도 계속 사라지면 소유로 보지 않고 거부한다") void aConflictWhoseRowVanishedBeforeTheReadIsNotTakenAsOwnership() { // given InMemoryRepository conflictThenReleased = @@ -166,6 +196,54 @@ class GrpcOperationLedgerRepositoryTest { .hasMessageContaining("no row to read"); } + /** + * A purge (or release) can delete the conflicting row between the insert and the read. The read + * then finds nothing, and the claim tries the insert once more instead of failing the call + * (re-review 2026-11-03, RR-02-04). + */ + @Test + @DisplayName("충돌한 행이 읽기 전에 사라지면 claim 은 insert 를 한 번 더 시도해 소유한다") + void aConflictWhoseRowVanishedOnceIsClaimedByTheRetriedInsert() { + // given + InMemoryRepository conflictOnceThenFree = + new InMemoryRepository() { + private boolean conflicted; + + @Override + public int tryInsertClaim( + String storageKey, + String callerFingerprint, + String fullMethodName, + String idempotencyKeyHash, + String requestFingerprint, + Instant claimedAt) { + if (!conflicted) { + conflicted = true; + return 0; // a row held the key and was purged before this read + } + return super.tryInsertClaim( + storageKey, + callerFingerprint, + fullMethodName, + idempotencyKeyHash, + requestFingerprint, + claimedAt); + } + }; + GrpcOperationLedger racing = new JpaGrpcOperationLedger(conflictOnceThenFree); + + // when + Optional existing = + racing.claim(identity("tenant-1.actor-1"), FINGERPRINT, NOW); + + // then + assertThat(existing).isEmpty(); + assertThat(racing.find(identity("tenant-1.actor-1"))) + .get() + .extracting(GrpcOperationLedgerRecord::state, GrpcOperationLedgerRecord::claimToken) + .containsExactly(GrpcOperationLedgerState.IN_PROGRESS, GrpcClaimToken.FIRST); + } + @Test @DisplayName("one caller's key does not collide with another's") void callersDoNotCollide() { @@ -179,7 +257,7 @@ class GrpcOperationLedgerRepositoryTest { GrpcOperationIdentity id = identity("tenant-1.actor-1"); ledger.claim(id, FINGERPRINT, NOW); - ledger.markCommitted(id, "outcome://document/42", NOW.plusSeconds(1)); + ledger.markCommitted(id, GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); GrpcOperationLedgerRecord record = ledger.find(id).orElseThrow(); assertThat(record.state()).isEqualTo(GrpcOperationLedgerState.COMMITTED); @@ -192,7 +270,7 @@ class GrpcOperationLedgerRepositoryTest { void retryingACommittedClaimPreservesTheWinner() { GrpcOperationIdentity id = identity("tenant-1.actor-1"); assertThat(ledger.claim(id, FINGERPRINT, NOW)).isEmpty(); - ledger.markCommitted(id, "outcome://document/42", NOW.plusSeconds(1)); + ledger.markCommitted(id, GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); GrpcOperationLedgerRecord retry = ledger.claim(id, OTHER_FINGERPRINT, NOW.plusSeconds(2)).orElseThrow(); @@ -206,7 +284,12 @@ class GrpcOperationLedgerRepositoryTest { @DisplayName("a commit without a claim is refused rather than silently inserted") void committingWithoutAClaimIsRefused() { assertThatThrownBy( - () -> ledger.markCommitted(identity("tenant-1.actor-1"), "outcome://document/42", NOW)) + () -> + ledger.markCommitted( + identity("tenant-1.actor-1"), + GrpcClaimToken.FIRST, + "outcome://document/42", + NOW)) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("durable and unguarded"); } @@ -216,9 +299,12 @@ class GrpcOperationLedgerRepositoryTest { void aTerminalRowDoesNotTransitionAgain() { GrpcOperationIdentity id = identity("tenant-1.actor-1"); ledger.claim(id, FINGERPRINT, NOW); - ledger.markCommitted(id, "outcome://document/42", NOW.plusSeconds(1)); + ledger.markCommitted(id, GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); - assertThatThrownBy(() -> ledger.markCommitted(id, "outcome://document/99", NOW.plusSeconds(2))) + assertThatThrownBy( + () -> + ledger.markCommitted( + id, GrpcClaimToken.FIRST, "outcome://document/99", NOW.plusSeconds(2))) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("cannot transition again"); } @@ -240,7 +326,7 @@ class GrpcOperationLedgerRepositoryTest { GrpcOperationIdentity id = identity("tenant-1.actor-1"); ledger.claim(id, FINGERPRINT, NOW); - ledger.markFailed(id, NOW.plusSeconds(1)); + ledger.markFailed(id, GrpcClaimToken.FIRST, NOW.plusSeconds(1)); assertThat(ledger.find(id).orElseThrow().state()) .isEqualTo(GrpcOperationLedgerState.FAILED_TERMINAL); @@ -259,7 +345,8 @@ class GrpcOperationLedgerRepositoryTest { FINGERPRINT, Optional.empty(), NOW, - Optional.of(NOW))) + Optional.of(NOW), + GrpcClaimToken.FIRST)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("cannot be replayed"); } diff --git a/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyDecision.java b/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyDecision.java index ecc9f40..ca7ebef 100644 --- a/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyDecision.java +++ b/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyDecision.java @@ -1,5 +1,6 @@ package dev.caskeleton.grpc.idempotency; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import java.time.Duration; import java.util.Optional; @@ -10,9 +11,18 @@ import java.util.Optional; * duplicate". A duplicate of a committed operation should be answered with the original result; a * duplicate of a running one is a race that has to wait or be refused; and a key reused for a * different request is a caller bug that must be reported rather than absorbed. + * + *

A {@code PROCEED} that won a claim carries the claim's {@link GrpcClaimToken}. The caller + * hands it back to {@code recordCommit} or {@code recordFailure}: the ledger records an outcome + * only for the attempt that still holds the claim, so a caller whose claim was taken over cannot + * write over the attempt that took it (re-review 2026-11-03, RR-02-04). */ public record GrpcIdempotencyDecision( - Outcome outcome, Optional outcomeReference, Duration retryAfter, String reason) { + Outcome outcome, + Optional outcomeReference, + Duration retryAfter, + String reason, + Optional claimToken) { /** The five ways an idempotency check can end. */ public enum Outcome { @@ -48,11 +58,31 @@ public record GrpcIdempotencyDecision( if (outcome == Outcome.WAIT_AND_POLL && retryAfter.isZero()) { throw new IllegalArgumentException("a WAIT_AND_POLL decision says how long to wait"); } + if (claimToken == null) { + throw new IllegalArgumentException("the claim token Optional must be present"); + } + if (outcome != Outcome.PROCEED && claimToken.isPresent()) { + throw new IllegalArgumentException("only a PROCEED decision can hold a claim"); + } } - /** Proceed with the operation. */ + /** Proceed with an operation that is not claimed: the method takes no idempotency key. */ public static GrpcIdempotencyDecision proceed(String reason) { - return new GrpcIdempotencyDecision(Outcome.PROCEED, Optional.empty(), Duration.ZERO, reason); + return new GrpcIdempotencyDecision( + Outcome.PROCEED, Optional.empty(), Duration.ZERO, reason, Optional.empty()); + } + + /** + * Proceed as the attempt that holds the claim. + * + * @param claimToken the token to hand back when recording the commit or the failure + */ + public static GrpcIdempotencyDecision proceedHolding(GrpcClaimToken claimToken, String reason) { + if (claimToken == null) { + throw new IllegalArgumentException("a claimed PROCEED carries its claim token"); + } + return new GrpcIdempotencyDecision( + Outcome.PROCEED, Optional.empty(), Duration.ZERO, reason, Optional.of(claimToken)); } /** Replay a stored outcome. */ @@ -61,13 +91,18 @@ public record GrpcIdempotencyDecision( Outcome.REPLAY, Optional.of(outcomeReference), Duration.ZERO, - "an identical request already committed; returning the stored outcome"); + "an identical request already committed; returning the stored outcome", + Optional.empty()); } /** Wait and poll for an in-progress claim. */ public static GrpcIdempotencyDecision waitAndPoll(Duration retryAfter) { return new GrpcIdempotencyDecision( - Outcome.WAIT_AND_POLL, Optional.empty(), retryAfter, "an identical request is in progress"); + Outcome.WAIT_AND_POLL, + Optional.empty(), + retryAfter, + "an identical request is in progress", + Optional.empty()); } /** Refuse a duplicate of an in-progress claim. */ @@ -76,7 +111,8 @@ public record GrpcIdempotencyDecision( Outcome.REJECT_IN_PROGRESS, Optional.empty(), Duration.ZERO, - "an identical request is in progress and this method does not wait for it"); + "an identical request is in progress and this method does not wait for it", + Optional.empty()); } /** Report a key reused for a different request. */ @@ -85,7 +121,8 @@ public record GrpcIdempotencyDecision( Outcome.FINGERPRINT_MISMATCH, Optional.empty(), Duration.ZERO, - "the idempotency key was already used for a different request"); + "the idempotency key was already used for a different request", + Optional.empty()); } /** Whether the use case should run. */ diff --git a/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java b/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java index b48d9a6..5387ffa 100644 --- a/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java +++ b/grpc/grpc-runtime/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java @@ -1,5 +1,6 @@ package dev.caskeleton.grpc.idempotency; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; @@ -21,7 +22,9 @@ import java.util.Optional; * *

A claim does not block its key forever. One still {@code IN_PROGRESS} after the claim TTL is * treated as abandoned by a dead owner and taken over by the next retry of the same request — one - * winner, like the re-claim of a terminal failure. + * winner, like the re-claim of a terminal failure. The owner taken over may only have been slow; it + * still holds the old {@link GrpcClaimToken}, so its later commit is refused and its later failure + * ignored instead of landing on the attempt that took the claim. * *

The ordering matters and is easy to get subtly wrong. The claim is attempted first, * atomically; only then is the existing record examined. Reading first and claiming second leaves a @@ -124,7 +127,8 @@ public final class GrpcIdempotencyInterceptor { callerFingerprint, method, GrpcRequestFingerprint.hashIdempotencyKey(idempotencyKey)); Optional existing = ledger.claim(identity, requestFingerprint, now); if (existing.isEmpty()) { - return GrpcIdempotencyDecision.proceed("claimed; this caller owns the operation"); + return GrpcIdempotencyDecision.proceedHolding( + GrpcClaimToken.FIRST, "claimed; this caller owns the operation"); } GrpcOperationLedgerRecord record = existing.get(); @@ -142,24 +146,32 @@ public final class GrpcIdempotencyInterceptor { + identity.storageKey() + "' has no outcome reference to replay"))); } - if (record.state() == GrpcOperationLedgerState.FAILED_TERMINAL - && ledger.reclaimFailed(identity, requestFingerprint, now)) { + if (record.state() == GrpcOperationLedgerState.FAILED_TERMINAL) { // A terminal failure is not replayed: the caller asked for the operation, it did not happen, // and repeating it is exactly what the key makes safe — but only for the one retry that // re-claims the row. Proceeding unclaimed runs concurrent retries twice and leaves the // commit without an owner. - return GrpcIdempotencyDecision.proceed( - "the previous attempt failed terminally; this caller re-claimed it for a new attempt"); + Optional reclaimed = ledger.reclaimFailed(identity, requestFingerprint, now); + if (reclaimed.isPresent()) { + return GrpcIdempotencyDecision.proceedHolding( + reclaimed.get(), + "the previous attempt failed terminally; this caller re-claimed it for a new attempt"); + } } if (record.state() == GrpcOperationLedgerState.IN_PROGRESS) { Instant cutoff = now.minus(claimTtl); - if (record.claimedAt().isBefore(cutoff) - && ledger.reclaimStale(identity, requestFingerprint, cutoff, now)) { - // The owner outlived the TTL without committing or failing, so it is presumed dead. Its - // mutation did not commit if the commit was recorded in the mutation's own transaction, - // which the ledger contract requires; when an adopter commits them separately, the - // abandoned attempt's outcome is unknown and this re-run can duplicate it. - return GrpcIdempotencyDecision.proceed( + // The owner outlived the TTL without committing or failing, so it is presumed dead. Its + // mutation did not commit if the commit was recorded in the mutation's own transaction, + // which the ledger contract requires; when an adopter commits them separately, the + // abandoned attempt's outcome is unknown and this re-run can duplicate it. An owner that was + // only slow is fenced by the new claim token: its commit is refused, its failure ignored. + Optional takenOver = + record.claimedAt().isBefore(cutoff) + ? ledger.reclaimStale(identity, requestFingerprint, cutoff, now) + : Optional.empty(); + if (takenOver.isPresent()) { + return GrpcIdempotencyDecision.proceedHolding( + takenOver.get(), "the previous claim was abandoned (older than the " + claimTtl + " claim TTL); this caller took it over for a new attempt"); @@ -178,26 +190,42 @@ public final class GrpcIdempotencyInterceptor { *

Called from inside the business transaction wherever the datastore permits it. A ledger * committed after the mutation leaves a window in which the write is durable and the claim is * not. + * + * @param claimToken the token of the {@code PROCEED} decision this attempt ran under + * @throws IllegalStateException when this attempt no longer holds the claim — another attempt + * took it over — so the business transaction rolls back instead of committing a second run */ public void recordCommit( GrpcMethodName method, String callerFingerprint, String idempotencyKey, + GrpcClaimToken claimToken, String outcomeReference, Instant now) { ledger.markCommitted( new GrpcOperationIdentity( callerFingerprint, method, GrpcRequestFingerprint.hashIdempotencyKey(idempotencyKey)), + claimToken, outcomeReference, now); } - /** Records a terminal failure against the claim. */ - public void recordFailure( - GrpcMethodName method, String callerFingerprint, String idempotencyKey, Instant now) { - ledger.markFailed( + /** + * Records a terminal failure against the claim, if this attempt still holds it. + * + * @param claimToken the token of the {@code PROCEED} decision this attempt ran under + * @return false when another attempt holds the claim now; its run is not failed by this one + */ + public boolean recordFailure( + GrpcMethodName method, + String callerFingerprint, + String idempotencyKey, + GrpcClaimToken claimToken, + Instant now) { + return ledger.markFailed( new GrpcOperationIdentity( callerFingerprint, method, GrpcRequestFingerprint.hashIdempotencyKey(idempotencyKey)), + claimToken, now); } } diff --git a/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java index 90c3421..dd50082 100644 --- a/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java +++ b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.grpc.idempotency; import static org.assertj.core.api.Assertions.assertThat; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger; import dev.caskeleton.grpc.core.GrpcMethodName; @@ -42,8 +43,11 @@ class GrpcCompletionReconcilerTest { GrpcCompletionReconciler reconciler = new GrpcCompletionReconciler(query); claim(); reconciler.reconcile(CREATE, CALLER, KEY, null); - ledger.reclaimStale(identity, fingerprint, afterTtl.minus(Duration.ofMinutes(5)), afterTtl); - ledger.markCommitted(identity, "outcome://document/42", afterTtl.plusSeconds(1)); + GrpcClaimToken takenOver = + ledger + .reclaimStale(identity, fingerprint, afterTtl.minus(Duration.ofMinutes(5)), afterTtl) + .orElseThrow(); + ledger.markCommitted(identity, takenOver, "outcome://document/42", afterTtl.plusSeconds(1)); // when GrpcCompletionResolution nextPass = reconciler.reconcile(CREATE, CALLER, KEY, null); @@ -93,6 +97,7 @@ class GrpcCompletionReconcilerTest { ledger.markCommitted( new dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity( CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY)), + GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); @@ -116,6 +121,7 @@ class GrpcCompletionReconcilerTest { ledger.markCommitted( new dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity( CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY)), + GrpcClaimToken.FIRST, "outcome://from-ledger", NOW.plusSeconds(1)); GrpcCompletionReconciler reconciler = new GrpcCompletionReconciler(query); diff --git a/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java index df26ea2..99fa068 100644 --- a/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java +++ b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java @@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.grpc.api.deadline.GrpcDeadlineProfile; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerState; @@ -98,8 +99,10 @@ class GrpcIdempotencyInterceptorTest { void aDuplicateOfACommittedOperationReplays() { GrpcIdempotencyInterceptor interceptor = interceptor(false); String fingerprint = GrpcRequestFingerprint.of("body"); - interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); - interceptor.recordCommit(CREATE, CALLER, KEY, "outcome://document/42", NOW.plusSeconds(1)); + GrpcClaimToken claim = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.recordCommit( + CREATE, CALLER, KEY, claim, "outcome://document/42", NOW.plusSeconds(1)); GrpcIdempotencyDecision replayed = interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2)); @@ -139,8 +142,9 @@ class GrpcIdempotencyInterceptorTest { void aTerminallyFailedOperationMayBeRetried() { GrpcIdempotencyInterceptor interceptor = interceptor(false); String fingerprint = GrpcRequestFingerprint.of("body"); - interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); - interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1)); + GrpcClaimToken claim = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.recordFailure(CREATE, CALLER, KEY, claim, NOW.plusSeconds(1)); assertThat(interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2)).outcome()) .isEqualTo(GrpcIdempotencyDecision.Outcome.PROCEED); @@ -151,8 +155,9 @@ class GrpcIdempotencyInterceptorTest { // given GrpcIdempotencyInterceptor interceptor = interceptor(false); String fingerprint = GrpcRequestFingerprint.of("body"); - interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); - interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1)); + GrpcClaimToken claim = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.recordFailure(CREATE, CALLER, KEY, claim, NOW.plusSeconds(1)); // when interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2)); @@ -172,8 +177,9 @@ class GrpcIdempotencyInterceptorTest { // given GrpcIdempotencyInterceptor interceptor = interceptor(false); String fingerprint = GrpcRequestFingerprint.of("body"); - interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); - interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1)); + GrpcClaimToken claim = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.recordFailure(CREATE, CALLER, KEY, claim, NOW.plusSeconds(1)); CountDownLatch start = new CountDownLatch(1); Callable retry = () -> { @@ -235,6 +241,70 @@ class GrpcIdempotencyInterceptorTest { .contains(AFTER_TTL); } + @Test + @DisplayName("claim 을 이긴 PROCEED 는 그 시도의 claim token 을 들고 있다") + void aProceedThatWonAClaimHoldsItsClaimToken() { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); + + // when + GrpcIdempotencyDecision takenOver = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL); + + // then + assertThat(takenOver.claimToken()).contains(GrpcClaimToken.FIRST.next()); + } + + /** + * The re-review probe's sequence through the interceptor (2026-11-03, RR-02-04): A is slow, B + * takes over after the TTL, A then fails. A's failure must not free the key for C while B runs. + */ + @Test + @DisplayName("TTL 뒤 넘겨받힌 옛 owner 의 실패 기록은 무시되어 다음 재시도는 진행 중으로 거부된다") + void aFailureRecordedByTheOwnerThatLostTheClaimDoesNotLetTheNextRetryRun() { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + GrpcClaimToken first = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL); + boolean recorded = + interceptor.recordFailure(CREATE, CALLER, KEY, first, AFTER_TTL.plusSeconds(1)); + + // when + GrpcIdempotencyDecision third = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL.plusSeconds(2)); + + // then + assertThat(recorded).isFalse(); + assertThat(third.outcome()).isEqualTo(GrpcIdempotencyDecision.Outcome.REJECT_IN_PROGRESS); + } + + @Test + @DisplayName("TTL 뒤 넘겨받힌 옛 owner 의 commit 은 거부되어 그 트랜잭션이 롤백된다") + void aCommitRecordedByTheOwnerThatLostTheClaimIsRefused() { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + GrpcClaimToken first = + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW).claimToken().orElseThrow(); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, AFTER_TTL); + + // when + Throwable refused = + org.assertj.core.api.Assertions.catchThrowable( + () -> + interceptor.recordCommit( + CREATE, CALLER, KEY, first, "outcome://document/42", AFTER_TTL.plusSeconds(1))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("no longer held"); + } + @Test @DisplayName("TTL이 지나기 직전의 중복 요청은 여전히 대기하거나 거부된다") void aDuplicateAtTheTtlBoundaryIsStillTreatedAsRunning() { diff --git a/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetentionTest.java b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetentionTest.java index fca364a..817514a 100644 --- a/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetentionTest.java +++ b/grpc/grpc-runtime/src/test/java/dev/caskeleton/grpc/idempotency/GrpcOperationLedgerRetentionTest.java @@ -3,6 +3,7 @@ package dev.caskeleton.grpc.idempotency; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger; import dev.caskeleton.grpc.core.GrpcMethodName; @@ -78,6 +79,7 @@ class GrpcOperationLedgerRetentionTest { // Finished at the claim moment; the completion moment is what the retention measures. private void committed(GrpcOperationIdentity identity, Instant claimedAt) { ledger.claim(identity, BODY, claimedAt); - ledger.markCommitted(identity, "outcome://document/" + identity.storageKey(), claimedAt); + ledger.markCommitted( + identity, GrpcClaimToken.FIRST, "outcome://document/" + identity.storageKey(), claimedAt); } } diff --git a/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidator.java b/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidator.java index a289278..85a3ed8 100644 --- a/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidator.java +++ b/grpc/grpc-spring-boot-starter/src/main/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidator.java @@ -24,6 +24,11 @@ import java.util.Set; * reflection in production publishes the schema, and a keyed method without a ledger accepts * idempotency keys it cannot honour. None of them fails a smoke test. * + *

The keyed-method rule has a limit it states in its own message: with the ledger flag on, the + * starter assembles the decision function, and the server stage that calls it is the adopter's. A + * running process cannot see whether that stage exists, so passing this check with the flag on does + * not mean keys are honoured (re-review 2026-11-03, RR-02-07; ADR-GRPC-004). + * *

Fails once with every violation, so a deployment learns the whole list in one restart. */ public final class GrpcPlatformStartupValidator { @@ -152,7 +157,10 @@ public final class GrpcPlatformStartupValidator { "method '" + method.canonical() + "' requires an idempotency key but app.grpc.platform.operation-ledger-enabled is" - + " false; the key would be accepted and never honoured"); + + " false; the key would be accepted and never honoured. Turning the flag on" + + " assembles the decision function only: the key is honoured once the server's" + + " IDEMPOTENCY stage interceptor or the service adapter calls decide/recordCommit," + + " which this starter does not wire"); } if (!policy.rpcType().stable()) { violations.add( diff --git a/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfigurationContextTest.java b/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfigurationContextTest.java index a07b65b..b3fcbd2 100644 --- a/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfigurationContextTest.java +++ b/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformAutoConfigurationContextTest.java @@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.grpc.api.deadline.GrpcDeadlineProfile; +import dev.caskeleton.grpc.api.ledger.GrpcClaimToken; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger; @@ -196,7 +197,8 @@ class GrpcPlatformAutoConfigurationContextTest { GrpcOperationIdentity identity = new GrpcOperationIdentity(CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY)); ledger.claim(identity, GrpcRequestFingerprint.of("body"), NOW); - ledger.markCommitted(identity, "outcome://document/42", NOW.plusSeconds(1)); + ledger.markCommitted( + identity, GrpcClaimToken.FIRST, "outcome://document/42", NOW.plusSeconds(1)); // when int purged = diff --git a/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidatorTest.java b/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidatorTest.java index dcffee9..d7307b1 100644 --- a/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidatorTest.java +++ b/grpc/grpc-spring-boot-starter/src/test/java/dev/caskeleton/grpc/boot/GrpcPlatformStartupValidatorTest.java @@ -200,6 +200,44 @@ class GrpcPlatformStartupValidatorTest { .anySatisfy(violation -> assertThat(violation).contains("never honoured")); } + /** + * The flag assembles the decision function only; the starter builds no server, so nothing calls + * it until the adopter's IDEMPOTENCY stage or service adapter does. The refusal must not read as + * "turn the flag on and keys are honoured" (re-review 2026-11-03, RR-02-07). + */ + @Test + @DisplayName("키 필수 메서드 거부 문구는 플래그가 결정 함수만 조립하고 서버 단계 연결은 따로라고 말한다") + void theKeyedMethodRefusalSaysTheFlagAssemblesTheDecisionOnly() { + // given + GrpcPlatformProperties properties = productionProperties(); + properties.setOperationLedgerEnabled(false); + GrpcMethodPolicyCatalog catalog = + GrpcMethodPolicyCatalog.builder() + .register( + new GrpcMethodPolicy( + CREATE, + RpcType.UNARY, + RpcIdempotencyProfile.IDEMPOTENCY_KEY_REQUIRED, + GrpcDeadlineProfile.of(Duration.ofSeconds(2)), + WaitForReadyPolicy.DISABLED, + false, + 1024, + 1024)) + .build(); + + // when + var violations = + GrpcPlatformStartupValidator.violations(properties, catalog, oneChannel(), stableServer()); + + // then + assertThat(violations) + .anySatisfy( + violation -> + assertThat(violation) + .contains("assembles the decision function only") + .contains("IDEMPOTENCY stage")); + } + @Test @DisplayName("a client-streaming method is refused as an Advanced capability") void clientStreamingIsRefusedAsAdvanced() { diff --git a/grpc/grpc-testkit/build.gradle b/grpc/grpc-testkit/build.gradle index 3387707..7dad319 100644 --- a/grpc/grpc-testkit/build.gradle +++ b/grpc/grpc-testkit/build.gradle @@ -5,8 +5,9 @@ plugins { // Certification. The Stable plan splits this across four modules (core / in-process / netty / // fault); this repository already expresses "these two runs are not the same kind of evidence" with // strict test lanes rather than with module boundaries, so the four become four lanes over one -// leaf (adaptation design §2). A lane that discovers nothing fails, and none of them can serve an -// up-to-date result — which is the property the split was protecting. +// leaf (adaptation design §2). A lane that discovers nothing fails, and the three hermetic lanes +// reuse a result only when their classes and their lane policy are unchanged — the performance lane +// never does. That is the property the split was protecting. strictTestLanes { lane('grpcInProcessContractTest') { @@ -33,8 +34,8 @@ strictTestLanes { } // The performance lane is excluded from the default `test` task. It measures a running server -// under load, and a measurement in the release gate is a flaky test on a shared CI runner; it runs -// when somebody asks for it, by name. +// under load, so it is not in the pull-request or release gate; it runs in the optional build's +// `performanceCheck`, which integration-main.yml's performance-check job runs on main and nightly. tasks.named('test') { useJUnitPlatform { excludeTags 'grpc-performance'