diff --git a/grpc/grpc-core-api/build.gradle b/grpc/grpc-core-api/build.gradle index ddde61d..84fc01a 100644 --- a/grpc/grpc-core-api/build.gradle +++ b/grpc/grpc-core-api/build.gradle @@ -1,5 +1,6 @@ plugins { id 'ca.platform-module' + id 'java-test-fixtures' } // The platform's port layer: identifiers, method policy, execution evidence, failure model, @@ -10,4 +11,8 @@ plugins { // a transport" is verifiable instead of aspirational — the same rule `messaging-core-api` holds. // A type here may not name io.grpc, Spring, Netty, protobuf or a database. dependencies { + // Test fixtures only, never main: the operation-ledger fake and the contract that both it and + // the JPA adapter must pass, so the fake cannot allow a transition the database refuses. + testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api' + testFixturesImplementation 'org.assertj:assertj-core' } diff --git a/grpc/grpc-core-api/gradle.lockfile b/grpc/grpc-core-api/gradle.lockfile index e2c9585..13a5bab 100644 --- a/grpc/grpc-core-api/gradle.lockfile +++ b/grpc/grpc-core-api/gradle.lockfile @@ -1,40 +1,40 @@ # This is a Gradle generated file for dependency locking. # Manual edits can break the build and are not advised. # This file is expected to be part of source control. -com.github.ben-manes.caffeine:caffeine:3.2.4=annotationProcessor,testAnnotationProcessor -com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.ben-manes.caffeine:caffeine:3.2.4=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs com.github.spotbugs:spotbugs:4.10.2=spotbugs com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs -com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor -com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor -com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs com.google.code.gson:gson:2.13.2=spotbugs -com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor com.google.errorprone:error_prone_annotations:2.41.0=spotbugs com.google.errorprone:error_prone_annotations:2.47.0=checkstyle -com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor -com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor -com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor -com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor -com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor -com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor com.google.guava:guava:33.6.0-jre=checkstyle -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor -com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor -com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins com.puppycrawl.tools:checkstyle:13.5.0=checkstyle commons-beanutils:commons-beanutils:1.11.0=checkstyle commons-collections:commons-collections:3.2.2=checkstyle commons-io:commons-io:2.21.0=spotbugs info.picocli:picocli:4.7.7=checkstyle -io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor -io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor -javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor jaxen:jaxen:2.0.6=spotbugs -net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs org.antlr:antlr4-runtime:4.13.2=checkstyle org.apache.bcel:bcel:6.12.0=spotbugs @@ -50,32 +50,32 @@ org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle org.apache.xbean:xbean-reflect:3.7=checkstyle -org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath -org.assertj:assertj-core:3.27.7=testCompileClasspath,testRuntimeClasspath +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testFixturesCompileClasspath +org.assertj:assertj-core:3.27.7=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle org.codehaus.plexus:plexus-utils:3.3.0=checkstyle org.dom4j:dom4j:2.2.0=spotbugs org.javassist:javassist:3.28.0-GA=checkstyle -org.jspecify:jspecify:1.0.1=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath -org.junit.jupiter:junit-jupiter-api:6.0.3=testCompileClasspath,testRuntimeClasspath +org.jspecify:jspecify:1.0.1=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath,testFixturesAnnotationProcessor,testFixturesCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.3=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath org.junit.jupiter:junit-jupiter-engine:6.0.3=testRuntimeClasspath org.junit.jupiter:junit-jupiter-params:6.0.3=testCompileClasspath,testRuntimeClasspath org.junit.jupiter:junit-jupiter:6.0.3=testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-commons:6.0.3=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.3=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath org.junit.platform:junit-platform-engine:6.0.3=testRuntimeClasspath org.junit.platform:junit-platform-launcher:6.0.3=testRuntimeClasspath -org.junit:junit-bom:6.0.3=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.0.3=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath org.junit:junit-bom:6.1.0=spotbugs org.mockito:mockito-core:5.20.0=mockitoAgent -org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath org.ow2.asm:asm-analysis:9.10.1=spotbugs org.ow2.asm:asm-commons:9.10.1=spotbugs org.ow2.asm:asm-tree:9.10.1=spotbugs org.ow2.asm:asm-util:9.10.1=spotbugs org.ow2.asm:asm:9.10.1=spotbugs -org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor org.reflections:reflections:0.10.2=checkstyle org.slf4j:slf4j-api:2.0.18=spotbugs,spotbugsSlf4j org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java index 9ab94f6..4342bde 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java @@ -40,6 +40,18 @@ public interface GrpcOperationLedger { /** Marks a claim terminally failed. */ void markFailed(GrpcOperationIdentity identity, Instant now); + /** + * Re-claims a {@code FAILED_TERMINAL} record for a new attempt of the same request. + * + *

A compare-and-set to {@code IN_PROGRESS}, atomic like {@link #claim}: of two concurrent + * re-claims exactly one returns true. A record in any other state, or made for a different + * request, is left untouched. A retry that proceeds without winning this leaves its commit with + * no owner and runs concurrently with every other retry of the same key. + * + * @return true when this call now owns the record as a fresh attempt claimed at {@code now} + */ + boolean reclaimFailed(GrpcOperationIdentity identity, String requestFingerprint, Instant now); + /** * Releases a claim that will never complete, so the caller may try again. * diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerState.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerState.java index 2908b57..2c2cfaa 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerState.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerState.java @@ -12,7 +12,10 @@ public enum GrpcOperationLedgerState { IN_PROGRESS(false), /** Committed, with a replayable outcome. */ COMMITTED(true), - /** Failed in a way that will not succeed on repetition. */ + /** + * The attempt failed and changed nothing. A retry of the same request may reclaim it (one winner) + * and run again; see {@code GrpcOperationLedger.reclaimFailed}. + */ FAILED_TERMINAL(true); private final boolean terminal; diff --git a/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedgerContractTest.java b/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedgerContractTest.java new file mode 100644 index 0000000..ff2916f --- /dev/null +++ b/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedgerContractTest.java @@ -0,0 +1,11 @@ +package dev.caskeleton.grpc.api.ledger; + +class InMemoryOperationLedgerContractTest extends GrpcOperationLedgerContract { + + private final InMemoryOperationLedger ledger = new InMemoryOperationLedger(); + + @Override + protected GrpcOperationLedger ledger() { + return ledger; + } +} diff --git a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java new file mode 100644 index 0000000..ebc1110 --- /dev/null +++ b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java @@ -0,0 +1,260 @@ +package dev.caskeleton.grpc.api.ledger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.grpc.core.GrpcMethodName; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.junit.jupiter.api.Test; + +/** + * The transitions every {@link GrpcOperationLedger} allows and refuses. + * + *

The in-memory fake and the JPA adapter (against real PostgreSQL) both extend this. A fake that + * allows a transition the database refuses lets every policy test pass for code that fails in + * production — which is how a retry of a {@code FAILED_TERMINAL} row once proceeded unclaimed. + */ +public abstract class GrpcOperationLedgerContract { + + private static final GrpcOperationIdentity ID = + new GrpcOperationIdentity( + "tenant-1.actor-1", + GrpcMethodName.parse("hyeonworks.document.v1.DocumentService/CreateDocument"), + "sha256:aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899"); + private static final String FINGERPRINT = + "sha256:0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0"; + private static final String OTHER_FINGERPRINT = + "sha256:1122334455667788990011223344556677889900112233445566778899001122"; + private static final String OUTCOME = "outcome://document/42"; + private static final Instant NOW = Instant.parse("2026-09-25T10:00:00Z"); + + /** The ledger under test, holding no records when a test starts. */ + protected abstract GrpcOperationLedger ledger(); + + @Test + void theFirstClaimOfAnOperationIsOwnedByItsCaller() { + assertThat(ledger().claim(ID, FINGERPRINT, NOW)).isEmpty(); + } + + @Test + void aSecondClaimReadsTheFirstClaimInProgress() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + Optional second = + ledger().claim(ID, OTHER_FINGERPRINT, NOW.plusSeconds(1)); + + // then + assertThat(second) + .get() + .extracting( + GrpcOperationLedgerRecord::state, + GrpcOperationLedgerRecord::requestFingerprint, + GrpcOperationLedgerRecord::claimedAt) + .containsExactly(GrpcOperationLedgerState.IN_PROGRESS, FINGERPRINT, NOW); + } + + @Test + void concurrentClaimsHaveExactlyOneOwner() throws Exception { + // when + List owned = concurrently(() -> ledger().claim(ID, FINGERPRINT, NOW).isEmpty()); + + // then + assertThat(owned).containsExactlyInAnyOrder(true, false); + } + + @Test + void aCommittedClaimKeepsTheOutcomeARetryReplays() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + + // then + assertThat(ledger().find(ID)) + .get() + .extracting( + GrpcOperationLedgerRecord::state, + GrpcOperationLedgerRecord::outcomeReference, + GrpcOperationLedgerRecord::completedAt) + .containsExactly( + GrpcOperationLedgerState.COMMITTED, + Optional.of(OUTCOME), + Optional.of(NOW.plusSeconds(1))); + } + + @Test + void committingWithoutAClaimIsRefused() { + // when + Throwable refused = catchThrowable(() -> ledger().markCommitted(ID, OUTCOME, NOW)); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("no claim exists"); + } + + @Test + void aCommittedClaimCannotBeCommittedAgain() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + + // when + Throwable refused = + catchThrowable( + () -> ledger().markCommitted(ID, "outcome://document/99", NOW.plusSeconds(2))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("cannot transition again"); + } + + @Test + void aFailedClaimIsTerminal() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + ledger().markFailed(ID, NOW.plusSeconds(1)); + + // then + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::state, GrpcOperationLedgerRecord::completedAt) + .containsExactly(GrpcOperationLedgerState.FAILED_TERMINAL, Optional.of(NOW.plusSeconds(1))); + } + + @Test + void aFailedClaimCannotBeCommittedWithoutBeingReclaimed() { + // given + failedClaim(); + + // when + Throwable refused = + catchThrowable(() -> ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(2))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("cannot transition again"); + } + + @Test + void aFailedClaimIsReclaimedByARetryOfTheSameRequest() { + // given + failedClaim(); + + // when + boolean reclaimed = ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + + // then + assertThat(reclaimed).isTrue(); + } + + @Test + void aReclaimedClaimIsANewAttemptInProgressSinceTheReclaim() { + // given + failedClaim(); + + // when + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + + // then + assertThat(ledger().find(ID)) + .get() + .extracting( + GrpcOperationLedgerRecord::state, + GrpcOperationLedgerRecord::claimedAt, + GrpcOperationLedgerRecord::completedAt) + .containsExactly( + GrpcOperationLedgerState.IN_PROGRESS, NOW.plusSeconds(2), Optional.empty()); + } + + @Test + void aReclaimedClaimCanBeCommitted() { + // given + failedClaim(); + ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + + // when + ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(3)); + + // then + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::state, GrpcOperationLedgerRecord::outcomeReference) + .containsExactly(GrpcOperationLedgerState.COMMITTED, Optional.of(OUTCOME)); + } + + @Test + void aFailedClaimIsNotReclaimedForADifferentRequest() { + failedClaim(); + + assertThat(ledger().reclaimFailed(ID, OTHER_FINGERPRINT, NOW.plusSeconds(2))).isFalse(); + } + + @Test + void anInProgressClaimIsNotReclaimed() { + ledger().claim(ID, FINGERPRINT, NOW); + + assertThat(ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(1))).isFalse(); + } + + @Test + void aCommittedClaimIsNotReclaimed() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1)); + + // when + boolean reclaimed = ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)); + + // then + assertThat(reclaimed).isFalse(); + } + + @Test + void concurrentReclaimsOfAFailedClaimHaveExactlyOneWinner() throws Exception { + // given + failedClaim(); + + // when + List reclaimed = + concurrently(() -> ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2))); + + // then + assertThat(reclaimed).containsExactlyInAnyOrder(true, false); + } + + private void failedClaim() { + ledger().claim(ID, FINGERPRINT, NOW); + ledger().markFailed(ID, NOW.plusSeconds(1)); + } + + /** Runs {@code attempt} on two threads released together and returns both results. */ + private static List concurrently(Callable attempt) throws Exception { + CountDownLatch start = new CountDownLatch(1); + Callable gated = + () -> { + start.await(); + return attempt.call(); + }; + try (ExecutorService executor = Executors.newFixedThreadPool(2)) { + Future first = executor.submit(gated); + Future second = executor.submit(gated); + start.countDown(); + return List.of(first.get(), second.get()); + } + } +} diff --git a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java new file mode 100644 index 0000000..f08ddac --- /dev/null +++ b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java @@ -0,0 +1,111 @@ +package dev.caskeleton.grpc.api.ledger; + +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * A hand-rolled ledger for tests of code that depends on {@link GrpcOperationLedger}. + * + *

{@code putIfAbsent} on a concurrent map is the in-memory equivalent of the unique constraint + * the real adapter relies on, and {@code replace(key, expected, next)} of its conditional update, + * so claim and re-claim are atomic here for the same reason they are there. The transitions match + * the JPA adapter's because {@link GrpcOperationLedgerContract} runs against both; a fake that + * allowed more would let the policy tests pass for code the database refuses. + */ +public final class InMemoryOperationLedger implements GrpcOperationLedger { + + private final ConcurrentMap records = + new ConcurrentHashMap<>(); + private volatile boolean unavailable; + + /** Makes every read fail, so the UNKNOWN path can be exercised. */ + public void makeUnavailable() { + unavailable = true; + } + + @Override + public Optional claim( + GrpcOperationIdentity identity, String requestFingerprint, Instant now) { + GrpcOperationLedgerRecord existing = + records.putIfAbsent( + identity.storageKey(), + GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now)); + return Optional.ofNullable(existing); + } + + @Override + public Optional find(GrpcOperationIdentity identity) { + if (unavailable) { + throw new IllegalStateException("ledger unavailable"); + } + return Optional.ofNullable(records.get(identity.storageKey())); + } + + @Override + public void markCommitted(GrpcOperationIdentity identity, String outcomeReference, Instant now) { + GrpcOperationLedgerRecord committed = + records.computeIfPresent( + identity.storageKey(), + (key, existing) -> + finish( + existing, + GrpcOperationLedgerState.COMMITTED, + Optional.of(outcomeReference), + now)); + if (committed == null) { + throw new IllegalStateException( + "cannot commit operation '" + identity.storageKey() + "': no claim exists"); + } + } + + @Override + public void markFailed(GrpcOperationIdentity identity, Instant now) { + records.computeIfPresent( + identity.storageKey(), + (key, existing) -> + finish(existing, GrpcOperationLedgerState.FAILED_TERMINAL, Optional.empty(), now)); + } + + @Override + public boolean reclaimFailed( + GrpcOperationIdentity identity, String requestFingerprint, Instant now) { + GrpcOperationLedgerRecord current = records.get(identity.storageKey()); + return current != null + && current.state() == GrpcOperationLedgerState.FAILED_TERMINAL + && current.sameRequestAs(requestFingerprint) + && records.replace( + identity.storageKey(), + current, + GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now)); + } + + @Override + public void release(GrpcOperationIdentity identity) { + records.remove(identity.storageKey()); + } + + // Only an IN_PROGRESS claim finishes; a terminal row is re-opened by reclaimFailed alone. + private static GrpcOperationLedgerRecord finish( + GrpcOperationLedgerRecord existing, + GrpcOperationLedgerState state, + Optional outcomeReference, + Instant now) { + if (existing.state() != GrpcOperationLedgerState.IN_PROGRESS) { + throw new IllegalStateException( + "operation '" + + existing.identity().storageKey() + + "' is already " + + existing.state() + + " and cannot transition again"); + } + return new GrpcOperationLedgerRecord( + existing.identity(), + state, + existing.requestFingerprint(), + outcomeReference, + existing.claimedAt(), + Optional.of(now)); + } +} diff --git a/grpc/grpc-operation-ledger-jpa/build.gradle b/grpc/grpc-operation-ledger-jpa/build.gradle index a4f9708..d979699 100644 --- a/grpc/grpc-operation-ledger-jpa/build.gradle +++ b/grpc/grpc-operation-ledger-jpa/build.gradle @@ -26,6 +26,10 @@ dependencies { integrationTestImplementation 'org.testcontainers:testcontainers-postgresql' integrationTestImplementation 'org.testcontainers:testcontainers-junit-jupiter' integrationTestImplementation 'org.postgresql:postgresql' + + // The shared ledger contract, run against this adapter through a real JPA provider. + integrationTestImplementation testFixtures(project(':grpc:grpc-core-api')) + integrationTestRuntimeOnly 'org.hibernate.orm:hibernate-core' } strictTestLanes { diff --git a/grpc/grpc-operation-ledger-jpa/gradle.lockfile b/grpc/grpc-operation-ledger-jpa/gradle.lockfile index 3647258..d9b0741 100644 --- a/grpc/grpc-operation-ledger-jpa/gradle.lockfile +++ b/grpc/grpc-operation-ledger-jpa/gradle.lockfile @@ -2,6 +2,7 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.fasterxml.jackson.core:jackson-annotations:2.21=integrationTestCompileClasspath,integrationTestRuntimeClasspath +com.fasterxml:classmate:1.7.3=integrationTestRuntimeClasspath com.github.ben-manes.caffeine:caffeine:3.2.4=annotationProcessor,integrationTestAnnotationProcessor,testAnnotationProcessor com.github.docker-java:docker-java-api:3.7.1=integrationTestCompileClasspath,integrationTestRuntimeClasspath com.github.docker-java:docker-java-transport-zerodep:3.7.1=integrationTestCompileClasspath,integrationTestRuntimeClasspath @@ -30,6 +31,7 @@ com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,integrat com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,integrationTestAnnotationProcessor,testAnnotationProcessor com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins com.puppycrawl.tools:checkstyle:13.5.0=checkstyle +com.sun.istack:istack-commons-runtime:4.1.2=integrationTestRuntimeClasspath commons-beanutils:commons-beanutils:1.11.0=checkstyle commons-codec:commons-codec:1.19.0=integrationTestCompileClasspath,integrationTestRuntimeClasspath commons-collections:commons-collections:3.2.2=checkstyle @@ -41,8 +43,12 @@ io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,integratio io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,integrationTestAnnotationProcessor,testAnnotationProcessor io.micrometer:micrometer-commons:1.16.7=compileClasspath,integrationTestCompileClasspath,integrationTestRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath io.micrometer:micrometer-observation:1.16.7=compileClasspath,integrationTestCompileClasspath,integrationTestRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +jakarta.activation:jakarta.activation-api:2.1.4=integrationTestRuntimeClasspath jakarta.annotation:jakarta.annotation-api:3.0.0=compileClasspath,integrationTestCompileClasspath,integrationTestRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +jakarta.inject:jakarta.inject-api:2.0.1=integrationTestRuntimeClasspath jakarta.persistence:jakarta.persistence-api:3.2.0=compileClasspath,integrationTestCompileClasspath,integrationTestRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +jakarta.transaction:jakarta.transaction-api:2.0.1=integrationTestRuntimeClasspath +jakarta.xml.bind:jakarta.xml.bind-api:4.0.5=integrationTestRuntimeClasspath javax.inject:javax.inject:1=annotationProcessor,integrationTestAnnotationProcessor,testAnnotationProcessor jaxen:jaxen:2.0.6=spotbugs net.bytebuddy:byte-buddy:1.17.8=integrationTestCompileClasspath,integrationTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath @@ -71,7 +77,14 @@ org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle org.codehaus.plexus:plexus-utils:3.3.0=checkstyle org.dom4j:dom4j:2.2.0=spotbugs +org.eclipse.angus:angus-activation:2.0.3=integrationTestRuntimeClasspath +org.glassfish.jaxb:jaxb-core:4.0.9=integrationTestRuntimeClasspath +org.glassfish.jaxb:jaxb-runtime:4.0.9=integrationTestRuntimeClasspath +org.glassfish.jaxb:txw2:4.0.9=integrationTestRuntimeClasspath +org.hibernate.models:hibernate-models:1.0.1=integrationTestRuntimeClasspath +org.hibernate.orm:hibernate-core:7.2.24.Final=integrationTestRuntimeClasspath org.javassist:javassist:3.28.0-GA=checkstyle +org.jboss.logging:jboss-logging:3.6.3.Final=integrationTestRuntimeClasspath org.jetbrains:annotations:17.0.0=integrationTestCompileClasspath,integrationTestRuntimeClasspath org.jspecify:jspecify:1.0.1=annotationProcessor,checkstyle,compileClasspath,integrationTestAnnotationProcessor,integrationTestCompileClasspath,integrationTestRuntimeClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath org.junit.jupiter:junit-jupiter-api:6.0.3=integrationTestCompileClasspath,integrationTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java new file mode 100644 index 0000000..0a17279 --- /dev/null +++ b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java @@ -0,0 +1,185 @@ +package dev.caskeleton.grpc.ledger; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; +import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; +import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerContract; +import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; +import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerState; +import dev.caskeleton.grpc.core.GrpcMethodName; +import jakarta.persistence.EntityManagerFactory; +import jakarta.persistence.PersistenceConfiguration; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Instant; +import java.util.Optional; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.data.jpa.repository.support.JpaRepositoryFactory; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.SharedEntityManagerCreator; +import org.springframework.transaction.support.TransactionTemplate; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.postgresql.PostgreSQLContainer; + +/** + * Runs the ledger contract against {@link JpaGrpcOperationLedger} on Hibernate and PostgreSQL. + * + *

Every ledger call runs in its own transaction, standing in for the caller's business + * transaction; the two threads of a concurrency test therefore race as two real transactions. + */ +@Testcontainers(disabledWithoutDocker = true) +class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract { + + @SuppressWarnings("resource") + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer("postgres:16-alpine") + .withDatabaseName("grpc") + .withUsername("grpc") + .withPassword("grpc"); + + private static EntityManagerFactory entityManagerFactory; + private static TransactionTemplate transactions; + private static JpaGrpcOperationLedger adapter; + private static GrpcOperationLedger ledger; + + @BeforeAll + static void startJpa() throws SQLException, IOException { + execute(migration()); + entityManagerFactory = + new PersistenceConfiguration("grpc-operation-ledger") + .managedClass(GrpcOperationLedgerEntity.class) + .property(PersistenceConfiguration.JDBC_URL, POSTGRES.getJdbcUrl()) + .property(PersistenceConfiguration.JDBC_USER, POSTGRES.getUsername()) + .property(PersistenceConfiguration.JDBC_PASSWORD, POSTGRES.getPassword()) + .createEntityManagerFactory(); + GrpcOperationLedgerRepository repository = + new JpaRepositoryFactory( + SharedEntityManagerCreator.createSharedEntityManager(entityManagerFactory)) + .getRepository(GrpcOperationLedgerRepository.class); + transactions = new TransactionTemplate(new JpaTransactionManager(entityManagerFactory)); + adapter = new JpaGrpcOperationLedger(repository); + ledger = new PerCallTransactionLedger(adapter, transactions); + } + + @AfterAll + static void stopJpa() { + entityManagerFactory.close(); + } + + @BeforeEach + void emptyTheLedger() throws SQLException { + execute("TRUNCATE grpc_operation_ledger"); + } + + @Override + protected GrpcOperationLedger ledger() { + return ledger; + } + + /** + * Adopters are told to claim and commit inside the business transaction. The claim loads the + * failed row into the persistence context, so without the re-claim clearing it the commit would + * read the stale FAILED_TERMINAL copy and refuse. + */ + @Test + void aRetryThatReclaimsAndCommitsInOneTransactionCommits() { + // given + GrpcOperationIdentity id = + new GrpcOperationIdentity( + "tenant-1.actor-1", + GrpcMethodName.parse("hyeonworks.document.v1.DocumentService/CreateDocument"), + "sha256:aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899"); + String fingerprint = "sha256:0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0"; + Instant now = Instant.parse("2026-09-25T10:00:00Z"); + ledger.claim(id, fingerprint, now); + ledger.markFailed(id, now.plusSeconds(1)); + + // when + transactions.executeWithoutResult( + status -> { + adapter.claim(id, fingerprint, now.plusSeconds(2)); + adapter.reclaimFailed(id, fingerprint, now.plusSeconds(2)); + adapter.markCommitted(id, "outcome://document/42", now.plusSeconds(3)); + }); + + // then + assertThat(ledger.find(id)) + .get() + .extracting(GrpcOperationLedgerRecord::state, GrpcOperationLedgerRecord::outcomeReference) + .containsExactly(GrpcOperationLedgerState.COMMITTED, Optional.of("outcome://document/42")); + } + + private static void execute(String sql) throws SQLException { + try (Connection connection = + DriverManager.getConnection( + POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()); + Statement statement = connection.createStatement()) { + statement.execute(sql); + } + } + + private static String migration() throws IOException { + try (InputStream stream = + JpaGrpcOperationLedgerContractIT.class + .getClassLoader() + .getResourceAsStream("db/migration/grpc/V001__create_grpc_operation_ledger.sql")) { + if (stream == null) { + throw new IOException("gRPC operation-ledger migration is missing"); + } + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + } + + /** Runs each call in its own transaction, as a caller's business transaction would. */ + private record PerCallTransactionLedger( + JpaGrpcOperationLedger delegate, TransactionTemplate transactions) + implements GrpcOperationLedger { + + @Override + public Optional claim( + GrpcOperationIdentity identity, String requestFingerprint, Instant now) { + return transactions.execute(status -> delegate.claim(identity, requestFingerprint, now)); + } + + @Override + public Optional find(GrpcOperationIdentity identity) { + return transactions.execute(status -> delegate.find(identity)); + } + + @Override + public void markCommitted( + GrpcOperationIdentity identity, String outcomeReference, Instant now) { + transactions.executeWithoutResult( + status -> delegate.markCommitted(identity, outcomeReference, now)); + } + + @Override + public void markFailed(GrpcOperationIdentity identity, Instant now) { + transactions.executeWithoutResult(status -> delegate.markFailed(identity, now)); + } + + @Override + public boolean reclaimFailed( + GrpcOperationIdentity identity, String requestFingerprint, Instant now) { + return Boolean.TRUE.equals( + transactions.execute( + status -> delegate.reclaimFailed(identity, requestFingerprint, now))); + } + + @Override + public void release(GrpcOperationIdentity identity) { + transactions.executeWithoutResult(status -> delegate.release(identity)); + } + } +} diff --git a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java index 5f72d6d..f1823b8 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java +++ b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java @@ -62,6 +62,36 @@ public interface GrpcOperationLedgerRepository @Param("requestFingerprint") String requestFingerprint, @Param("claimedAt") Instant claimedAt); + /** + * Re-claims a terminally failed row as a new attempt of the same request. + * + *

One conditional update, so PostgreSQL's row lock decides the race: a concurrent second + * re-claim re-evaluates the condition against the winner's {@code IN_PROGRESS} row and updates + * nothing. It clears the persistence context, because a row this transaction already loaded (the + * claim's read-on-conflict does) would otherwise still read {@code FAILED_TERMINAL} and refuse + * the commit. + * + * @return one when this caller re-claimed the row, zero otherwise + */ + @Modifying(flushAutomatically = true, clearAutomatically = true) + @Transactional + @Query( + value = + """ + UPDATE grpc_operation_ledger + SET state = 'IN_PROGRESS', + claimed_at = :claimedAt, + completed_at = NULL + WHERE storage_key = :storageKey + AND state = 'FAILED_TERMINAL' + AND request_fingerprint = :requestFingerprint + """, + nativeQuery = true) + int tryReclaimFailed( + @Param("storageKey") String storageKey, + @Param("requestFingerprint") String requestFingerprint, + @Param("claimedAt") Instant claimedAt); + /** Updates an existing claim after it has been loaded. */ GrpcOperationLedgerEntity save(GrpcOperationLedgerEntity entity); diff --git a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java index 7c03805..443847a 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java +++ b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java @@ -88,6 +88,13 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { }); } + @Override + public boolean reclaimFailed( + GrpcOperationIdentity identity, String requestFingerprint, Instant now) { + requireIdentity(identity); + return repository.tryReclaimFailed(identity.storageKey(), requestFingerprint, now) == 1; + } + @Override public void release(GrpcOperationIdentity identity) { requireIdentity(identity); diff --git a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java index fba811e..e1457b5 100644 --- a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java +++ b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java @@ -59,6 +59,12 @@ class GrpcOperationLedgerRepositoryTest { return rows.putIfAbsent(storageKey, claim) == null ? 1 : 0; } + @Override + public int tryReclaimFailed(String storageKey, String requestFingerprint, Instant claimedAt) { + throw new UnsupportedOperationException( + "the re-claim is certified against PostgreSQL by JpaGrpcOperationLedgerContractIT"); + } + @Override public GrpcOperationLedgerEntity save(GrpcOperationLedgerEntity entity) { rows.put(entity.getStorageKey(), entity); diff --git a/grpc/grpc-policy/build.gradle b/grpc/grpc-policy/build.gradle index 17a4f5b..6d3694e 100644 --- a/grpc/grpc-policy/build.gradle +++ b/grpc/grpc-policy/build.gradle @@ -13,4 +13,7 @@ dependencies { // signatures, so an adopter cannot compile against this module without them. api "io.grpc:grpc-api" api "io.grpc:grpc-stub" + + // The shared operation-ledger fake, held to the same contract as the JPA adapter. + testImplementation testFixtures(project(':grpc:grpc-core-api')) } diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java index 7307ed4..e232669 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java @@ -106,12 +106,16 @@ public final class GrpcIdempotencyInterceptor { + identity.storageKey() + "' has no outcome reference to replay"))); } - if (record.state() == GrpcOperationLedgerState.FAILED_TERMINAL) { + if (record.state() == GrpcOperationLedgerState.FAILED_TERMINAL + && ledger.reclaimFailed(identity, requestFingerprint, now)) { // A terminal failure is not replayed: the caller asked for the operation, it did not happen, - // and repeating it is exactly what the key makes safe. + // and repeating it is exactly what the key makes safe — but only for the one retry that + // re-claims the row. Proceeding unclaimed runs concurrent retries twice and leaves the + // commit without an owner. return GrpcIdempotencyDecision.proceed( - "the previous attempt failed terminally; the key makes repeating it safe"); + "the previous attempt failed terminally; this caller re-claimed it for a new attempt"); } + // Running, or a failed attempt another retry re-claimed first. return waitForInProgress ? GrpcIdempotencyDecision.waitAndPoll(inProgressPollInterval) : GrpcIdempotencyDecision.rejectInProgress(); diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java index afb8e98..7bffcbc 100644 --- a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java +++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.grpc.idempotency; import static org.assertj.core.api.Assertions.assertThat; +import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger; import dev.caskeleton.grpc.core.GrpcMethodName; import java.time.Instant; import java.util.Optional; diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java index 51b8dea..7a09d60 100644 --- a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java +++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java @@ -4,6 +4,10 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import dev.caskeleton.grpc.api.deadline.GrpcDeadlineProfile; +import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; +import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; +import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerState; +import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger; import dev.caskeleton.grpc.api.policy.GrpcMethodPolicy; import dev.caskeleton.grpc.api.policy.GrpcMethodPolicyCatalog; import dev.caskeleton.grpc.api.policy.RpcIdempotencyProfile; @@ -12,6 +16,12 @@ import dev.caskeleton.grpc.core.GrpcMethodName; import dev.caskeleton.grpc.core.RpcType; import java.time.Duration; import java.time.Instant; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -133,6 +143,57 @@ class GrpcIdempotencyInterceptorTest { .isEqualTo(GrpcIdempotencyDecision.Outcome.PROCEED); } + @Test + void aRetryOfATerminallyFailedOperationReclaimsItBeforeProceeding() { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); + interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1)); + + // when + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2)); + + // then + assertThat( + ledger + .find( + new GrpcOperationIdentity( + CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY))) + .map(GrpcOperationLedgerRecord::state)) + .contains(GrpcOperationLedgerState.IN_PROGRESS); + } + + @Test + void concurrentRetriesOfATerminallyFailedOperationProceedOnlyOnce() throws Exception { + // given + GrpcIdempotencyInterceptor interceptor = interceptor(false); + String fingerprint = GrpcRequestFingerprint.of("body"); + interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW); + interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1)); + CountDownLatch start = new CountDownLatch(1); + Callable retry = + () -> { + start.await(); + return interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2)).outcome(); + }; + + // when + List outcomes; + try (ExecutorService executor = Executors.newFixedThreadPool(2)) { + Future first = executor.submit(retry); + Future second = executor.submit(retry); + start.countDown(); + outcomes = List.of(first.get(), second.get()); + } + + // then + assertThat(outcomes) + .containsExactlyInAnyOrder( + GrpcIdempotencyDecision.Outcome.PROCEED, + GrpcIdempotencyDecision.Outcome.REJECT_IN_PROGRESS); + } + @Test @DisplayName("one caller's key does not suppress another caller's write") void oneCallersKeyDoesNotSuppressAnother() { diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/InMemoryOperationLedger.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/InMemoryOperationLedger.java deleted file mode 100644 index 8d79058..0000000 --- a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/InMemoryOperationLedger.java +++ /dev/null @@ -1,81 +0,0 @@ -package dev.caskeleton.grpc.idempotency; - -import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; -import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; -import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord; -import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerState; -import java.time.Instant; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -/** - * A hand-rolled ledger for the policy tests. - * - *

{@code putIfAbsent} on a concurrent map is the in-memory equivalent of the unique constraint - * the real adapter relies on, so the claim is atomic here for the same reason it is there. A fake - * that read and then wrote would let the policy tests pass while the property they exist to check - * does not hold. - */ -final class InMemoryOperationLedger implements GrpcOperationLedger { - - private final ConcurrentMap records = - new ConcurrentHashMap<>(); - private boolean unavailable; - - /** Makes every read fail, so the UNKNOWN path can be exercised. */ - void makeUnavailable() { - unavailable = true; - } - - @Override - public Optional claim( - GrpcOperationIdentity identity, String requestFingerprint, Instant now) { - GrpcOperationLedgerRecord existing = - records.putIfAbsent( - identity.storageKey(), - GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now)); - return Optional.ofNullable(existing); - } - - @Override - public Optional find(GrpcOperationIdentity identity) { - if (unavailable) { - throw new IllegalStateException("ledger unavailable"); - } - return Optional.ofNullable(records.get(identity.storageKey())); - } - - @Override - public void markCommitted(GrpcOperationIdentity identity, String outcomeReference, Instant now) { - records.computeIfPresent( - identity.storageKey(), - (key, existing) -> - new GrpcOperationLedgerRecord( - existing.identity(), - GrpcOperationLedgerState.COMMITTED, - existing.requestFingerprint(), - Optional.of(outcomeReference), - existing.claimedAt(), - Optional.of(now))); - } - - @Override - public void markFailed(GrpcOperationIdentity identity, Instant now) { - records.computeIfPresent( - identity.storageKey(), - (key, existing) -> - new GrpcOperationLedgerRecord( - existing.identity(), - GrpcOperationLedgerState.FAILED_TERMINAL, - existing.requestFingerprint(), - Optional.empty(), - existing.claimedAt(), - Optional.of(now))); - } - - @Override - public void release(GrpcOperationIdentity identity) { - records.remove(identity.storageKey()); - } -}