diff --git a/grpc/grpc-core-api/build.gradle b/grpc/grpc-core-api/build.gradle index ddde61d..84fc01a 100644 --- a/grpc/grpc-core-api/build.gradle +++ b/grpc/grpc-core-api/build.gradle @@ -1,5 +1,6 @@ plugins { id 'ca.platform-module' + id 'java-test-fixtures' } // The platform's port layer: identifiers, method policy, execution evidence, failure model, @@ -10,4 +11,8 @@ plugins { // a transport" is verifiable instead of aspirational — the same rule `messaging-core-api` holds. // A type here may not name io.grpc, Spring, Netty, protobuf or a database. dependencies { + // Test fixtures only, never main: the operation-ledger fake and the contract that both it and + // the JPA adapter must pass, so the fake cannot allow a transition the database refuses. + testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api' + testFixturesImplementation 'org.assertj:assertj-core' } diff --git a/grpc/grpc-core-api/gradle.lockfile b/grpc/grpc-core-api/gradle.lockfile index e2c9585..13a5bab 100644 --- a/grpc/grpc-core-api/gradle.lockfile +++ b/grpc/grpc-core-api/gradle.lockfile @@ -1,40 +1,40 @@ # This is a Gradle generated file for dependency locking. # Manual edits can break the build and are not advised. # This file is expected to be part of source control. -com.github.ben-manes.caffeine:caffeine:3.2.4=annotationProcessor,testAnnotationProcessor -com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor +com.github.ben-manes.caffeine:caffeine:3.2.4=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs com.github.spotbugs:spotbugs:4.10.2=spotbugs com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs -com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor -com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor -com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor +com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs com.google.code.gson:gson:2.13.2=spotbugs -com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor com.google.errorprone:error_prone_annotations:2.41.0=spotbugs com.google.errorprone:error_prone_annotations:2.47.0=checkstyle -com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor -com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor -com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor -com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor -com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor -com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor com.google.guava:guava:33.6.0-jre=checkstyle -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor -com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor -com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor,testFixturesAnnotationProcessor +com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins com.puppycrawl.tools:checkstyle:13.5.0=checkstyle commons-beanutils:commons-beanutils:1.11.0=checkstyle commons-collections:commons-collections:3.2.2=checkstyle commons-io:commons-io:2.21.0=spotbugs info.picocli:picocli:4.7.7=checkstyle -io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor -io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor -javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor +io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor +javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor jaxen:jaxen:2.0.6=spotbugs -net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath +net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs org.antlr:antlr4-runtime:4.13.2=checkstyle org.apache.bcel:bcel:6.12.0=spotbugs @@ -50,32 +50,32 @@ org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle org.apache.xbean:xbean-reflect:3.7=checkstyle -org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath -org.assertj:assertj-core:3.27.7=testCompileClasspath,testRuntimeClasspath +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testFixturesCompileClasspath +org.assertj:assertj-core:3.27.7=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle org.codehaus.plexus:plexus-utils:3.3.0=checkstyle org.dom4j:dom4j:2.2.0=spotbugs org.javassist:javassist:3.28.0-GA=checkstyle -org.jspecify:jspecify:1.0.1=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath -org.junit.jupiter:junit-jupiter-api:6.0.3=testCompileClasspath,testRuntimeClasspath +org.jspecify:jspecify:1.0.1=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath,testFixturesAnnotationProcessor,testFixturesCompileClasspath +org.junit.jupiter:junit-jupiter-api:6.0.3=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath org.junit.jupiter:junit-jupiter-engine:6.0.3=testRuntimeClasspath org.junit.jupiter:junit-jupiter-params:6.0.3=testCompileClasspath,testRuntimeClasspath org.junit.jupiter:junit-jupiter:6.0.3=testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-commons:6.0.3=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:6.0.3=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath org.junit.platform:junit-platform-engine:6.0.3=testRuntimeClasspath org.junit.platform:junit-platform-launcher:6.0.3=testRuntimeClasspath -org.junit:junit-bom:6.0.3=testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.0.3=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath org.junit:junit-bom:6.1.0=spotbugs org.mockito:mockito-core:5.20.0=mockitoAgent -org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath org.ow2.asm:asm-analysis:9.10.1=spotbugs org.ow2.asm:asm-commons:9.10.1=spotbugs org.ow2.asm:asm-tree:9.10.1=spotbugs org.ow2.asm:asm-util:9.10.1=spotbugs org.ow2.asm:asm:9.10.1=spotbugs -org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor +org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor org.reflections:reflections:0.10.2=checkstyle org.slf4j:slf4j-api:2.0.18=spotbugs,spotbugsSlf4j org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java index 9ab94f6..4342bde 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java @@ -40,6 +40,18 @@ public interface GrpcOperationLedger { /** Marks a claim terminally failed. */ void markFailed(GrpcOperationIdentity identity, Instant now); + /** + * Re-claims a {@code FAILED_TERMINAL} record for a new attempt of the same request. + * + *
A compare-and-set to {@code IN_PROGRESS}, atomic like {@link #claim}: of two concurrent + * re-claims exactly one returns true. A record in any other state, or made for a different + * request, is left untouched. A retry that proceeds without winning this leaves its commit with + * no owner and runs concurrently with every other retry of the same key. + * + * @return true when this call now owns the record as a fresh attempt claimed at {@code now} + */ + boolean reclaimFailed(GrpcOperationIdentity identity, String requestFingerprint, Instant now); + /** * Releases a claim that will never complete, so the caller may try again. * diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerState.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerState.java index 2908b57..2c2cfaa 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerState.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerState.java @@ -12,7 +12,10 @@ public enum GrpcOperationLedgerState { IN_PROGRESS(false), /** Committed, with a replayable outcome. */ COMMITTED(true), - /** Failed in a way that will not succeed on repetition. */ + /** + * The attempt failed and changed nothing. A retry of the same request may reclaim it (one winner) + * and run again; see {@code GrpcOperationLedger.reclaimFailed}. + */ FAILED_TERMINAL(true); private final boolean terminal; diff --git a/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedgerContractTest.java b/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedgerContractTest.java new file mode 100644 index 0000000..ff2916f --- /dev/null +++ b/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedgerContractTest.java @@ -0,0 +1,11 @@ +package dev.caskeleton.grpc.api.ledger; + +class InMemoryOperationLedgerContractTest extends GrpcOperationLedgerContract { + + private final InMemoryOperationLedger ledger = new InMemoryOperationLedger(); + + @Override + protected GrpcOperationLedger ledger() { + return ledger; + } +} diff --git a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java new file mode 100644 index 0000000..ebc1110 --- /dev/null +++ b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java @@ -0,0 +1,260 @@ +package dev.caskeleton.grpc.api.ledger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.grpc.core.GrpcMethodName; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.junit.jupiter.api.Test; + +/** + * The transitions every {@link GrpcOperationLedger} allows and refuses. + * + *
The in-memory fake and the JPA adapter (against real PostgreSQL) both extend this. A fake that
+ * allows a transition the database refuses lets every policy test pass for code that fails in
+ * production — which is how a retry of a {@code FAILED_TERMINAL} row once proceeded unclaimed.
+ */
+public abstract class GrpcOperationLedgerContract {
+
+ private static final GrpcOperationIdentity ID =
+ new GrpcOperationIdentity(
+ "tenant-1.actor-1",
+ GrpcMethodName.parse("hyeonworks.document.v1.DocumentService/CreateDocument"),
+ "sha256:aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899");
+ private static final String FINGERPRINT =
+ "sha256:0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0";
+ private static final String OTHER_FINGERPRINT =
+ "sha256:1122334455667788990011223344556677889900112233445566778899001122";
+ private static final String OUTCOME = "outcome://document/42";
+ private static final Instant NOW = Instant.parse("2026-09-25T10:00:00Z");
+
+ /** The ledger under test, holding no records when a test starts. */
+ protected abstract GrpcOperationLedger ledger();
+
+ @Test
+ void theFirstClaimOfAnOperationIsOwnedByItsCaller() {
+ assertThat(ledger().claim(ID, FINGERPRINT, NOW)).isEmpty();
+ }
+
+ @Test
+ void aSecondClaimReadsTheFirstClaimInProgress() {
+ // given
+ ledger().claim(ID, FINGERPRINT, NOW);
+
+ // when
+ Optional {@code putIfAbsent} on a concurrent map is the in-memory equivalent of the unique constraint
+ * the real adapter relies on, and {@code replace(key, expected, next)} of its conditional update,
+ * so claim and re-claim are atomic here for the same reason they are there. The transitions match
+ * the JPA adapter's because {@link GrpcOperationLedgerContract} runs against both; a fake that
+ * allowed more would let the policy tests pass for code the database refuses.
+ */
+public final class InMemoryOperationLedger implements GrpcOperationLedger {
+
+ private final ConcurrentMap Every ledger call runs in its own transaction, standing in for the caller's business
+ * transaction; the two threads of a concurrency test therefore race as two real transactions.
+ */
+@Testcontainers(disabledWithoutDocker = true)
+class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract {
+
+ @SuppressWarnings("resource")
+ @Container
+ private static final PostgreSQLContainer POSTGRES =
+ new PostgreSQLContainer("postgres:16-alpine")
+ .withDatabaseName("grpc")
+ .withUsername("grpc")
+ .withPassword("grpc");
+
+ private static EntityManagerFactory entityManagerFactory;
+ private static TransactionTemplate transactions;
+ private static JpaGrpcOperationLedger adapter;
+ private static GrpcOperationLedger ledger;
+
+ @BeforeAll
+ static void startJpa() throws SQLException, IOException {
+ execute(migration());
+ entityManagerFactory =
+ new PersistenceConfiguration("grpc-operation-ledger")
+ .managedClass(GrpcOperationLedgerEntity.class)
+ .property(PersistenceConfiguration.JDBC_URL, POSTGRES.getJdbcUrl())
+ .property(PersistenceConfiguration.JDBC_USER, POSTGRES.getUsername())
+ .property(PersistenceConfiguration.JDBC_PASSWORD, POSTGRES.getPassword())
+ .createEntityManagerFactory();
+ GrpcOperationLedgerRepository repository =
+ new JpaRepositoryFactory(
+ SharedEntityManagerCreator.createSharedEntityManager(entityManagerFactory))
+ .getRepository(GrpcOperationLedgerRepository.class);
+ transactions = new TransactionTemplate(new JpaTransactionManager(entityManagerFactory));
+ adapter = new JpaGrpcOperationLedger(repository);
+ ledger = new PerCallTransactionLedger(adapter, transactions);
+ }
+
+ @AfterAll
+ static void stopJpa() {
+ entityManagerFactory.close();
+ }
+
+ @BeforeEach
+ void emptyTheLedger() throws SQLException {
+ execute("TRUNCATE grpc_operation_ledger");
+ }
+
+ @Override
+ protected GrpcOperationLedger ledger() {
+ return ledger;
+ }
+
+ /**
+ * Adopters are told to claim and commit inside the business transaction. The claim loads the
+ * failed row into the persistence context, so without the re-claim clearing it the commit would
+ * read the stale FAILED_TERMINAL copy and refuse.
+ */
+ @Test
+ void aRetryThatReclaimsAndCommitsInOneTransactionCommits() {
+ // given
+ GrpcOperationIdentity id =
+ new GrpcOperationIdentity(
+ "tenant-1.actor-1",
+ GrpcMethodName.parse("hyeonworks.document.v1.DocumentService/CreateDocument"),
+ "sha256:aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899");
+ String fingerprint = "sha256:0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0";
+ Instant now = Instant.parse("2026-09-25T10:00:00Z");
+ ledger.claim(id, fingerprint, now);
+ ledger.markFailed(id, now.plusSeconds(1));
+
+ // when
+ transactions.executeWithoutResult(
+ status -> {
+ adapter.claim(id, fingerprint, now.plusSeconds(2));
+ adapter.reclaimFailed(id, fingerprint, now.plusSeconds(2));
+ adapter.markCommitted(id, "outcome://document/42", now.plusSeconds(3));
+ });
+
+ // then
+ assertThat(ledger.find(id))
+ .get()
+ .extracting(GrpcOperationLedgerRecord::state, GrpcOperationLedgerRecord::outcomeReference)
+ .containsExactly(GrpcOperationLedgerState.COMMITTED, Optional.of("outcome://document/42"));
+ }
+
+ private static void execute(String sql) throws SQLException {
+ try (Connection connection =
+ DriverManager.getConnection(
+ POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword());
+ Statement statement = connection.createStatement()) {
+ statement.execute(sql);
+ }
+ }
+
+ private static String migration() throws IOException {
+ try (InputStream stream =
+ JpaGrpcOperationLedgerContractIT.class
+ .getClassLoader()
+ .getResourceAsStream("db/migration/grpc/V001__create_grpc_operation_ledger.sql")) {
+ if (stream == null) {
+ throw new IOException("gRPC operation-ledger migration is missing");
+ }
+ return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ }
+
+ /** Runs each call in its own transaction, as a caller's business transaction would. */
+ private record PerCallTransactionLedger(
+ JpaGrpcOperationLedger delegate, TransactionTemplate transactions)
+ implements GrpcOperationLedger {
+
+ @Override
+ public Optional One conditional update, so PostgreSQL's row lock decides the race: a concurrent second
+ * re-claim re-evaluates the condition against the winner's {@code IN_PROGRESS} row and updates
+ * nothing. It clears the persistence context, because a row this transaction already loaded (the
+ * claim's read-on-conflict does) would otherwise still read {@code FAILED_TERMINAL} and refuse
+ * the commit.
+ *
+ * @return one when this caller re-claimed the row, zero otherwise
+ */
+ @Modifying(flushAutomatically = true, clearAutomatically = true)
+ @Transactional
+ @Query(
+ value =
+ """
+ UPDATE grpc_operation_ledger
+ SET state = 'IN_PROGRESS',
+ claimed_at = :claimedAt,
+ completed_at = NULL
+ WHERE storage_key = :storageKey
+ AND state = 'FAILED_TERMINAL'
+ AND request_fingerprint = :requestFingerprint
+ """,
+ nativeQuery = true)
+ int tryReclaimFailed(
+ @Param("storageKey") String storageKey,
+ @Param("requestFingerprint") String requestFingerprint,
+ @Param("claimedAt") Instant claimedAt);
+
/** Updates an existing claim after it has been loaded. */
GrpcOperationLedgerEntity save(GrpcOperationLedgerEntity entity);
diff --git a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java
index 7c03805..443847a 100644
--- a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java
+++ b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java
@@ -88,6 +88,13 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger {
});
}
+ @Override
+ public boolean reclaimFailed(
+ GrpcOperationIdentity identity, String requestFingerprint, Instant now) {
+ requireIdentity(identity);
+ return repository.tryReclaimFailed(identity.storageKey(), requestFingerprint, now) == 1;
+ }
+
@Override
public void release(GrpcOperationIdentity identity) {
requireIdentity(identity);
diff --git a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java
index fba811e..e1457b5 100644
--- a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java
+++ b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java
@@ -59,6 +59,12 @@ class GrpcOperationLedgerRepositoryTest {
return rows.putIfAbsent(storageKey, claim) == null ? 1 : 0;
}
+ @Override
+ public int tryReclaimFailed(String storageKey, String requestFingerprint, Instant claimedAt) {
+ throw new UnsupportedOperationException(
+ "the re-claim is certified against PostgreSQL by JpaGrpcOperationLedgerContractIT");
+ }
+
@Override
public GrpcOperationLedgerEntity save(GrpcOperationLedgerEntity entity) {
rows.put(entity.getStorageKey(), entity);
diff --git a/grpc/grpc-policy/build.gradle b/grpc/grpc-policy/build.gradle
index 17a4f5b..6d3694e 100644
--- a/grpc/grpc-policy/build.gradle
+++ b/grpc/grpc-policy/build.gradle
@@ -13,4 +13,7 @@ dependencies {
// signatures, so an adopter cannot compile against this module without them.
api "io.grpc:grpc-api"
api "io.grpc:grpc-stub"
+
+ // The shared operation-ledger fake, held to the same contract as the JPA adapter.
+ testImplementation testFixtures(project(':grpc:grpc-core-api'))
}
diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java
index 7307ed4..e232669 100644
--- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java
+++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptor.java
@@ -106,12 +106,16 @@ public final class GrpcIdempotencyInterceptor {
+ identity.storageKey()
+ "' has no outcome reference to replay")));
}
- if (record.state() == GrpcOperationLedgerState.FAILED_TERMINAL) {
+ if (record.state() == GrpcOperationLedgerState.FAILED_TERMINAL
+ && ledger.reclaimFailed(identity, requestFingerprint, now)) {
// A terminal failure is not replayed: the caller asked for the operation, it did not happen,
- // and repeating it is exactly what the key makes safe.
+ // and repeating it is exactly what the key makes safe — but only for the one retry that
+ // re-claims the row. Proceeding unclaimed runs concurrent retries twice and leaves the
+ // commit without an owner.
return GrpcIdempotencyDecision.proceed(
- "the previous attempt failed terminally; the key makes repeating it safe");
+ "the previous attempt failed terminally; this caller re-claimed it for a new attempt");
}
+ // Running, or a failed attempt another retry re-claimed first.
return waitForInProgress
? GrpcIdempotencyDecision.waitAndPoll(inProgressPollInterval)
: GrpcIdempotencyDecision.rejectInProgress();
diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java
index afb8e98..7bffcbc 100644
--- a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java
+++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcCompletionReconcilerTest.java
@@ -2,6 +2,7 @@ package dev.caskeleton.grpc.idempotency;
import static org.assertj.core.api.Assertions.assertThat;
+import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger;
import dev.caskeleton.grpc.core.GrpcMethodName;
import java.time.Instant;
import java.util.Optional;
diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java
index 51b8dea..7a09d60 100644
--- a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java
+++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/idempotency/GrpcIdempotencyInterceptorTest.java
@@ -4,6 +4,10 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import dev.caskeleton.grpc.api.deadline.GrpcDeadlineProfile;
+import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity;
+import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord;
+import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerState;
+import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger;
import dev.caskeleton.grpc.api.policy.GrpcMethodPolicy;
import dev.caskeleton.grpc.api.policy.GrpcMethodPolicyCatalog;
import dev.caskeleton.grpc.api.policy.RpcIdempotencyProfile;
@@ -12,6 +16,12 @@ import dev.caskeleton.grpc.core.GrpcMethodName;
import dev.caskeleton.grpc.core.RpcType;
import java.time.Duration;
import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
@@ -133,6 +143,57 @@ class GrpcIdempotencyInterceptorTest {
.isEqualTo(GrpcIdempotencyDecision.Outcome.PROCEED);
}
+ @Test
+ void aRetryOfATerminallyFailedOperationReclaimsItBeforeProceeding() {
+ // given
+ GrpcIdempotencyInterceptor interceptor = interceptor(false);
+ String fingerprint = GrpcRequestFingerprint.of("body");
+ interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW);
+ interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1));
+
+ // when
+ interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2));
+
+ // then
+ assertThat(
+ ledger
+ .find(
+ new GrpcOperationIdentity(
+ CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY)))
+ .map(GrpcOperationLedgerRecord::state))
+ .contains(GrpcOperationLedgerState.IN_PROGRESS);
+ }
+
+ @Test
+ void concurrentRetriesOfATerminallyFailedOperationProceedOnlyOnce() throws Exception {
+ // given
+ GrpcIdempotencyInterceptor interceptor = interceptor(false);
+ String fingerprint = GrpcRequestFingerprint.of("body");
+ interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW);
+ interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1));
+ CountDownLatch start = new CountDownLatch(1);
+ Callable {@code putIfAbsent} on a concurrent map is the in-memory equivalent of the unique constraint
- * the real adapter relies on, so the claim is atomic here for the same reason it is there. A fake
- * that read and then wrote would let the policy tests pass while the property they exist to check
- * does not hold.
- */
-final class InMemoryOperationLedger implements GrpcOperationLedger {
-
- private final ConcurrentMap