refactor: 부가 기능을 capability 모듈로 포장하고 기본 조립을 web-only로 전환
설계: docs/superpowers/specs/2026-09-25-capability-module-packaging-design.md
계획/진행 기록: docs/superpowers/plans/2026-09-25-capability-module-packaging.md
- Wave 0: 규칙 고정(AGENTS.md capability 완료 기준·테스트 규칙) + 차단 결함 7건
(compose overlay 검사, redis-sdk CI job, 적용된 V9/V13 원복 + V16, @EnableScheduling,
messaging bridge 순서, prod 에러 상세 validator, sample loggers read-only)
- Wave 1: 데이터·보안 Critical 10건 (durable operation 25P02, operation 유스케이스 이동,
outbox 재인코딩, Kafka TLS/SASL, 알림 트랜잭션, SNS 서명, tus checksum, GraphQL fragment,
Redis lane, gRPC FAILED_TERMINAL)
- Wave 2: 리팩터링 잔재·실험용 자리 삭제, outbox v1 fencing(V17), 멱등성 V2 정본화
- Wave 3: 라이브러리형 코드 정리(동작하지 않는 설정·인터페이스·gate 제거, web-advanced 분리)
- Wave 4: capability 11개를 "있으면 켜짐" 모듈로 포장, 끄는 장치 제거, 기본 조립 web-only,
qualificationBootJar 분리
- Wave 5: 빌드 로직 축소(17.2k -> 10.8k줄), sample 테스트 컨벤션 적용
삭제한 코드는 docs/capability-status.md "제거된 코드" 표에 복구 커밋(27022328)과 함께 기록.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
2702232830
commit
fd80de369d
@@ -1,5 +1,6 @@
|
||||
plugins {
|
||||
id 'ca.platform-module'
|
||||
id 'java-test-fixtures'
|
||||
}
|
||||
|
||||
// The platform's port layer: identifiers, method policy, execution evidence, failure model,
|
||||
@@ -10,4 +11,8 @@ plugins {
|
||||
// a transport" is verifiable instead of aspirational — the same rule `messaging-core-api` holds.
|
||||
// A type here may not name io.grpc, Spring, Netty, protobuf or a database.
|
||||
dependencies {
|
||||
// Test fixtures only, never main: the operation-ledger fake and the contract that both it and
|
||||
// the JPA adapter must pass, so the fake cannot allow a transition the database refuses.
|
||||
testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api'
|
||||
testFixturesImplementation 'org.assertj:assertj-core'
|
||||
}
|
||||
|
||||
@@ -1,40 +1,40 @@
|
||||
# This is a Gradle generated file for dependency locking.
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.github.ben-manes.caffeine:caffeine:3.2.4=annotationProcessor,testAnnotationProcessor
|
||||
com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor
|
||||
com.github.ben-manes.caffeine:caffeine:3.2.4=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs
|
||||
com.github.spotbugs:spotbugs:4.10.2=spotbugs
|
||||
com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs
|
||||
com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor
|
||||
com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor
|
||||
com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor
|
||||
com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.google.code.findbugs:jsr305:3.0.2=checkstyle,spotbugs
|
||||
com.google.code.gson:gson:2.13.2=spotbugs
|
||||
com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor
|
||||
com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.google.errorprone:error_prone_annotations:2.41.0=spotbugs
|
||||
com.google.errorprone:error_prone_annotations:2.47.0=checkstyle
|
||||
com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor
|
||||
com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor
|
||||
com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor
|
||||
com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor
|
||||
com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor
|
||||
com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor
|
||||
com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.google.guava:guava:33.6.0-jre=checkstyle
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor
|
||||
com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor
|
||||
com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins
|
||||
com.puppycrawl.tools:checkstyle:13.5.0=checkstyle
|
||||
commons-beanutils:commons-beanutils:1.11.0=checkstyle
|
||||
commons-collections:commons-collections:3.2.2=checkstyle
|
||||
commons-io:commons-io:2.21.0=spotbugs
|
||||
info.picocli:picocli:4.7.7=checkstyle
|
||||
io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor
|
||||
io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor
|
||||
javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor
|
||||
io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
jaxen:jaxen:2.0.6=spotbugs
|
||||
net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath
|
||||
net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath
|
||||
net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs
|
||||
org.antlr:antlr4-runtime:4.13.2=checkstyle
|
||||
org.apache.bcel:bcel:6.12.0=spotbugs
|
||||
@@ -50,32 +50,32 @@ org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle
|
||||
org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle
|
||||
org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle
|
||||
org.apache.xbean:xbean-reflect:3.7=checkstyle
|
||||
org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath
|
||||
org.assertj:assertj-core:3.27.7=testCompileClasspath,testRuntimeClasspath
|
||||
org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testFixturesCompileClasspath
|
||||
org.assertj:assertj-core:3.27.7=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath
|
||||
org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle
|
||||
org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle
|
||||
org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle
|
||||
org.codehaus.plexus:plexus-utils:3.3.0=checkstyle
|
||||
org.dom4j:dom4j:2.2.0=spotbugs
|
||||
org.javassist:javassist:3.28.0-GA=checkstyle
|
||||
org.jspecify:jspecify:1.0.1=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath
|
||||
org.junit.jupiter:junit-jupiter-api:6.0.3=testCompileClasspath,testRuntimeClasspath
|
||||
org.jspecify:jspecify:1.0.1=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath,testFixturesAnnotationProcessor,testFixturesCompileClasspath
|
||||
org.junit.jupiter:junit-jupiter-api:6.0.3=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath
|
||||
org.junit.jupiter:junit-jupiter-engine:6.0.3=testRuntimeClasspath
|
||||
org.junit.jupiter:junit-jupiter-params:6.0.3=testCompileClasspath,testRuntimeClasspath
|
||||
org.junit.jupiter:junit-jupiter:6.0.3=testCompileClasspath,testRuntimeClasspath
|
||||
org.junit.platform:junit-platform-commons:6.0.3=testCompileClasspath,testRuntimeClasspath
|
||||
org.junit.platform:junit-platform-commons:6.0.3=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath
|
||||
org.junit.platform:junit-platform-engine:6.0.3=testRuntimeClasspath
|
||||
org.junit.platform:junit-platform-launcher:6.0.3=testRuntimeClasspath
|
||||
org.junit:junit-bom:6.0.3=testCompileClasspath,testRuntimeClasspath
|
||||
org.junit:junit-bom:6.0.3=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath
|
||||
org.junit:junit-bom:6.1.0=spotbugs
|
||||
org.mockito:mockito-core:5.20.0=mockitoAgent
|
||||
org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath
|
||||
org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testFixturesCompileClasspath,testFixturesRuntimeClasspath,testRuntimeClasspath
|
||||
org.ow2.asm:asm-analysis:9.10.1=spotbugs
|
||||
org.ow2.asm:asm-commons:9.10.1=spotbugs
|
||||
org.ow2.asm:asm-tree:9.10.1=spotbugs
|
||||
org.ow2.asm:asm-util:9.10.1=spotbugs
|
||||
org.ow2.asm:asm:9.10.1=spotbugs
|
||||
org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor
|
||||
org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor,testFixturesAnnotationProcessor
|
||||
org.reflections:reflections:0.10.2=checkstyle
|
||||
org.slf4j:slf4j-api:2.0.18=spotbugs,spotbugsSlf4j
|
||||
org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j
|
||||
|
||||
+12
@@ -40,6 +40,18 @@ public interface GrpcOperationLedger {
|
||||
/** Marks a claim terminally failed. */
|
||||
void markFailed(GrpcOperationIdentity identity, Instant now);
|
||||
|
||||
/**
|
||||
* Re-claims a {@code FAILED_TERMINAL} record for a new attempt of the same request.
|
||||
*
|
||||
* <p>A compare-and-set to {@code IN_PROGRESS}, atomic like {@link #claim}: of two concurrent
|
||||
* re-claims exactly one returns true. A record in any other state, or made for a different
|
||||
* request, is left untouched. A retry that proceeds without winning this leaves its commit with
|
||||
* no owner and runs concurrently with every other retry of the same key.
|
||||
*
|
||||
* @return true when this call now owns the record as a fresh attempt claimed at {@code now}
|
||||
*/
|
||||
boolean reclaimFailed(GrpcOperationIdentity identity, String requestFingerprint, Instant now);
|
||||
|
||||
/**
|
||||
* Releases a claim that will never complete, so the caller may try again.
|
||||
*
|
||||
|
||||
+4
-1
@@ -12,7 +12,10 @@ public enum GrpcOperationLedgerState {
|
||||
IN_PROGRESS(false),
|
||||
/** Committed, with a replayable outcome. */
|
||||
COMMITTED(true),
|
||||
/** Failed in a way that will not succeed on repetition. */
|
||||
/**
|
||||
* The attempt failed and changed nothing. A retry of the same request may reclaim it (one winner)
|
||||
* and run again; see {@code GrpcOperationLedger.reclaimFailed}.
|
||||
*/
|
||||
FAILED_TERMINAL(true);
|
||||
|
||||
private final boolean terminal;
|
||||
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
package dev.caskeleton.grpc.api.ledger;
|
||||
|
||||
class InMemoryOperationLedgerContractTest extends GrpcOperationLedgerContract {
|
||||
|
||||
private final InMemoryOperationLedger ledger = new InMemoryOperationLedger();
|
||||
|
||||
@Override
|
||||
protected GrpcOperationLedger ledger() {
|
||||
return ledger;
|
||||
}
|
||||
}
|
||||
+260
@@ -0,0 +1,260 @@
|
||||
package dev.caskeleton.grpc.api.ledger;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import dev.caskeleton.grpc.core.GrpcMethodName;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* The transitions every {@link GrpcOperationLedger} allows and refuses.
|
||||
*
|
||||
* <p>The in-memory fake and the JPA adapter (against real PostgreSQL) both extend this. A fake that
|
||||
* allows a transition the database refuses lets every policy test pass for code that fails in
|
||||
* production — which is how a retry of a {@code FAILED_TERMINAL} row once proceeded unclaimed.
|
||||
*/
|
||||
public abstract class GrpcOperationLedgerContract {
|
||||
|
||||
private static final GrpcOperationIdentity ID =
|
||||
new GrpcOperationIdentity(
|
||||
"tenant-1.actor-1",
|
||||
GrpcMethodName.parse("hyeonworks.document.v1.DocumentService/CreateDocument"),
|
||||
"sha256:aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899");
|
||||
private static final String FINGERPRINT =
|
||||
"sha256:0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0";
|
||||
private static final String OTHER_FINGERPRINT =
|
||||
"sha256:1122334455667788990011223344556677889900112233445566778899001122";
|
||||
private static final String OUTCOME = "outcome://document/42";
|
||||
private static final Instant NOW = Instant.parse("2026-09-25T10:00:00Z");
|
||||
|
||||
/** The ledger under test, holding no records when a test starts. */
|
||||
protected abstract GrpcOperationLedger ledger();
|
||||
|
||||
@Test
|
||||
void theFirstClaimOfAnOperationIsOwnedByItsCaller() {
|
||||
assertThat(ledger().claim(ID, FINGERPRINT, NOW)).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aSecondClaimReadsTheFirstClaimInProgress() {
|
||||
// given
|
||||
ledger().claim(ID, FINGERPRINT, NOW);
|
||||
|
||||
// when
|
||||
Optional<GrpcOperationLedgerRecord> second =
|
||||
ledger().claim(ID, OTHER_FINGERPRINT, NOW.plusSeconds(1));
|
||||
|
||||
// then
|
||||
assertThat(second)
|
||||
.get()
|
||||
.extracting(
|
||||
GrpcOperationLedgerRecord::state,
|
||||
GrpcOperationLedgerRecord::requestFingerprint,
|
||||
GrpcOperationLedgerRecord::claimedAt)
|
||||
.containsExactly(GrpcOperationLedgerState.IN_PROGRESS, FINGERPRINT, NOW);
|
||||
}
|
||||
|
||||
@Test
|
||||
void concurrentClaimsHaveExactlyOneOwner() throws Exception {
|
||||
// when
|
||||
List<Boolean> owned = concurrently(() -> ledger().claim(ID, FINGERPRINT, NOW).isEmpty());
|
||||
|
||||
// then
|
||||
assertThat(owned).containsExactlyInAnyOrder(true, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aCommittedClaimKeepsTheOutcomeARetryReplays() {
|
||||
// given
|
||||
ledger().claim(ID, FINGERPRINT, NOW);
|
||||
|
||||
// when
|
||||
ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1));
|
||||
|
||||
// then
|
||||
assertThat(ledger().find(ID))
|
||||
.get()
|
||||
.extracting(
|
||||
GrpcOperationLedgerRecord::state,
|
||||
GrpcOperationLedgerRecord::outcomeReference,
|
||||
GrpcOperationLedgerRecord::completedAt)
|
||||
.containsExactly(
|
||||
GrpcOperationLedgerState.COMMITTED,
|
||||
Optional.of(OUTCOME),
|
||||
Optional.of(NOW.plusSeconds(1)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void committingWithoutAClaimIsRefused() {
|
||||
// when
|
||||
Throwable refused = catchThrowable(() -> ledger().markCommitted(ID, OUTCOME, NOW));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("no claim exists");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aCommittedClaimCannotBeCommittedAgain() {
|
||||
// given
|
||||
ledger().claim(ID, FINGERPRINT, NOW);
|
||||
ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1));
|
||||
|
||||
// when
|
||||
Throwable refused =
|
||||
catchThrowable(
|
||||
() -> ledger().markCommitted(ID, "outcome://document/99", NOW.plusSeconds(2)));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("cannot transition again");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aFailedClaimIsTerminal() {
|
||||
// given
|
||||
ledger().claim(ID, FINGERPRINT, NOW);
|
||||
|
||||
// when
|
||||
ledger().markFailed(ID, NOW.plusSeconds(1));
|
||||
|
||||
// then
|
||||
assertThat(ledger().find(ID))
|
||||
.get()
|
||||
.extracting(GrpcOperationLedgerRecord::state, GrpcOperationLedgerRecord::completedAt)
|
||||
.containsExactly(GrpcOperationLedgerState.FAILED_TERMINAL, Optional.of(NOW.plusSeconds(1)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aFailedClaimCannotBeCommittedWithoutBeingReclaimed() {
|
||||
// given
|
||||
failedClaim();
|
||||
|
||||
// when
|
||||
Throwable refused =
|
||||
catchThrowable(() -> ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(2)));
|
||||
|
||||
// then
|
||||
assertThat(refused)
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("cannot transition again");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aFailedClaimIsReclaimedByARetryOfTheSameRequest() {
|
||||
// given
|
||||
failedClaim();
|
||||
|
||||
// when
|
||||
boolean reclaimed = ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2));
|
||||
|
||||
// then
|
||||
assertThat(reclaimed).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aReclaimedClaimIsANewAttemptInProgressSinceTheReclaim() {
|
||||
// given
|
||||
failedClaim();
|
||||
|
||||
// when
|
||||
ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2));
|
||||
|
||||
// then
|
||||
assertThat(ledger().find(ID))
|
||||
.get()
|
||||
.extracting(
|
||||
GrpcOperationLedgerRecord::state,
|
||||
GrpcOperationLedgerRecord::claimedAt,
|
||||
GrpcOperationLedgerRecord::completedAt)
|
||||
.containsExactly(
|
||||
GrpcOperationLedgerState.IN_PROGRESS, NOW.plusSeconds(2), Optional.empty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aReclaimedClaimCanBeCommitted() {
|
||||
// given
|
||||
failedClaim();
|
||||
ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2));
|
||||
|
||||
// when
|
||||
ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(3));
|
||||
|
||||
// then
|
||||
assertThat(ledger().find(ID))
|
||||
.get()
|
||||
.extracting(GrpcOperationLedgerRecord::state, GrpcOperationLedgerRecord::outcomeReference)
|
||||
.containsExactly(GrpcOperationLedgerState.COMMITTED, Optional.of(OUTCOME));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aFailedClaimIsNotReclaimedForADifferentRequest() {
|
||||
failedClaim();
|
||||
|
||||
assertThat(ledger().reclaimFailed(ID, OTHER_FINGERPRINT, NOW.plusSeconds(2))).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void anInProgressClaimIsNotReclaimed() {
|
||||
ledger().claim(ID, FINGERPRINT, NOW);
|
||||
|
||||
assertThat(ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(1))).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aCommittedClaimIsNotReclaimed() {
|
||||
// given
|
||||
ledger().claim(ID, FINGERPRINT, NOW);
|
||||
ledger().markCommitted(ID, OUTCOME, NOW.plusSeconds(1));
|
||||
|
||||
// when
|
||||
boolean reclaimed = ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2));
|
||||
|
||||
// then
|
||||
assertThat(reclaimed).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void concurrentReclaimsOfAFailedClaimHaveExactlyOneWinner() throws Exception {
|
||||
// given
|
||||
failedClaim();
|
||||
|
||||
// when
|
||||
List<Boolean> reclaimed =
|
||||
concurrently(() -> ledger().reclaimFailed(ID, FINGERPRINT, NOW.plusSeconds(2)));
|
||||
|
||||
// then
|
||||
assertThat(reclaimed).containsExactlyInAnyOrder(true, false);
|
||||
}
|
||||
|
||||
private void failedClaim() {
|
||||
ledger().claim(ID, FINGERPRINT, NOW);
|
||||
ledger().markFailed(ID, NOW.plusSeconds(1));
|
||||
}
|
||||
|
||||
/** Runs {@code attempt} on two threads released together and returns both results. */
|
||||
private static List<Boolean> concurrently(Callable<Boolean> attempt) throws Exception {
|
||||
CountDownLatch start = new CountDownLatch(1);
|
||||
Callable<Boolean> gated =
|
||||
() -> {
|
||||
start.await();
|
||||
return attempt.call();
|
||||
};
|
||||
try (ExecutorService executor = Executors.newFixedThreadPool(2)) {
|
||||
Future<Boolean> first = executor.submit(gated);
|
||||
Future<Boolean> second = executor.submit(gated);
|
||||
start.countDown();
|
||||
return List.of(first.get(), second.get());
|
||||
}
|
||||
}
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
package dev.caskeleton.grpc.api.ledger;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
|
||||
/**
|
||||
* A hand-rolled ledger for tests of code that depends on {@link GrpcOperationLedger}.
|
||||
*
|
||||
* <p>{@code putIfAbsent} on a concurrent map is the in-memory equivalent of the unique constraint
|
||||
* the real adapter relies on, and {@code replace(key, expected, next)} of its conditional update,
|
||||
* so claim and re-claim are atomic here for the same reason they are there. The transitions match
|
||||
* the JPA adapter's because {@link GrpcOperationLedgerContract} runs against both; a fake that
|
||||
* allowed more would let the policy tests pass for code the database refuses.
|
||||
*/
|
||||
public final class InMemoryOperationLedger implements GrpcOperationLedger {
|
||||
|
||||
private final ConcurrentMap<String, GrpcOperationLedgerRecord> records =
|
||||
new ConcurrentHashMap<>();
|
||||
private volatile boolean unavailable;
|
||||
|
||||
/** Makes every read fail, so the UNKNOWN path can be exercised. */
|
||||
public void makeUnavailable() {
|
||||
unavailable = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<GrpcOperationLedgerRecord> claim(
|
||||
GrpcOperationIdentity identity, String requestFingerprint, Instant now) {
|
||||
GrpcOperationLedgerRecord existing =
|
||||
records.putIfAbsent(
|
||||
identity.storageKey(),
|
||||
GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now));
|
||||
return Optional.ofNullable(existing);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<GrpcOperationLedgerRecord> find(GrpcOperationIdentity identity) {
|
||||
if (unavailable) {
|
||||
throw new IllegalStateException("ledger unavailable");
|
||||
}
|
||||
return Optional.ofNullable(records.get(identity.storageKey()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markCommitted(GrpcOperationIdentity identity, String outcomeReference, Instant now) {
|
||||
GrpcOperationLedgerRecord committed =
|
||||
records.computeIfPresent(
|
||||
identity.storageKey(),
|
||||
(key, existing) ->
|
||||
finish(
|
||||
existing,
|
||||
GrpcOperationLedgerState.COMMITTED,
|
||||
Optional.of(outcomeReference),
|
||||
now));
|
||||
if (committed == null) {
|
||||
throw new IllegalStateException(
|
||||
"cannot commit operation '" + identity.storageKey() + "': no claim exists");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markFailed(GrpcOperationIdentity identity, Instant now) {
|
||||
records.computeIfPresent(
|
||||
identity.storageKey(),
|
||||
(key, existing) ->
|
||||
finish(existing, GrpcOperationLedgerState.FAILED_TERMINAL, Optional.empty(), now));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean reclaimFailed(
|
||||
GrpcOperationIdentity identity, String requestFingerprint, Instant now) {
|
||||
GrpcOperationLedgerRecord current = records.get(identity.storageKey());
|
||||
return current != null
|
||||
&& current.state() == GrpcOperationLedgerState.FAILED_TERMINAL
|
||||
&& current.sameRequestAs(requestFingerprint)
|
||||
&& records.replace(
|
||||
identity.storageKey(),
|
||||
current,
|
||||
GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void release(GrpcOperationIdentity identity) {
|
||||
records.remove(identity.storageKey());
|
||||
}
|
||||
|
||||
// Only an IN_PROGRESS claim finishes; a terminal row is re-opened by reclaimFailed alone.
|
||||
private static GrpcOperationLedgerRecord finish(
|
||||
GrpcOperationLedgerRecord existing,
|
||||
GrpcOperationLedgerState state,
|
||||
Optional<String> outcomeReference,
|
||||
Instant now) {
|
||||
if (existing.state() != GrpcOperationLedgerState.IN_PROGRESS) {
|
||||
throw new IllegalStateException(
|
||||
"operation '"
|
||||
+ existing.identity().storageKey()
|
||||
+ "' is already "
|
||||
+ existing.state()
|
||||
+ " and cannot transition again");
|
||||
}
|
||||
return new GrpcOperationLedgerRecord(
|
||||
existing.identity(),
|
||||
state,
|
||||
existing.requestFingerprint(),
|
||||
outcomeReference,
|
||||
existing.claimedAt(),
|
||||
Optional.of(now));
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,10 @@ dependencies {
|
||||
integrationTestImplementation 'org.testcontainers:testcontainers-postgresql'
|
||||
integrationTestImplementation 'org.testcontainers:testcontainers-junit-jupiter'
|
||||
integrationTestImplementation 'org.postgresql:postgresql'
|
||||
|
||||
// The shared ledger contract, run against this adapter through a real JPA provider.
|
||||
integrationTestImplementation testFixtures(project(':grpc:grpc-core-api'))
|
||||
integrationTestRuntimeOnly 'org.hibernate.orm:hibernate-core'
|
||||
}
|
||||
|
||||
strictTestLanes {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.21=integrationTestCompileClasspath,integrationTestRuntimeClasspath
|
||||
com.fasterxml:classmate:1.7.3=integrationTestRuntimeClasspath
|
||||
com.github.ben-manes.caffeine:caffeine:3.2.4=annotationProcessor,integrationTestAnnotationProcessor,testAnnotationProcessor
|
||||
com.github.docker-java:docker-java-api:3.7.1=integrationTestCompileClasspath,integrationTestRuntimeClasspath
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.7.1=integrationTestCompileClasspath,integrationTestRuntimeClasspath
|
||||
@@ -30,6 +31,7 @@ com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,integrat
|
||||
com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,integrationTestAnnotationProcessor,testAnnotationProcessor
|
||||
com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins
|
||||
com.puppycrawl.tools:checkstyle:13.5.0=checkstyle
|
||||
com.sun.istack:istack-commons-runtime:4.1.2=integrationTestRuntimeClasspath
|
||||
commons-beanutils:commons-beanutils:1.11.0=checkstyle
|
||||
commons-codec:commons-codec:1.19.0=integrationTestCompileClasspath,integrationTestRuntimeClasspath
|
||||
commons-collections:commons-collections:3.2.2=checkstyle
|
||||
@@ -41,8 +43,12 @@ io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,integratio
|
||||
io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,integrationTestAnnotationProcessor,testAnnotationProcessor
|
||||
io.micrometer:micrometer-commons:1.16.7=compileClasspath,integrationTestCompileClasspath,integrationTestRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
|
||||
io.micrometer:micrometer-observation:1.16.7=compileClasspath,integrationTestCompileClasspath,integrationTestRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
|
||||
jakarta.activation:jakarta.activation-api:2.1.4=integrationTestRuntimeClasspath
|
||||
jakarta.annotation:jakarta.annotation-api:3.0.0=compileClasspath,integrationTestCompileClasspath,integrationTestRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
|
||||
jakarta.inject:jakarta.inject-api:2.0.1=integrationTestRuntimeClasspath
|
||||
jakarta.persistence:jakarta.persistence-api:3.2.0=compileClasspath,integrationTestCompileClasspath,integrationTestRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
|
||||
jakarta.transaction:jakarta.transaction-api:2.0.1=integrationTestRuntimeClasspath
|
||||
jakarta.xml.bind:jakarta.xml.bind-api:4.0.5=integrationTestRuntimeClasspath
|
||||
javax.inject:javax.inject:1=annotationProcessor,integrationTestAnnotationProcessor,testAnnotationProcessor
|
||||
jaxen:jaxen:2.0.6=spotbugs
|
||||
net.bytebuddy:byte-buddy:1.17.8=integrationTestCompileClasspath,integrationTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
|
||||
@@ -71,7 +77,14 @@ org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle
|
||||
org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle
|
||||
org.codehaus.plexus:plexus-utils:3.3.0=checkstyle
|
||||
org.dom4j:dom4j:2.2.0=spotbugs
|
||||
org.eclipse.angus:angus-activation:2.0.3=integrationTestRuntimeClasspath
|
||||
org.glassfish.jaxb:jaxb-core:4.0.9=integrationTestRuntimeClasspath
|
||||
org.glassfish.jaxb:jaxb-runtime:4.0.9=integrationTestRuntimeClasspath
|
||||
org.glassfish.jaxb:txw2:4.0.9=integrationTestRuntimeClasspath
|
||||
org.hibernate.models:hibernate-models:1.0.1=integrationTestRuntimeClasspath
|
||||
org.hibernate.orm:hibernate-core:7.2.24.Final=integrationTestRuntimeClasspath
|
||||
org.javassist:javassist:3.28.0-GA=checkstyle
|
||||
org.jboss.logging:jboss-logging:3.6.3.Final=integrationTestRuntimeClasspath
|
||||
org.jetbrains:annotations:17.0.0=integrationTestCompileClasspath,integrationTestRuntimeClasspath
|
||||
org.jspecify:jspecify:1.0.1=annotationProcessor,checkstyle,compileClasspath,integrationTestAnnotationProcessor,integrationTestCompileClasspath,integrationTestRuntimeClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
|
||||
org.junit.jupiter:junit-jupiter-api:6.0.3=integrationTestCompileClasspath,integrationTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
|
||||
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
package dev.caskeleton.grpc.ledger;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity;
|
||||
import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger;
|
||||
import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerContract;
|
||||
import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord;
|
||||
import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerState;
|
||||
import dev.caskeleton.grpc.core.GrpcMethodName;
|
||||
import jakarta.persistence.EntityManagerFactory;
|
||||
import jakarta.persistence.PersistenceConfiguration;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.time.Instant;
|
||||
import java.util.Optional;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.data.jpa.repository.support.JpaRepositoryFactory;
|
||||
import org.springframework.orm.jpa.JpaTransactionManager;
|
||||
import org.springframework.orm.jpa.SharedEntityManagerCreator;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
import org.testcontainers.postgresql.PostgreSQLContainer;
|
||||
|
||||
/**
|
||||
* Runs the ledger contract against {@link JpaGrpcOperationLedger} on Hibernate and PostgreSQL.
|
||||
*
|
||||
* <p>Every ledger call runs in its own transaction, standing in for the caller's business
|
||||
* transaction; the two threads of a concurrency test therefore race as two real transactions.
|
||||
*/
|
||||
@Testcontainers(disabledWithoutDocker = true)
|
||||
class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract {
|
||||
|
||||
@SuppressWarnings("resource")
|
||||
@Container
|
||||
private static final PostgreSQLContainer POSTGRES =
|
||||
new PostgreSQLContainer("postgres:16-alpine")
|
||||
.withDatabaseName("grpc")
|
||||
.withUsername("grpc")
|
||||
.withPassword("grpc");
|
||||
|
||||
private static EntityManagerFactory entityManagerFactory;
|
||||
private static TransactionTemplate transactions;
|
||||
private static JpaGrpcOperationLedger adapter;
|
||||
private static GrpcOperationLedger ledger;
|
||||
|
||||
@BeforeAll
|
||||
static void startJpa() throws SQLException, IOException {
|
||||
execute(migration());
|
||||
entityManagerFactory =
|
||||
new PersistenceConfiguration("grpc-operation-ledger")
|
||||
.managedClass(GrpcOperationLedgerEntity.class)
|
||||
.property(PersistenceConfiguration.JDBC_URL, POSTGRES.getJdbcUrl())
|
||||
.property(PersistenceConfiguration.JDBC_USER, POSTGRES.getUsername())
|
||||
.property(PersistenceConfiguration.JDBC_PASSWORD, POSTGRES.getPassword())
|
||||
.createEntityManagerFactory();
|
||||
GrpcOperationLedgerRepository repository =
|
||||
new JpaRepositoryFactory(
|
||||
SharedEntityManagerCreator.createSharedEntityManager(entityManagerFactory))
|
||||
.getRepository(GrpcOperationLedgerRepository.class);
|
||||
transactions = new TransactionTemplate(new JpaTransactionManager(entityManagerFactory));
|
||||
adapter = new JpaGrpcOperationLedger(repository);
|
||||
ledger = new PerCallTransactionLedger(adapter, transactions);
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
static void stopJpa() {
|
||||
entityManagerFactory.close();
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void emptyTheLedger() throws SQLException {
|
||||
execute("TRUNCATE grpc_operation_ledger");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected GrpcOperationLedger ledger() {
|
||||
return ledger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adopters are told to claim and commit inside the business transaction. The claim loads the
|
||||
* failed row into the persistence context, so without the re-claim clearing it the commit would
|
||||
* read the stale FAILED_TERMINAL copy and refuse.
|
||||
*/
|
||||
@Test
|
||||
void aRetryThatReclaimsAndCommitsInOneTransactionCommits() {
|
||||
// given
|
||||
GrpcOperationIdentity id =
|
||||
new GrpcOperationIdentity(
|
||||
"tenant-1.actor-1",
|
||||
GrpcMethodName.parse("hyeonworks.document.v1.DocumentService/CreateDocument"),
|
||||
"sha256:aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899");
|
||||
String fingerprint = "sha256:0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0";
|
||||
Instant now = Instant.parse("2026-09-25T10:00:00Z");
|
||||
ledger.claim(id, fingerprint, now);
|
||||
ledger.markFailed(id, now.plusSeconds(1));
|
||||
|
||||
// when
|
||||
transactions.executeWithoutResult(
|
||||
status -> {
|
||||
adapter.claim(id, fingerprint, now.plusSeconds(2));
|
||||
adapter.reclaimFailed(id, fingerprint, now.plusSeconds(2));
|
||||
adapter.markCommitted(id, "outcome://document/42", now.plusSeconds(3));
|
||||
});
|
||||
|
||||
// then
|
||||
assertThat(ledger.find(id))
|
||||
.get()
|
||||
.extracting(GrpcOperationLedgerRecord::state, GrpcOperationLedgerRecord::outcomeReference)
|
||||
.containsExactly(GrpcOperationLedgerState.COMMITTED, Optional.of("outcome://document/42"));
|
||||
}
|
||||
|
||||
private static void execute(String sql) throws SQLException {
|
||||
try (Connection connection =
|
||||
DriverManager.getConnection(
|
||||
POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword());
|
||||
Statement statement = connection.createStatement()) {
|
||||
statement.execute(sql);
|
||||
}
|
||||
}
|
||||
|
||||
private static String migration() throws IOException {
|
||||
try (InputStream stream =
|
||||
JpaGrpcOperationLedgerContractIT.class
|
||||
.getClassLoader()
|
||||
.getResourceAsStream("db/migration/grpc/V001__create_grpc_operation_ledger.sql")) {
|
||||
if (stream == null) {
|
||||
throw new IOException("gRPC operation-ledger migration is missing");
|
||||
}
|
||||
return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs each call in its own transaction, as a caller's business transaction would. */
|
||||
private record PerCallTransactionLedger(
|
||||
JpaGrpcOperationLedger delegate, TransactionTemplate transactions)
|
||||
implements GrpcOperationLedger {
|
||||
|
||||
@Override
|
||||
public Optional<GrpcOperationLedgerRecord> claim(
|
||||
GrpcOperationIdentity identity, String requestFingerprint, Instant now) {
|
||||
return transactions.execute(status -> delegate.claim(identity, requestFingerprint, now));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<GrpcOperationLedgerRecord> find(GrpcOperationIdentity identity) {
|
||||
return transactions.execute(status -> delegate.find(identity));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markCommitted(
|
||||
GrpcOperationIdentity identity, String outcomeReference, Instant now) {
|
||||
transactions.executeWithoutResult(
|
||||
status -> delegate.markCommitted(identity, outcomeReference, now));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markFailed(GrpcOperationIdentity identity, Instant now) {
|
||||
transactions.executeWithoutResult(status -> delegate.markFailed(identity, now));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean reclaimFailed(
|
||||
GrpcOperationIdentity identity, String requestFingerprint, Instant now) {
|
||||
return Boolean.TRUE.equals(
|
||||
transactions.execute(
|
||||
status -> delegate.reclaimFailed(identity, requestFingerprint, now)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void release(GrpcOperationIdentity identity) {
|
||||
transactions.executeWithoutResult(status -> delegate.release(identity));
|
||||
}
|
||||
}
|
||||
}
|
||||
+30
@@ -62,6 +62,36 @@ public interface GrpcOperationLedgerRepository
|
||||
@Param("requestFingerprint") String requestFingerprint,
|
||||
@Param("claimedAt") Instant claimedAt);
|
||||
|
||||
/**
|
||||
* Re-claims a terminally failed row as a new attempt of the same request.
|
||||
*
|
||||
* <p>One conditional update, so PostgreSQL's row lock decides the race: a concurrent second
|
||||
* re-claim re-evaluates the condition against the winner's {@code IN_PROGRESS} row and updates
|
||||
* nothing. It clears the persistence context, because a row this transaction already loaded (the
|
||||
* claim's read-on-conflict does) would otherwise still read {@code FAILED_TERMINAL} and refuse
|
||||
* the commit.
|
||||
*
|
||||
* @return one when this caller re-claimed the row, zero otherwise
|
||||
*/
|
||||
@Modifying(flushAutomatically = true, clearAutomatically = true)
|
||||
@Transactional
|
||||
@Query(
|
||||
value =
|
||||
"""
|
||||
UPDATE grpc_operation_ledger
|
||||
SET state = 'IN_PROGRESS',
|
||||
claimed_at = :claimedAt,
|
||||
completed_at = NULL
|
||||
WHERE storage_key = :storageKey
|
||||
AND state = 'FAILED_TERMINAL'
|
||||
AND request_fingerprint = :requestFingerprint
|
||||
""",
|
||||
nativeQuery = true)
|
||||
int tryReclaimFailed(
|
||||
@Param("storageKey") String storageKey,
|
||||
@Param("requestFingerprint") String requestFingerprint,
|
||||
@Param("claimedAt") Instant claimedAt);
|
||||
|
||||
/** Updates an existing claim after it has been loaded. */
|
||||
GrpcOperationLedgerEntity save(GrpcOperationLedgerEntity entity);
|
||||
|
||||
|
||||
+7
@@ -88,6 +88,13 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger {
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean reclaimFailed(
|
||||
GrpcOperationIdentity identity, String requestFingerprint, Instant now) {
|
||||
requireIdentity(identity);
|
||||
return repository.tryReclaimFailed(identity.storageKey(), requestFingerprint, now) == 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void release(GrpcOperationIdentity identity) {
|
||||
requireIdentity(identity);
|
||||
|
||||
+6
@@ -59,6 +59,12 @@ class GrpcOperationLedgerRepositoryTest {
|
||||
return rows.putIfAbsent(storageKey, claim) == null ? 1 : 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int tryReclaimFailed(String storageKey, String requestFingerprint, Instant claimedAt) {
|
||||
throw new UnsupportedOperationException(
|
||||
"the re-claim is certified against PostgreSQL by JpaGrpcOperationLedgerContractIT");
|
||||
}
|
||||
|
||||
@Override
|
||||
public GrpcOperationLedgerEntity save(GrpcOperationLedgerEntity entity) {
|
||||
rows.put(entity.getStorageKey(), entity);
|
||||
|
||||
@@ -13,4 +13,7 @@ dependencies {
|
||||
// signatures, so an adopter cannot compile against this module without them.
|
||||
api "io.grpc:grpc-api"
|
||||
api "io.grpc:grpc-stub"
|
||||
|
||||
// The shared operation-ledger fake, held to the same contract as the JPA adapter.
|
||||
testImplementation testFixtures(project(':grpc:grpc-core-api'))
|
||||
}
|
||||
|
||||
+7
-3
@@ -106,12 +106,16 @@ public final class GrpcIdempotencyInterceptor {
|
||||
+ identity.storageKey()
|
||||
+ "' has no outcome reference to replay")));
|
||||
}
|
||||
if (record.state() == GrpcOperationLedgerState.FAILED_TERMINAL) {
|
||||
if (record.state() == GrpcOperationLedgerState.FAILED_TERMINAL
|
||||
&& ledger.reclaimFailed(identity, requestFingerprint, now)) {
|
||||
// A terminal failure is not replayed: the caller asked for the operation, it did not happen,
|
||||
// and repeating it is exactly what the key makes safe.
|
||||
// and repeating it is exactly what the key makes safe — but only for the one retry that
|
||||
// re-claims the row. Proceeding unclaimed runs concurrent retries twice and leaves the
|
||||
// commit without an owner.
|
||||
return GrpcIdempotencyDecision.proceed(
|
||||
"the previous attempt failed terminally; the key makes repeating it safe");
|
||||
"the previous attempt failed terminally; this caller re-claimed it for a new attempt");
|
||||
}
|
||||
// Running, or a failed attempt another retry re-claimed first.
|
||||
return waitForInProgress
|
||||
? GrpcIdempotencyDecision.waitAndPoll(inProgressPollInterval)
|
||||
: GrpcIdempotencyDecision.rejectInProgress();
|
||||
|
||||
+1
@@ -2,6 +2,7 @@ package dev.caskeleton.grpc.idempotency;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger;
|
||||
import dev.caskeleton.grpc.core.GrpcMethodName;
|
||||
import java.time.Instant;
|
||||
import java.util.Optional;
|
||||
|
||||
+61
@@ -4,6 +4,10 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.grpc.api.deadline.GrpcDeadlineProfile;
|
||||
import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity;
|
||||
import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord;
|
||||
import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerState;
|
||||
import dev.caskeleton.grpc.api.ledger.InMemoryOperationLedger;
|
||||
import dev.caskeleton.grpc.api.policy.GrpcMethodPolicy;
|
||||
import dev.caskeleton.grpc.api.policy.GrpcMethodPolicyCatalog;
|
||||
import dev.caskeleton.grpc.api.policy.RpcIdempotencyProfile;
|
||||
@@ -12,6 +16,12 @@ import dev.caskeleton.grpc.core.GrpcMethodName;
|
||||
import dev.caskeleton.grpc.core.RpcType;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -133,6 +143,57 @@ class GrpcIdempotencyInterceptorTest {
|
||||
.isEqualTo(GrpcIdempotencyDecision.Outcome.PROCEED);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRetryOfATerminallyFailedOperationReclaimsItBeforeProceeding() {
|
||||
// given
|
||||
GrpcIdempotencyInterceptor interceptor = interceptor(false);
|
||||
String fingerprint = GrpcRequestFingerprint.of("body");
|
||||
interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW);
|
||||
interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1));
|
||||
|
||||
// when
|
||||
interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2));
|
||||
|
||||
// then
|
||||
assertThat(
|
||||
ledger
|
||||
.find(
|
||||
new GrpcOperationIdentity(
|
||||
CALLER, CREATE, GrpcRequestFingerprint.hashIdempotencyKey(KEY)))
|
||||
.map(GrpcOperationLedgerRecord::state))
|
||||
.contains(GrpcOperationLedgerState.IN_PROGRESS);
|
||||
}
|
||||
|
||||
@Test
|
||||
void concurrentRetriesOfATerminallyFailedOperationProceedOnlyOnce() throws Exception {
|
||||
// given
|
||||
GrpcIdempotencyInterceptor interceptor = interceptor(false);
|
||||
String fingerprint = GrpcRequestFingerprint.of("body");
|
||||
interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW);
|
||||
interceptor.recordFailure(CREATE, CALLER, KEY, NOW.plusSeconds(1));
|
||||
CountDownLatch start = new CountDownLatch(1);
|
||||
Callable<GrpcIdempotencyDecision.Outcome> retry =
|
||||
() -> {
|
||||
start.await();
|
||||
return interceptor.decide(CREATE, CALLER, KEY, fingerprint, NOW.plusSeconds(2)).outcome();
|
||||
};
|
||||
|
||||
// when
|
||||
List<GrpcIdempotencyDecision.Outcome> outcomes;
|
||||
try (ExecutorService executor = Executors.newFixedThreadPool(2)) {
|
||||
Future<GrpcIdempotencyDecision.Outcome> first = executor.submit(retry);
|
||||
Future<GrpcIdempotencyDecision.Outcome> second = executor.submit(retry);
|
||||
start.countDown();
|
||||
outcomes = List.of(first.get(), second.get());
|
||||
}
|
||||
|
||||
// then
|
||||
assertThat(outcomes)
|
||||
.containsExactlyInAnyOrder(
|
||||
GrpcIdempotencyDecision.Outcome.PROCEED,
|
||||
GrpcIdempotencyDecision.Outcome.REJECT_IN_PROGRESS);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("one caller's key does not suppress another caller's write")
|
||||
void oneCallersKeyDoesNotSuppressAnother() {
|
||||
|
||||
-81
@@ -1,81 +0,0 @@
|
||||
package dev.caskeleton.grpc.idempotency;
|
||||
|
||||
import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity;
|
||||
import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger;
|
||||
import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerRecord;
|
||||
import dev.caskeleton.grpc.api.ledger.GrpcOperationLedgerState;
|
||||
import java.time.Instant;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
|
||||
/**
|
||||
* A hand-rolled ledger for the policy tests.
|
||||
*
|
||||
* <p>{@code putIfAbsent} on a concurrent map is the in-memory equivalent of the unique constraint
|
||||
* the real adapter relies on, so the claim is atomic here for the same reason it is there. A fake
|
||||
* that read and then wrote would let the policy tests pass while the property they exist to check
|
||||
* does not hold.
|
||||
*/
|
||||
final class InMemoryOperationLedger implements GrpcOperationLedger {
|
||||
|
||||
private final ConcurrentMap<String, GrpcOperationLedgerRecord> records =
|
||||
new ConcurrentHashMap<>();
|
||||
private boolean unavailable;
|
||||
|
||||
/** Makes every read fail, so the UNKNOWN path can be exercised. */
|
||||
void makeUnavailable() {
|
||||
unavailable = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<GrpcOperationLedgerRecord> claim(
|
||||
GrpcOperationIdentity identity, String requestFingerprint, Instant now) {
|
||||
GrpcOperationLedgerRecord existing =
|
||||
records.putIfAbsent(
|
||||
identity.storageKey(),
|
||||
GrpcOperationLedgerRecord.claim(identity, requestFingerprint, now));
|
||||
return Optional.ofNullable(existing);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<GrpcOperationLedgerRecord> find(GrpcOperationIdentity identity) {
|
||||
if (unavailable) {
|
||||
throw new IllegalStateException("ledger unavailable");
|
||||
}
|
||||
return Optional.ofNullable(records.get(identity.storageKey()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markCommitted(GrpcOperationIdentity identity, String outcomeReference, Instant now) {
|
||||
records.computeIfPresent(
|
||||
identity.storageKey(),
|
||||
(key, existing) ->
|
||||
new GrpcOperationLedgerRecord(
|
||||
existing.identity(),
|
||||
GrpcOperationLedgerState.COMMITTED,
|
||||
existing.requestFingerprint(),
|
||||
Optional.of(outcomeReference),
|
||||
existing.claimedAt(),
|
||||
Optional.of(now)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markFailed(GrpcOperationIdentity identity, Instant now) {
|
||||
records.computeIfPresent(
|
||||
identity.storageKey(),
|
||||
(key, existing) ->
|
||||
new GrpcOperationLedgerRecord(
|
||||
existing.identity(),
|
||||
GrpcOperationLedgerState.FAILED_TERMINAL,
|
||||
existing.requestFingerprint(),
|
||||
Optional.empty(),
|
||||
existing.claimedAt(),
|
||||
Optional.of(now)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void release(GrpcOperationIdentity identity) {
|
||||
records.remove(identity.storageKey());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user