diff --git a/build.gradle b/build.gradle index 1c5ff1b..ecdce3e 100644 --- a/build.gradle +++ b/build.gradle @@ -75,9 +75,9 @@ tasks.register('releaseCheck') { } // The stages this build's strict lanes may belong to, executed by workflows as -// `./gradlew -p optional-platforms ` from `src`. +// `./gradlew -p optional-platforms ` from `src`. A command counts when it starts in this +// directory (working-directory joined with -p), which the plugin reads from where this build is. strictLaneStages { stage ':ci', ':integrationCheck', ':performanceCheck', ':releaseCheck' workflowDirectory = layout.projectDirectory.dir('../../.github/workflows') - workflowProjectDirectory = 'optional-platforms' } diff --git a/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md b/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md index 7307c65..7898463 100644 --- a/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md +++ b/docs/adr/ADR-GRPC-004-retry-ownership-and-durable-idempotency.md @@ -83,6 +83,17 @@ load-check-save let two commits that both read `IN_PROGRESS` both write; now exa A claim whose conflicting row is purged between insert and read retries the insert once instead of failing the call. +`release(identity, claimToken)` is fenced the same way (re-review 2026-11-03 round 2, RR-01-15): it is a +conditional `DELETE … WHERE state = 'IN_PROGRESS' AND claim_attempt = :attempt` and returns whether it +deleted. It used to delete by key alone, so any caller could erase the claim of the attempt that held +it, or a finished record a retry replays. The counter lives in the row, so a deleted row's key starts +again at attempt 1: the token separates the attempts of one row, not attempts across a delete. Both +deletions are limited accordingly — release only for a claim whose owner is known to be gone, purge +only for rows finished longer ago than any retry. Rejected: dropping `release` from the port +(`markFailed` then `reclaimFailed` keeps the counter, but a deployment that must free a key for a +different request has no other path), and a tombstone row that keeps the counter (a second state to +purge, for a window the two limits already close). + Chosen over the claim moment as the token (`WHERE claimed_at = :claimedAt`), which needs no column: PostgreSQL stores microseconds while `Instant` carries nanoseconds, so the token a caller holds and the one stored can differ by rounding, and two processes' clocks can give two attempts the same diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcClaimToken.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcClaimToken.java index 7b5fe35..269b00e 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcClaimToken.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcClaimToken.java @@ -12,7 +12,12 @@ package dev.caskeleton.grpc.api.ledger; * that still holds the claim. * *

A counter rather than the claim moment: it increases by one with every hand-over, so two - * attempts never share a value, whatever the clocks of the processes that claimed it. + * attempts of one record never share a value, whatever the clocks of the processes that claimed it. + * The counter lives in the record: once the record is deleted ({@code release}, or the purge of a + * finished record), the next claim of the key starts again at {@link #FIRST}. Both deletions are + * therefore limited to records no live attempt can still write to — a released claim's owner is + * known to be gone, and a purged record finished longer ago than any retry (review 2026-11-03 + * RR-01-15). * * @param attempt one for the first claim of an operation, one more for each hand-over */ diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java index 381979a..19c239c 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedger.java @@ -135,10 +135,22 @@ public interface GrpcOperationLedger { int purgeCompletedBefore(Instant completedBefore, int limit); /** - * Releases a claim that will never complete, so the caller may try again. + * Releases a claim that will never complete, so the caller may try again: deletes the record if + * it is still {@code IN_PROGRESS} and {@code claimToken} still holds it. * *

Only for a claim whose owner is known to be gone. Releasing one whose owner is merely slow - * re-admits the duplicate. + * re-admits the duplicate. Fenced like every other transition (review 2026-11-03 RR-01-15): an + * attempt that lost the claim cannot delete the claim of the attempt that took it over, and a + * finished record — the answer a retry replays — is never released. + * + *

The record is gone afterwards, so the next claim of the key starts again at {@link + * GrpcClaimToken#FIRST}. An earlier attempt that lost the claim and is still running would then + * hold a token equal to the new claim's; releasing is safe only when no such attempt can still + * record an outcome — which is what "the owner is known to be gone" has to cover. + * + * @param claimToken the token of the attempt that holds the claim + * @return true when the record was deleted; false when it is missing, finished, or held by + * another attempt, in which case nothing changed */ - void release(GrpcOperationIdentity identity); + boolean release(GrpcOperationIdentity identity, GrpcClaimToken claimToken); } diff --git a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java index d91f8de..ac84f97 100644 --- a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java +++ b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/GrpcOperationLedgerContract.java @@ -429,6 +429,60 @@ public abstract class GrpcOperationLedgerContract { .isEqualTo(takenOver); } + /** + * Release deletes the row, so it is fenced like every other transition: only the attempt that + * holds an unfinished claim may give it up (review 2026-11-03 RR-01-15). + */ + @Test + @DisplayName("진행 중 claim 을 가진 시도가 release 하면 키가 비고 다음 claim 이 첫 시도가 된다") + void releaseByTheAttemptHoldingAnUnfinishedClaimFreesTheKey() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + + // when + boolean released = ledger().release(ID, GrpcClaimToken.FIRST); + + // then + assertThat(List.of(released, ledger().claim(ID, FINGERPRINT, NOW.plusSeconds(1)).isEmpty())) + .containsExactly(true, true); + } + + @Test + @DisplayName("claim 을 넘겨준 옛 시도의 release 는 무시되고 새 시도의 claim 이 남는다") + void releaseByTheAttemptThatLostTheClaimChangesNothing() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().reclaimStale(ID, FINGERPRINT, CUTOFF, LATER); + + // when + boolean released = ledger().release(ID, GrpcClaimToken.FIRST); + + // then + assertThat(released).isFalse(); + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::state, GrpcOperationLedgerRecord::claimToken) + .containsExactly(GrpcOperationLedgerState.IN_PROGRESS, GrpcClaimToken.FIRST.next()); + } + + @Test + @DisplayName("끝난 claim 은 release 로 지워지지 않는다 — 재시도에 결과를 돌려줄 기록이다") + void releaseOfAFinishedClaimChangesNothing() { + // given + ledger().claim(ID, FINGERPRINT, NOW); + ledger().markCommitted(ID, GrpcClaimToken.FIRST, OUTCOME, NOW.plusSeconds(1)); + + // when + boolean released = ledger().release(ID, GrpcClaimToken.FIRST); + + // then + assertThat(released).isFalse(); + assertThat(ledger().find(ID)) + .get() + .extracting(GrpcOperationLedgerRecord::state) + .isEqualTo(GrpcOperationLedgerState.COMMITTED); + } + /** * The re-review probe's sequence (2026-11-03, RR-02-04): A claims and is slow, B takes the claim * over after the TTL, then A reports a failure. Recording it would mark B's running attempt diff --git a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java index bc26942..7aa3866 100644 --- a/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java +++ b/grpc/grpc-core-api/src/testFixtures/java/dev/caskeleton/grpc/api/ledger/InMemoryOperationLedger.java @@ -156,8 +156,12 @@ public final class InMemoryOperationLedger implements GrpcOperationLedger { } @Override - public void release(GrpcOperationIdentity identity) { - records.remove(identity.storageKey()); + public boolean release(GrpcOperationIdentity identity, GrpcClaimToken claimToken) { + GrpcOperationLedgerRecord existing = records.get(identity.storageKey()); + return existing != null + && existing.state() == GrpcOperationLedgerState.IN_PROGRESS + && existing.claimToken().equals(claimToken) + && records.remove(identity.storageKey(), existing); } // An attempt that lost the claim must not commit beside the attempt that holds it; checked diff --git a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrationLockWaitIT.java b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrationLockWaitIT.java new file mode 100644 index 0000000..15fc741 --- /dev/null +++ b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrationLockWaitIT.java @@ -0,0 +1,155 @@ +package dev.caskeleton.grpc.ledger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.postgresql.ds.PGSimpleDataSource; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.postgresql.PostgreSQLContainer; + +/** + * The ledger migrations after V001 lock a table already in use and give up within their bound + * instead of stopping it (re-review 2026-11-03 round 2, RR-02-09: the rule V21 follows). + * + *

A transaction that has written the ledger stays open for fifteen seconds — a claim in flight, + * an operator's {@code BEGIN}. A write is the holder both migrations wait for: V003's ALTER TABLE + * would wait for a read too, V002's plain CREATE INDEX only for a write. The migration runs the way + * Flyway runs it, in one transaction. Unbounded it waits until the writer leaves, with every claim + * queued behind it, and then applies; bounded it fails with {@code lock_not_available} first and + * its rollback leaves the ledger as it was. + */ +@Testcontainers(disabledWithoutDocker = true) +class GrpcOperationLedgerMigrationLockWaitIT { + + private static final String LOCK_NOT_AVAILABLE = "55P03"; + private static final Duration WRITER_STAYS = Duration.ofSeconds(15); + private static final Duration GIVES_UP_WITHIN = Duration.ofSeconds(10); + + @SuppressWarnings("resource") + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer(System.getProperty("ca.test.image.postgres-16")) + .withDatabaseName("grpc") + .withUsername("grpc") + .withPassword("grpc"); + + @ParameterizedTest + @ValueSource( + strings = { + "V002__index_grpc_operation_ledger_finished_rows.sql", + "V003__add_grpc_operation_ledger_claim_attempt.sql" + }) + @DisplayName("ledger에 쓴 트랜잭션이 열려 있으면 ledger를 잠그는 마이그레이션은 상한 안에 실패하고 아무것도 바꾸지 않는다") + void aLedgerMigrationGivesUpOnAnOpenWriterAndChangesNothing(String migration) throws Exception { + PGSimpleDataSource dataSource = dataSource(); + try (Connection setup = dataSource.getConnection(); + Statement statement = setup.createStatement()) { + // given + statement.execute("DROP TABLE IF EXISTS grpc_operation_ledger"); + statement.execute(GrpcOperationLedgerMigrations.before(migration)); + String schemaBefore = schema(statement); + + // when + Attempt attempt = + whileAWriterHolds(dataSource, GrpcOperationLedgerMigrations.named(migration)); + + // then + assertThat( + List.of( + String.valueOf(attempt.sqlState()), + attempt.took().compareTo(GIVES_UP_WITHIN) < 0, + schema(statement))) + .containsExactly(LOCK_NOT_AVAILABLE, true, schemaBefore); + } + } + + /** Runs the migration in one transaction while another one holds a write to the ledger. */ + private static Attempt whileAWriterHolds(PGSimpleDataSource dataSource, String migration) + throws SQLException { + try (Connection writer = dataSource.getConnection(); + Connection migrator = dataSource.getConnection()) { + writer.setAutoCommit(false); + try (Statement write = writer.createStatement()) { + write.execute("LOCK TABLE grpc_operation_ledger IN ROW EXCLUSIVE MODE"); + } + ScheduledExecutorService release = Executors.newSingleThreadScheduledExecutor(); + ScheduledFuture released = + release.schedule( + () -> { + writer.rollback(); + return null; + }, + WRITER_STAYS.toSeconds(), + TimeUnit.SECONDS); + try { + migrator.setAutoCommit(false); + long started = System.nanoTime(); + Throwable thrown = + catchThrowable( + () -> { + try (Statement statement = migrator.createStatement()) { + statement.execute(migration); + } + migrator.commit(); + }); + Duration took = Duration.ofNanos(System.nanoTime() - started); + migrator.rollback(); + return new Attempt(thrown, took); + } finally { + released.cancel(true); + release.shutdownNow(); + } + } + } + + /** The ledger's columns, indexes and constraints, enough to see whether a migration landed. */ + private static String schema(Statement statement) throws SQLException { + try (ResultSet row = + statement.executeQuery( + "SELECT (SELECT string_agg(column_name, ',' ORDER BY column_name)" + + " FROM information_schema.columns" + + " WHERE table_name = 'grpc_operation_ledger')" + + " || '|' || (SELECT string_agg(indexname, ',' ORDER BY indexname)" + + " FROM pg_indexes WHERE tablename = 'grpc_operation_ledger')" + + " || '|' || (SELECT string_agg(conname, ',' ORDER BY conname) FROM pg_constraint" + + " WHERE conrelid = 'grpc_operation_ledger'::regclass)")) { + row.next(); + return row.getString(1); + } + } + + private static PGSimpleDataSource dataSource() { + PGSimpleDataSource dataSource = new PGSimpleDataSource(); + dataSource.setUrl(POSTGRES.getJdbcUrl()); + dataSource.setUser(POSTGRES.getUsername()); + dataSource.setPassword(POSTGRES.getPassword()); + return dataSource; + } + + /** What the migration did while the writer held the ledger. */ + private record Attempt(Throwable thrown, Duration took) { + + String sqlState() { + for (Throwable cause = thrown; cause != null; cause = cause.getCause()) { + if (cause instanceof SQLException sql) { + return sql.getSQLState(); + } + } + return null; + } + } +} diff --git a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java index d84695d..749302b 100644 --- a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java +++ b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerMigrations.java @@ -23,16 +23,38 @@ final class GrpcOperationLedgerMigrations { /** Every migration's SQL, concatenated in version order. */ static String all() throws IOException { + return before(null); + } + + /** The SQL of every migration before the one named, concatenated in version order. */ + static String before(String fileName) throws IOException { StringBuilder sql = new StringBuilder(); for (String path : IN_ORDER) { - try (InputStream stream = - GrpcOperationLedgerMigrations.class.getClassLoader().getResourceAsStream(path)) { - if (stream == null) { - throw new IOException("gRPC operation-ledger migration is missing: " + path); - } - sql.append(new String(stream.readAllBytes(), StandardCharsets.UTF_8)).append('\n'); + if (path.endsWith("/" + fileName)) { + break; } + sql.append(read(path)).append('\n'); } return sql.toString(); } + + /** One migration's SQL, by its file name. */ + static String named(String fileName) throws IOException { + for (String path : IN_ORDER) { + if (path.endsWith("/" + fileName)) { + return read(path); + } + } + throw new IOException("no gRPC operation-ledger migration is named " + fileName); + } + + private static String read(String path) throws IOException { + try (InputStream stream = + GrpcOperationLedgerMigrations.class.getClassLoader().getResourceAsStream(path)) { + if (stream == null) { + throw new IOException("gRPC operation-ledger migration is missing: " + path); + } + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + } } diff --git a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java index ec357db..b4b645d 100644 --- a/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java +++ b/grpc/grpc-operation-ledger-jpa/src/integrationTest/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedgerContractIT.java @@ -233,8 +233,9 @@ class JpaGrpcOperationLedgerContractIT extends GrpcOperationLedgerContract { } @Override - public void release(GrpcOperationIdentity identity) { - transactions.executeWithoutResult(status -> delegate.release(identity)); + public boolean release(GrpcOperationIdentity identity, GrpcClaimToken claimToken) { + return Boolean.TRUE.equals( + transactions.execute(status -> delegate.release(identity, claimToken))); } } } diff --git a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java index 9a04367..c24507f 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java +++ b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepository.java @@ -228,8 +228,25 @@ public interface GrpcOperationLedgerRepository /** Reads a claim by its derived storage key. */ Optional findById(String storageKey); - /** Removes a claim. Only for one whose owner is known to be gone. */ - void deleteById(String storageKey); + /** + * Deletes the claim if attempt {@code claimAttempt} still holds it and it has not finished; the + * same fencing as {@link #tryMarkFailed}. Only for a claim whose owner is known to be gone + * (review 2026-11-03 RR-01-15). + * + * @return one when the claim was deleted, zero otherwise + */ + @Modifying(flushAutomatically = true, clearAutomatically = true) + @Transactional + @Query( + value = + """ + DELETE FROM grpc_operation_ledger + WHERE storage_key = :storageKey + AND state = 'IN_PROGRESS' + AND claim_attempt = :claimAttempt + """, + nativeQuery = true) + int tryRelease(@Param("storageKey") String storageKey, @Param("claimAttempt") long claimAttempt); /** Whether a claim exists. */ boolean existsById(String storageKey); diff --git a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java index 9db0c3d..5901103 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java +++ b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java @@ -201,9 +201,10 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { } @Override - public void release(GrpcOperationIdentity identity) { + public boolean release(GrpcOperationIdentity identity, GrpcClaimToken claimToken) { requireIdentity(identity); - repository.deleteById(identity.storageKey()); + requireToken(claimToken); + return repository.tryRelease(identity.storageKey(), claimToken.attempt()) == 1; } private static void requireToken(GrpcClaimToken claimToken) { diff --git a/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V002__index_grpc_operation_ledger_finished_rows.sql b/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V002__index_grpc_operation_ledger_finished_rows.sql index a0c6d7d..3c6adef 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V002__index_grpc_operation_ledger_finished_rows.sql +++ b/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V002__index_grpc_operation_ledger_finished_rows.sql @@ -11,6 +11,19 @@ -- A plain CREATE INDEX, not CONCURRENTLY: Flyway runs each migration in a transaction, and the -- platform is not adopted anywhere yet, so the table is empty when this runs. An adopter that -- applies it to a large existing ledger should run the CONCURRENTLY form by hand first. +-- +-- This file was edited after it was first written, before any deployment applied it (the branch +-- that added it was not merged, and the ledger is adopted nowhere yet), to add the bound below. A +-- development database that applied the earlier text fails Flyway validation: recreate it. + +-- Bound the wait for the lock this migration takes (SHARE on grpc_operation_ledger, held until it +-- commits; it stops writes, not reads). Unbounded, it queues behind any open transaction that has +-- written to the table, and every write of the table after it queues behind the migration — every +-- claim writes it. Five seconds, then the migration fails, rolls back whole and the database stays +-- at V001: deploy again once that transaction has ended (re-review 2026-11-03 round 2, RR-02-09; +-- the rule is docs/jpa/migration-guide.md "Bound the lock wait"). +SET LOCAL lock_timeout = '5s'; + CREATE INDEX ix_grpc_operation_ledger_finished_completed_at ON grpc_operation_ledger (completed_at) WHERE state IN ('COMMITTED', 'FAILED_TERMINAL'); diff --git a/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql b/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql index 23b0368..89007ad 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql +++ b/grpc/grpc-operation-ledger-jpa/src/main/resources/db/migration/grpc/V003__add_grpc_operation_ledger_claim_attempt.sql @@ -13,6 +13,19 @@ -- -- Existing rows are their first attempt: no row has been handed over under a token yet. The table is -- small and the platform is not adopted anywhere, so a plain ALTER with a default is enough. +-- +-- This file was edited after it was first written, before any deployment applied it (the branch +-- that added it was not merged, and the ledger is adopted nowhere yet), to add the bound below. A +-- development database that applied the earlier text fails Flyway validation: recreate it. + +-- Bound the wait for the lock this migration takes (ACCESS EXCLUSIVE on grpc_operation_ledger, held +-- until it commits). Unbounded, it queues behind any open transaction that has touched the table, +-- and every read and write of the table after it queues behind the migration — every claim reads +-- and writes it. Five seconds, then the migration fails, rolls back whole and the database stays at +-- V002: deploy again once that transaction has ended (re-review 2026-11-03 round 2, RR-02-09; the +-- rule is docs/jpa/migration-guide.md "Bound the lock wait"). +SET LOCAL lock_timeout = '5s'; + ALTER TABLE grpc_operation_ledger ADD COLUMN claim_attempt BIGINT NOT NULL DEFAULT 1; diff --git a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java index f2f9075..37c0478 100644 --- a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java +++ b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java @@ -16,6 +16,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; +import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -120,8 +121,13 @@ class GrpcOperationLedgerRepositoryTest { } @Override - public void deleteById(String storageKey) { - rows.remove(storageKey); + public int tryRelease(String storageKey, long claimAttempt) { + GrpcOperationLedgerEntity row = rows.get(storageKey); + boolean held = + row != null + && row.getState() == GrpcOperationLedgerState.IN_PROGRESS + && row.getClaimToken().equals(new GrpcClaimToken(claimAttempt)); + return held && rows.remove(storageKey, row) ? 1 : 0; } @Override @@ -320,18 +326,24 @@ class GrpcOperationLedgerRepositoryTest { assertThat(stored.sameRequestAs(OTHER_FINGERPRINT)).isFalse(); } + /** + * A failed record is finished: a retry re-claims it with a new token ({@code reclaimFailed}), so + * release, which deletes and resets the token, leaves it (review 2026-11-03 RR-01-15). + */ @Test - @DisplayName("a failed claim is terminal, and releasing removes it") - void failureIsTerminalAndReleaseRemoves() { + @DisplayName("실패한 claim 은 끝난 기록이라 release 로 지워지지 않는다") + void aFailedClaimIsTerminalAndReleaseLeavesIt() { + // given GrpcOperationIdentity id = identity("tenant-1.actor-1"); ledger.claim(id, FINGERPRINT, NOW); - ledger.markFailed(id, GrpcClaimToken.FIRST, NOW.plusSeconds(1)); - assertThat(ledger.find(id).orElseThrow().state()) - .isEqualTo(GrpcOperationLedgerState.FAILED_TERMINAL); - ledger.release(id); - assertThat(ledger.find(id)).isEmpty(); + // when + boolean released = ledger.release(id, GrpcClaimToken.FIRST); + + // then + assertThat(List.of(released, ledger.find(id).orElseThrow().state())) + .containsExactly(false, GrpcOperationLedgerState.FAILED_TERMINAL); } @Test