package dev.caskeleton.adapter.outbound.persistence.operation; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import dev.caskeleton.adapter.outbound.persistence.platform.JpaPlatformContractSupport; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; /** * The durable operation store against a real PostgreSQL. * *

Every claim in this store is about concurrency or about a constraint, and neither survives * being tested against a fake. {@code FOR UPDATE SKIP LOCKED} does not exist in H2's PostgreSQL * mode with the semantics that matter, a {@code CHECK} constraint a fake does not enforce is a * comment, and a lease race has no meaning where only one connection exists. * *

A dead worker is written the only way that is faithful here: the row a crash leaves behind — a * lease nobody will renew — and then recovery runs against it. That is precisely what a killed * process is from the database's side. */ @Tag("jpa-contract") class DurableOperationStoreContractTest { private static final Instant NOW = Instant.parse("2026-08-25T09:00:00Z"); private static final Duration LEASE = Duration.ofMinutes(2); private static JpaPlatformContractSupport support; @BeforeAll static void startServer() { support = JpaPlatformContractSupport.start(); } @AfterAll static void stopServer() { if (support != null) { support.close(); } } @BeforeEach void migrate() throws SQLException, IOException { try (Connection connection = support.connection(); Statement statement = connection.createStatement()) { statement.execute("DROP SCHEMA public CASCADE"); statement.execute("CREATE SCHEMA public"); statement.execute(migration()); } } @Test @DisplayName("a lease left behind by a dead worker is reclaimed by another") void expiredLeaseCanBeReclaimedAfterWorkerCrash() throws SQLException { submit("op-1", "hash-1"); claim("worker-a", NOW); // The worker is gone. Nothing renews the lease, and nothing else may touch the operation until // it lapses — that exclusivity is the point of the lease. assertThat(claim("worker-b", NOW.plusSeconds(30))).isEmpty(); assertThat(claim("worker-b", NOW.plus(LEASE).plusSeconds(1))).contains("op-1"); assertThat(state("op-1")).isEqualTo("RUNNING"); assertThat(leaseOwner("op-1")).isEqualTo("worker-b"); } @Test @DisplayName("a reclaimed operation keeps the moment it first started") void reclaimKeepsTheOriginalStartTime() throws SQLException { submit("op-1", "hash-1"); claim("worker-a", NOW); claim("worker-b", NOW.plus(LEASE).plusSeconds(1)); // COALESCE, not an unconditional assignment. Overwriting it would make every retry look like a // fresh start and hide how long the operation has really been running. assertThat(startedAt("op-1")).isEqualTo(NOW); } @Test @DisplayName("two workers claiming at once take different operations") void concurrentWorkersDoNotTakeTheSameOperation() throws Exception { submit("op-1", "hash-1"); submit("op-2", "hash-2"); List claimed = new ArrayList<>(); try (Connection first = support.connection(); Connection second = support.connection()) { first.setAutoCommit(false); second.setAutoCommit(false); // Interleaved on purpose and inside open transactions: with plain FOR UPDATE the second // claim blocks here until the first commits, and with no locking at all both take op-1. claimed.add(claimOn(first, "worker-a", NOW).orElseThrow()); claimed.add(claimOn(second, "worker-b", NOW).orElseThrow()); first.commit(); second.commit(); } assertThat(claimed).containsExactlyInAnyOrder("op-1", "op-2"); } @Test @DisplayName("a worker whose lease lapsed cannot record a result over its successor") void aStaleWorkerCannotOverwriteTheNewOwner() throws SQLException { submit("op-1", "hash-1"); claim("worker-a", NOW); claim("worker-b", NOW.plus(LEASE).plusSeconds(1)); // worker-a is alive and still finishing. Letting it report here is how one operation ends up // with two answers, and which one survives would come down to timing. assertThat(succeed("op-1", "worker-a", "/results/a")).isZero(); assertThat(succeed("op-1", "worker-b", "/results/b")).isOne(); assertThat(resultReference("op-1")).isEqualTo("/results/b"); } @Test @DisplayName("a heartbeat from a worker that lost the lease fails rather than extending it") void heartbeatFromAStaleWorkerFails() throws SQLException { submit("op-1", "hash-1"); claim("worker-a", NOW); claim("worker-b", NOW.plus(LEASE).plusSeconds(1)); assertThat(heartbeat("op-1", "worker-a", NOW.plus(LEASE).plusSeconds(2))).isZero(); assertThat(heartbeat("op-1", "worker-b", NOW.plus(LEASE).plusSeconds(2))).isOne(); } @Test @DisplayName("cancel works once and never reverses a finished operation") void cancelIsIdempotentAndNeverReversesTerminal() throws SQLException { submit("op-1", "hash-1"); claim("worker-a", NOW); succeed("op-1", "worker-a", "/results/a"); // The work happened. Recording it as CANCELED would replace the only record of a real outcome // with a claim that it never occurred. assertThat(cancel("op-1", "alice")).isZero(); assertThat(state("op-1")).isEqualTo("SUCCEEDED"); submit("op-2", "hash-2"); assertThat(cancel("op-2", "alice")).isOne(); assertThat(cancel("op-2", "alice")).isZero(); assertThat(state("op-2")).isEqualTo("CANCELED"); } @Test @DisplayName("another principal cannot cancel an operation they did not submit") void cancelIsScopedToThePrincipal() throws SQLException { submit("op-1", "hash-1"); assertThat(cancel("op-1", "mallory")).isZero(); assertThat(state("op-1")).isEqualTo("PENDING"); } @Test @DisplayName("a cancelled operation is not handed to a worker") void cancelledOperationsAreNotClaimable() throws SQLException { submit("op-1", "hash-1"); cancel("op-1", "alice"); assertThat(claim("worker-a", NOW)).isEmpty(); } @Test @DisplayName("an identical resubmission is refused by the unique constraint") void resubmissionCollidesOnScope() throws SQLException { submit("op-1", "hash-1"); // The constraint is the arbiter rather than a prior read: two identical submissions can arrive // at the same instant, and a read-then-insert lets both through. assertThatThrownBy(() -> submit("op-2", "hash-1")).isInstanceOf(SQLException.class); } @Test @DisplayName("a succeeded row with no result is refused by the database") void succeededRowRequiresAResult() throws SQLException { submit("op-1", "hash-1"); assertThatThrownBy( () -> execute( "UPDATE durable_operation SET state = 'SUCCEEDED', completed_at = now()," + " lease_owner = NULL WHERE operation_id = 'op-1'")) .isInstanceOf(SQLException.class) .hasMessageContaining("ck_durable_operation_succeeded_has_result"); } @Test @DisplayName("a running row with no lease is refused by the database") void runningRowRequiresALease() { assertThatThrownBy( () -> { submit("op-1", "hash-1"); execute("UPDATE durable_operation SET state = 'RUNNING' WHERE operation_id = 'op-1'"); }) .isInstanceOf(SQLException.class) .hasMessageContaining("ck_durable_operation_running_has_lease"); } @Test @DisplayName("a state outside the published vocabulary is refused") void stateVocabularyIsClosed() { // A worker that could write its own state would extend the published contract by writing a // row, and a client branching on status would meet a value that is in no version of the API. assertThatThrownBy( () -> { submit("op-1", "hash-1"); execute( "UPDATE durable_operation SET state = 'ALMOST_DONE' WHERE operation_id = 'op-1'"); }) .isInstanceOf(SQLException.class) .hasMessageContaining("ck_durable_operation_state"); } @Test @DisplayName("an aged-out operation is expired, and a finished one is left alone") void expirySweepLeavesRealOutcomesAlone() throws SQLException { submit("op-1", "hash-1"); submit("op-2", "hash-2"); claim("worker-a", NOW); List expired = expireStale(NOW.plus(Duration.ofDays(2))); assertThat(expired).containsExactlyInAnyOrder("op-1", "op-2"); submit("op-3", "hash-3"); claim("worker-b", NOW); succeed("op-3", "worker-b", "/results/c"); assertThat(expireStale(NOW.plus(Duration.ofDays(2)))).isEmpty(); assertThat(state("op-3")).isEqualTo("SUCCEEDED"); } @Test @DisplayName("reclaim returns lapsed leases to the queue") void reclaimReturnsLapsedLeases() throws SQLException { submit("op-1", "hash-1"); claim("worker-a", NOW); assertThat(reclaim(NOW.plusSeconds(30))).isEmpty(); assertThat(reclaim(NOW.plus(LEASE).plusSeconds(1))).containsExactly("op-1"); assertThat(state("op-1")).isEqualTo("PENDING"); assertThat(leaseOwner("op-1")).isNull(); } private void submit(String operationId, String hash) throws SQLException { try (Connection connection = support.connection(); PreparedStatement statement = connection.prepareStatement( """ INSERT INTO durable_operation (operation_id, tenant, principal, operation_name, request_hash, payload, state, submitted_at, expires_at) VALUES (?, '', 'alice', 'transfers.create', ?, '{}', 'PENDING', ?, ?) """)) { statement.setString(1, operationId); statement.setString(2, hash); statement.setTimestamp(3, Timestamp.from(NOW)); statement.setTimestamp(4, Timestamp.from(NOW.plus(Duration.ofDays(1)))); statement.executeUpdate(); } } private Optional claim(String workerId, Instant now) throws SQLException { try (Connection connection = support.connection()) { return claimOn(connection, workerId, now); } } private Optional claimOn(Connection connection, String workerId, Instant now) throws SQLException { try (PreparedStatement statement = connection.prepareStatement( """ UPDATE durable_operation SET state = 'RUNNING', lease_owner = ?, lease_acquired_at = ?, lease_expires_at = ?, started_at = COALESCE(started_at, ?), row_version = row_version + 1 WHERE operation_id = ( SELECT operation_id FROM durable_operation WHERE (state = 'PENDING' OR (state = 'RUNNING' AND lease_expires_at <= ?)) AND expires_at > ? ORDER BY submitted_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING operation_id """)) { statement.setString(1, workerId); statement.setTimestamp(2, Timestamp.from(now)); statement.setTimestamp(3, Timestamp.from(now.plus(LEASE))); statement.setTimestamp(4, Timestamp.from(now)); statement.setTimestamp(5, Timestamp.from(now)); statement.setTimestamp(6, Timestamp.from(now)); try (ResultSet rows = statement.executeQuery()) { return rows.next() ? Optional.of(rows.getString(1)) : Optional.empty(); } } } @Test void analysisProbeExpiredLeaseCanStillSucceedBeforeTakeover() throws SQLException { submit("op-expired", "hash-expired"); claim("worker-a", NOW); Instant afterLease = NOW.plus(LEASE).plusSeconds(1); int updated = succeedAt("op-expired", "worker-a", "/results/stale", afterLease); System.out.println("durableExpiredLease.completionAt=" + afterLease); System.out.println("durableExpiredLease.leaseExpiredAtCompletion=true"); System.out.println("durableExpiredLease.succeedUpdatedRows=" + updated); System.out.println("durableExpiredLease.finalState=" + state("op-expired")); assertThat(updated).isOne(); } private int succeedAt(String operationId, String workerId, String resultReference, Instant completedAt) throws SQLException { try (Connection connection = support.connection(); PreparedStatement statement = connection.prepareStatement( "UPDATE durable_operation SET state = 'SUCCEEDED', result_reference = ?, completed_at = ?, " + "lease_owner = NULL, lease_acquired_at = NULL, lease_expires_at = NULL, " + "row_version = row_version + 1 " + "WHERE operation_id = ? AND state = 'RUNNING' AND lease_owner = ?")) { statement.setString(1, resultReference); statement.setTimestamp(2, Timestamp.from(completedAt)); statement.setString(3, operationId); statement.setString(4, workerId); return statement.executeUpdate(); } } private int succeed(String operationId, String workerId, String resultReference) throws SQLException { try (Connection connection = support.connection(); PreparedStatement statement = connection.prepareStatement( """ UPDATE durable_operation SET state = 'SUCCEEDED', result_reference = ?, completed_at = ?, lease_owner = NULL, lease_acquired_at = NULL, lease_expires_at = NULL, row_version = row_version + 1 WHERE operation_id = ? AND state = 'RUNNING' AND lease_owner = ? """)) { statement.setString(1, resultReference); statement.setTimestamp(2, Timestamp.from(NOW.plusSeconds(10))); statement.setString(3, operationId); statement.setString(4, workerId); return statement.executeUpdate(); } } private int heartbeat(String operationId, String workerId, Instant now) throws SQLException { try (Connection connection = support.connection(); PreparedStatement statement = connection.prepareStatement( """ UPDATE durable_operation SET lease_expires_at = ?, row_version = row_version + 1 WHERE operation_id = ? AND state = 'RUNNING' AND lease_owner = ? AND lease_expires_at > ? """)) { statement.setTimestamp(1, Timestamp.from(now.plus(LEASE))); statement.setString(2, operationId); statement.setString(3, workerId); statement.setTimestamp(4, Timestamp.from(now)); return statement.executeUpdate(); } } private int cancel(String operationId, String principal) throws SQLException { try (Connection connection = support.connection(); PreparedStatement statement = connection.prepareStatement( """ UPDATE durable_operation SET state = 'CANCELED', completed_at = ?, lease_owner = NULL, lease_acquired_at = NULL, lease_expires_at = NULL, row_version = row_version + 1 WHERE operation_id = ? AND principal = ? AND state IN ('PENDING', 'RUNNING') """)) { statement.setTimestamp(1, Timestamp.from(NOW.plusSeconds(5))); statement.setString(2, operationId); statement.setString(3, principal); return statement.executeUpdate(); } } private List reclaim(Instant now) throws SQLException { return idsFrom( """ UPDATE durable_operation SET state = 'PENDING', lease_owner = NULL, lease_acquired_at = NULL, lease_expires_at = NULL, row_version = row_version + 1 WHERE state = 'RUNNING' AND lease_expires_at <= ? RETURNING operation_id """, now); } private List expireStale(Instant now) throws SQLException { return idsFrom( """ UPDATE durable_operation SET state = 'EXPIRED', completed_at = ?, lease_owner = NULL, lease_acquired_at = NULL, lease_expires_at = NULL, row_version = row_version + 1 WHERE state IN ('PENDING', 'RUNNING') AND expires_at <= ? RETURNING operation_id """, now, now); } private List idsFrom(String sql, Instant... arguments) throws SQLException { try (Connection connection = support.connection(); PreparedStatement statement = connection.prepareStatement(sql)) { for (int index = 0; index < arguments.length; index++) { statement.setTimestamp(index + 1, Timestamp.from(arguments[index])); } List ids = new ArrayList<>(); try (ResultSet rows = statement.executeQuery()) { while (rows.next()) { ids.add(rows.getString(1)); } } return ids; } } private String state(String operationId) throws SQLException { return column(operationId, "state"); } private String leaseOwner(String operationId) throws SQLException { return column(operationId, "lease_owner"); } private String resultReference(String operationId) throws SQLException { return column(operationId, "result_reference"); } private Instant startedAt(String operationId) throws SQLException { try (Connection connection = support.connection(); PreparedStatement statement = connection.prepareStatement( "SELECT started_at FROM durable_operation WHERE operation_id = ?")) { statement.setString(1, operationId); try (ResultSet rows = statement.executeQuery()) { rows.next(); return rows.getTimestamp(1).toInstant(); } } } private String column(String operationId, String name) throws SQLException { try (Connection connection = support.connection(); PreparedStatement statement = connection.prepareStatement( "SELECT " + name + " FROM durable_operation WHERE operation_id = ?")) { statement.setString(1, operationId); try (ResultSet rows = statement.executeQuery()) { rows.next(); return rows.getString(1); } } } private void execute(String sql) throws SQLException { try (Connection connection = support.connection(); Statement statement = connection.createStatement()) { statement.execute(sql); } } private static String migration() throws IOException { try (InputStream resource = DurableOperationStoreContractTest.class .getClassLoader() .getResourceAsStream("db/migration/postgresql/V11__durable_operation.sql")) { if (resource == null) { throw new IOException("the durable operation migration is missing from the classpath"); } return new String(resource.readAllBytes(), StandardCharsets.UTF_8); } } }