diff --git a/docs/runbooks/lock-acquisition-timeout.md b/docs/runbooks/lock-acquisition-timeout.md index a01fdb8e..2f3f8bf4 100644 --- a/docs/runbooks/lock-acquisition-timeout.md +++ b/docs/runbooks/lock-acquisition-timeout.md @@ -14,7 +14,10 @@ status: stub - HTTP 409 with `error.code=LOCK_ACQUISITION_TIMEOUT` and `Retry-After: 1`; GraphQL extension `code=LOCK_ACQUISITION_TIMEOUT`, gRPC trailer `error-code: LOCK_ACQUISITION_TIMEOUT` (`ABORTED`), all - `retryable=true`. A 4xx answer, so no server ERROR line — look at the metric. + `retryable=true`. +- No server log line. A 4xx answer is not an incident, so no transport logs it, and the lock adapters + do not log a timeout either. The exception's message (lock key, wait time) reaches no log and no + client, so which lock timed out is recorded nowhere. Read the metric and the lock table below. - Before 2026-11-07 the template answered this as 500 `INTERNAL_ERROR` with an "unhandled exception" ERROR line (the exception was not an `ApiErrorCarrier`). A service on an older template sees that instead. @@ -22,8 +25,39 @@ status: stub ## Diagnosis -- Check `lock.acquisition` metric for lock name and duration -- Identify lock holders (check DB `integration_lock` table) +1. Contention rate. With `APP_MULTI_INSTANCE_ENABLED=true` the JDBC lock is metered: compare + `lock.acquisition{outcome="timeout"}` with `lock.acquisition{outcome="acquired"}`. The counter has + no lock name and no wait time — a key is never a metric tag (`docs/registries/metrics.yaml`). The + in-process lock (`APP_MULTI_INSTANCE_ENABLED=false`, the default) has no metric and keeps nothing in + the database; the steps below apply only to the JDBC lock. +2. Who holds locks now. `INT_LOCK` has one row per held lock. `CLIENT_ID` is the holding instance's + lock repository id, and the times are UTC. A row whose `EXPIRED_AFTER` has passed is a dead lease: + the next acquisition of that key takes it over. The oldest live rows are the long holders. + + +```sql +SELECT lock_key, region, client_id, created_date, expired_after +FROM int_lock +ORDER BY created_date; +``` + +3. The holder of one lock whose key you know from the code (for example `orders/42`). `LOCK_KEY` is + not the key: Spring Integration stores the key's name-based UUID (`UUID.nameUUIDFromBytes` of the + key's UTF-8 bytes), so a key cannot be read back from the table. This computes the UUID from the + key and finds its row: + + +```sql +SELECT region, client_id, created_date, expired_after +FROM int_lock +WHERE lock_key = ( + SELECT overlay(overlay(h placing '3' from 13) + placing to_hex(8 | (('x' || substr(h, 17, 1))::bit(4)::int & 3)) from 17)::uuid::text + FROM md5('') AS h); +``` + +`:adapter:outbound:persistence-jpa:postgresqlDistributedLockIntegrationTest` runs both blocks, read +from this file, against real PostgreSQL while locks are held. ## Action diff --git a/src/adapter/outbound/persistence-jpa/build.gradle b/src/adapter/outbound/persistence-jpa/build.gradle index 5d5abe27..4329176e 100644 --- a/src/adapter/outbound/persistence-jpa/build.gradle +++ b/src/adapter/outbound/persistence-jpa/build.gradle @@ -273,6 +273,14 @@ tasks.named('postgresqlConcurrentIndexMigrationIntegrationTest', Test) { systemProperty 'migration.failed.runbook', migrationFailedRunbook.absolutePath } +// The lock-timeout runbook gives an operator SQL to find who holds a lock; the lock lane reads it out +// of the runbook the same way, so the query an operator runs is the one that ran against PostgreSQL. +tasks.named('postgresqlDistributedLockIntegrationTest', Test) { + def lockTimeoutRunbook = new File(rootProject.projectDir.parentFile, 'docs/runbooks/lock-acquisition-timeout.md') + inputs.file(lockTimeoutRunbook).withPathSensitivity(PathSensitivity.RELATIVE).withPropertyName('lockTimeoutRunbook') + systemProperty 'lock.acquisition-timeout.runbook', lockTimeoutRunbook.absolutePath +} + tasks.named('postgresqlSecurityBaselineIntegrationTest') { // Cross-leaf task edge: see the handoff. verifyCleanArchitectureDependencies inspects // configurations, not the task graph, so this edge is invisible to it. diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlDistributedLockIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlDistributedLockIntegrationTest.java index 07b6d24c..a841d395 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlDistributedLockIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlDistributedLockIntegrationTest.java @@ -8,8 +8,12 @@ import dev.caskeleton.application.lock.DistributedLock; import dev.caskeleton.application.lock.DistributedLockPort; import dev.caskeleton.application.lock.LockAcquisitionTimeoutException; import dev.caskeleton.application.lock.testkit.DistributedLockContract; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ConcurrentModificationException; +import java.util.List; +import java.util.Map; +import java.util.UUID; import javax.sql.DataSource; import org.flywaydb.core.Flyway; import org.junit.jupiter.api.AfterAll; @@ -19,11 +23,18 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.springframework.integration.jdbc.lock.DefaultLockRepository; import org.springframework.integration.jdbc.lock.JdbcLockRegistry; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.datasource.DataSourceTransactionManager; -/** Real PostgreSQL contract for the Spring Integration JDBC distributed-lock adapter. */ +/** + * Real PostgreSQL contract for the Spring Integration JDBC distributed-lock adapter, and the SQL + * the lock-timeout runbook gives an operator to find a lock's holder, read out of the runbook + * itself. + */ class PostgreSqlDistributedLockIntegrationTest { + private static final String LOCK_TIMEOUT_RUNBOOK = "lock.acquisition-timeout.runbook"; + private static PostgreSqlReadinessSupport postgres; private static DataSource dataSource; @@ -120,6 +131,65 @@ class PostgreSqlDistributedLockIntegrationTest { } } + @Test + void theRunbookListsAHeldLockWithTheInstanceHoldingIt() throws Exception { + // given + OutboxRunbookSql runbook = OutboxRunbookSql.fromSystemProperty(LOCK_TIMEOUT_RUNBOOK); + Duration ttl = Duration.ofSeconds(30); + String holder = UUID.randomUUID().toString(); + DefaultLockRepository repository = repository(dataSource, holder); + String key = "orders/runbook-holders-" + System.nanoTime(); + DistributedLock held = adapter(repository, ttl).tryAcquire(key, Duration.ofSeconds(1), ttl); + try { + // when + List> rows = + new JdbcTemplate(dataSource).queryForList(runbook.block("lock-holders")); + + // then + assertThat(rows) + .filteredOn(row -> holder.equals(row.get("client_id"))) + .extracting(row -> row.get("lock_key")) + .containsExactly(nameBasedUuidOf(key)); + } finally { + held.close(); + repository.stop(); + } + } + + @Test + void theRunbookFindsTheInstanceHoldingTheLockOfAKeyItIsGiven() throws Exception { + // given + OutboxRunbookSql runbook = OutboxRunbookSql.fromSystemProperty(LOCK_TIMEOUT_RUNBOOK); + Duration ttl = Duration.ofSeconds(30); + String holder = UUID.randomUUID().toString(); + DefaultLockRepository holderRepository = repository(dataSource, holder); + DefaultLockRepository neighbourRepository = repository(dataSource); + String key = "orders/runbook-key-" + System.nanoTime(); + DistributedLock held = + adapter(holderRepository, ttl).tryAcquire(key, Duration.ofSeconds(1), ttl); + DistributedLock neighbour = + adapter(neighbourRepository, ttl).tryAcquire(key + "/next", Duration.ofSeconds(1), ttl); + try { + // when + List> rows = + new JdbcTemplate(dataSource) + .queryForList(runbook.block("lock-holder-of-key", Map.of("lock_key", key))); + + // then + assertThat(rows).extracting(row -> row.get("client_id")).containsExactly(holder); + } finally { + held.close(); + neighbour.close(); + holderRepository.stop(); + neighbourRepository.stop(); + } + } + + /** What Spring Integration stores as {@code LOCK_KEY} for a key that is not itself a UUID. */ + private static String nameBasedUuidOf(String key) { + return UUID.nameUUIDFromBytes(key.getBytes(StandardCharsets.UTF_8)).toString(); + } + private static DistributedLock acquireWithin( LockRegistryDistributedLock adapter, String key, Duration leaseTtl, Duration bound) throws InterruptedException { @@ -143,7 +213,11 @@ class PostgreSqlDistributedLockIntegrationTest { } private static DefaultLockRepository repository(DataSource dataSource) { - DefaultLockRepository repository = new DefaultLockRepository(dataSource); + return repository(dataSource, UUID.randomUUID().toString()); + } + + private static DefaultLockRepository repository(DataSource dataSource, String clientId) { + DefaultLockRepository repository = new DefaultLockRepository(dataSource, clientId); repository.setCheckDatabaseOnStart(false); repository.setTransactionManager(new DataSourceTransactionManager(dataSource)); repository.afterPropertiesSet(); diff --git a/src/application-core/README.md b/src/application-core/README.md index 5668a61e..a55c887e 100644 --- a/src/application-core/README.md +++ b/src/application-core/README.md @@ -582,7 +582,10 @@ application 계층은 락 획득/해제 계약만 알고, 실제 구현은 adapt - `ApiErrorCarrier` 를 구현한다. 그래서 전송이 따로 처리기를 두지 않아도 web 은 409 + `Retry-After: 1`(레지스트리 값), GraphQL·gRPC 는 같은 코드로 답하고, 4xx 라 서버 로그를 남기지 않는다. 전에는 carrier 가 아니어서 모든 전송이 500 `INTERNAL_ERROR`(재시도 불가)와 "unhandled" ERROR 로 답했다(재리뷰 - RR7-01-04). 진단용 `getMessage()`(key·waitTime 포함)는 서버 로그 전용이고 API 클라이언트에 전달하면 안 된다. + RR7-01-04). `getMessage()`(key·waitTime 포함)는 클라이언트에 보내지 않고, 로그에도 남지 않는다. 4xx carrier 라 + 어느 전송도 남기지 않고, 락 adapter 도 timeout 을 남기지 않는다. 어느 락이 막혔는지 기록이 필요하면 호출자가 + `key()` 를 직접 남긴다(엔티티 id 는 빼고). 운영 절차는 `docs/runbooks/lock-acquisition-timeout.md` + (`lock.acquisition` metric, `INT_LOCK` 보유자 조회)다(재리뷰 RR8-01-02). --- diff --git a/src/application-core/src/main/java/dev/caskeleton/application/lock/LockAcquisitionTimeoutException.java b/src/application-core/src/main/java/dev/caskeleton/application/lock/LockAcquisitionTimeoutException.java index 60dc410d..4f247129 100644 --- a/src/application-core/src/main/java/dev/caskeleton/application/lock/LockAcquisitionTimeoutException.java +++ b/src/application-core/src/main/java/dev/caskeleton/application/lock/LockAcquisitionTimeoutException.java @@ -11,7 +11,12 @@ import java.time.Duration; * through {@link ApiErrorCarrier}, the hook every inbound transport reads a code from: the web * adapter answers 409 with {@code Retry-After}, GraphQL and gRPC the same code, and none of them * logs it as an unhandled failure. Without the hook it was answered as a 500 {@code INTERNAL_ERROR} - * (review 2026-11-06 RR7-01-04). {@link #getMessage()} is server-log-only. See README. + * (review 2026-11-06 RR7-01-04). + * + *

{@link #getMessage()} names the key and the wait time. It is never sent to a client, and no + * transport logs it either: a 4xx carrier is an answer, not an incident, so this timeout leaves no + * server line, and the lock adapters do not log one. A caller that wants a record of which lock + * timed out logs {@link #key()} itself, keeping entity ids out of the line. See README. */ public final class LockAcquisitionTimeoutException extends RuntimeException implements ApiErrorCarrier {