diff --git a/docs/architecture/graphql-api-surface.txt b/docs/architecture/graphql-api-surface.txt index cd61da7e..5857b247 100644 --- a/docs/architecture/graphql-api-surface.txt +++ b/docs/architecture/graphql-api-surface.txt @@ -1,8 +1,8 @@ # GraphQL leaf public API surface — every public top-level type in src/main/java. # A public type in a single-jar leaf is reachable from every adopter's code, so -# additions are reviewed rather than discovered. `api` and `spi` are the intended -# external surface; the rest are candidates to become internal when this leaf is -# split into capability artifacts. +# additions are reviewed rather than discovered. `api` is the intended external +# surface; the rest are candidates to become internal when this leaf is split into +# capability artifacts. # Update only after review with: # ./gradlew :adapter:inbound:graphql:updateGraphQlApiSurface -PapproveGraphQlApiSurfaceChange # types: 161 diff --git a/docs/fileserver/http-contract.md b/docs/fileserver/http-contract.md index a5fb6aa6..d5518f01 100644 --- a/docs/fileserver/http-contract.md +++ b/docs/fileserver/http-contract.md @@ -9,7 +9,7 @@ |---|---|---|---| | POST | `/v1/files` | `201` READY, `202` VERIFYING | multipart 단일 업로드 | | POST | `/v1/files:raw` | `201`, `202` | 요청 본문 전체가 파일이다 | -| POST | `/v1/files:batch` | `200` | part별 결과를 순서대로 준다. 명시적으로 원자적이지 않다 | +| POST | `/v1/files:batch` | `200` | 공용 bulk envelope(`docs/web/http-contract.md` 1절). 명시적으로 원자적이지 않다 | | GET | `/v1/files/{fileId}` | `200` | 공개 metadata. content key나 경로는 절대 주지 않는다 | | GET | `/v1/files/{fileId}/content` | `200`, `206`, `304` | 다운로드 | | HEAD | `/v1/files/{fileId}/content` | `200`, `304` | 같은 헤더, 본문 없음 | @@ -76,7 +76,6 @@ servlet chain에서 `app.fileserver-platform.security.admin-roles`로 막고, ```json { "success": false, - "data": null, "error": { "code": "UPLOAD_OFFSET_MISMATCH", "category": "CONFLICT", @@ -97,8 +96,10 @@ servlet chain에서 `app.fileserver-platform.security.admin-roles`로 막고, `ambiguous`는 client가 재시도 전에 읽어야 하는 필드다. 결과를 모르는 실패는 이미 효과가 났을 수 있다. `retryable`은 이번 발생에 대한 판단이고, 그 code가 허락하는 범위보다 넓어지지 않는다. -batch 업로드에서 거부된 part는 `results[]` 항목에 같은 `error` 객체(`code`, `category`, `message`, -`retryable`)를 담는다. +batch 업로드는 공용 bulk envelope다. 모든 part가 성공하면 `success: true`와 `results[]`(part마다 `clientPartId`, +`status`, `fileId`)다. 한 part라도 거부되면 `success: false`, `error.code = BATCH_PARTIAL_FAILURE`, `error.details[]`다. +`details[]` 항목은 part 순서의 `index`, `status`(`ok`·`error`), 성공한 part의 `id`(fileId), 거부된 part의 `code`와 +category 고정 문구 `message`다. 성공한 part는 되돌리지 않는다. ### Envelope 규칙의 프로토콜 예외 @@ -111,6 +112,43 @@ batch 업로드에서 거부된 part는 `results[]` 항목에 같은 `error` 객 `409 application/problem+json`, `type = https://iana.org/assignments/http-problem-types#mismatching-upload-offset`, `expectedOffset`과 `providedOffset`. Envelope가 아닌 실패 본문은 이것 하나뿐이다. +## 오류 코드 + +`error.code`는 `FileserverErrorCode`(application-fileserver)가 정한다. 아래 표가 그 전부이고, client는 이 값으로 +분기한다. `retryable`은 그 code가 허락하는 최대치다. 실제 응답의 `retryable`은 그 발생에 대한 판단이라 `true`인 +code도 `false`로 올 수 있다(위 「실패 본문」). `Retry-After`는 retryable인 발생에만 붙는다. + +| code | status | category | retryable | `Retry-After` | 언제 | +| --- | --- | --- | --- | --- | --- | +| `BAD_REQUEST` | 400 | `VALIDATION` | false | — | 헤더나 헤더 조합이 잘못됐다 | +| `UNAUTHENTICATED` | 401 | `AUTH` | false | — | 인증하지 않았다 | +| `ACCESS_DENIED` | 403 | `AUTHZ` | false | — | 권한이 없다. 존재를 숨기는 profile에서는 `FILE_NOT_FOUND`로 온다 | +| `FILE_NOT_FOUND` | 404 | `NOT_FOUND` | false | — | 파일이 없거나 숨겼다 | +| `FILE_ALREADY_EXISTS` | 409 | `CONFLICT` | false | — | 대상 이름이 이미 있다(copy·move는 새로 만들기만 한다) | +| `FILE_NOT_READY` | 409 | `CONFLICT` | true | 2 | 검증이 끝나지 않아 아직 읽을 수 없다 | +| `UPLOAD_OFFSET_MISMATCH` | 409 | `CONFLICT` | true | — | 이어 쓰기 offset이 서버 offset과 다르다. `details.currentOffset`부터 다시 보낸다 | +| `CONCURRENT_MODIFICATION` | 409 | `CONFLICT` | true | 1 | 같은 자원을 다른 요청이 바꾸는 중이다 | +| `UPLOAD_EXPIRED` | 410 | `CONFLICT` | false | — | 만료되거나 끝난(취소, 거부, 격리) 업로드다 | +| `CONTENT_LENGTH_REQUIRED` | 411 | `VALIDATION` | false | — | `require-content-length` profile인데 길이가 없다 | +| `PRECONDITION_FAILED` | 412 | `CONFLICT` | false | — | `If-Match`·`If-Unmodified-Since`가 맞지 않는다 | +| `FILE_TOO_LARGE` | 413 | `VALIDATION` | false | — | 파일 크기 정책을 넘었다 | +| `QUOTA_EXCEEDED` | 413 | `CONFLICT` | false | — | 저장 할당량을 넘었다 | +| `UNSUPPORTED_MEDIA_TYPE` | 415 | `VALIDATION` | false | — | 받지 않는 media type이다 | +| `RANGE_NOT_SATISFIABLE` | 416 | `VALIDATION` | false | — | Range를 만족할 수 없다. `Content-Range: bytes */<길이>`를 싣는다 | +| `INTEGRITY_MISMATCH` | 422 | `DATA_INTEGRITY` | false | — | 선언한 digest와 받은 바이트가 다르다 | +| `MALWARE_DETECTED` | 422 | `DATA_INTEGRITY` | false | — | 검사기가 거부했다 | +| `INVALID_PATH` | 422 | `VALIDATION` | false | — | 경로 형식이 틀렸다 | +| `PATH_OUTSIDE_NAMESPACE` | 422 | `VALIDATION` | false | — | 경로가 namespace 밖을 가리킨다 | +| `TRANSFER_ADMISSION_REJECTED` | 429 | `RATE_LIMIT` | true | 1 | 호출자 자신의 동시 전송 permit이 찼다. 자기 전송이 끝나면 풀린다 | +| `CHECKSUM_MISMATCH` | 460 | `DATA_INTEGRITY` | false | — | tus `Upload-Checksum`이 그 요청의 바이트와 다르다. 그 chunk만 버린다 | +| `PARTIAL_WRITE` | 500 | `TRANSIENT_DEPENDENCY` | false | — | 쓰기가 중간에 끊겼다 | +| `AMBIGUOUS_COMPLETION` | 500 | `DATA_INTEGRITY` | false | — | 결과를 모른다. 이미 효과가 났을 수 있어 그대로 재시도하지 않는다 | +| `ATOMIC_PUBLISH_UNSUPPORTED` | 503 | `PERMANENT_DEPENDENCY` | false | — | 저장소가 원자적 게시를 지원하지 않는다(배포 설정 결함) | +| `STORAGE_UNAVAILABLE` | 503 | `TRANSIENT_DEPENDENCY` | true | 30 | 저장소나 검사기를 쓸 수 없다 | +| `SERVER_BUSY` | 503 | `TRANSIENT_DEPENDENCY` | true | 1 | instance 전체의 전송 permit이나 전송 worker pool이 찼다. 공용 서버 포화 코드와 같다 | +| `TRANSFER_TIMEOUT` | 504 | `TRANSIENT_DEPENDENCY` | true | 5 | 전송이 시간 안에 끝나지 않았다 | +| `STORAGE_FULL` | 507 | `TRANSIENT_DEPENDENCY` | false | — | 저장 용량이 없다 | + ## 헤더 계약 | 헤더 | 계약 | diff --git a/docs/registries/error-codes.yaml b/docs/registries/error-codes.yaml index 134ffd8f..d3d531db 100644 --- a/docs/registries/error-codes.yaml +++ b/docs/registries/error-codes.yaml @@ -380,6 +380,21 @@ errors: compatibility_impact: none required_test: contract-verification:rate-limit + # source: 2026-11-12 리뷰 결정 ② — 서버 포화(허가 상한, 꽉 찬 work pool)는 전송마다 다른 코드였다(web-advanced + # ADMISSION_REFUSED 503, fileserver TRANSFER_ADMISSION_REJECTED 429, gRPC RATE_LIMIT_EXCEEDED). 이 코드 하나로 모은다. + # 호출자 자신의 빈도·동시성 초과만 429(RATE_LIMIT_EXCEEDED, fileserver TRANSFER_ADMISSION_REJECTED)다. + - code: SERVER_BUSY + category: TRANSIENT_DEPENDENCY + http_status: 503 + retryable: true + retry_after_seconds: 1 + owner_branch: 2026-11-12-round9-review + owner_layer: presentation + client_safe_message: "일시적인 오류입니다. 잠시 후 다시 시도해 주세요." + runbook_link: "runbook://server/busy" + compatibility_impact: behavior-change + required_test: contract-verification:server-saturation + # source: docs/web/http-contract.md §2 (2026-11-01 소유자 결정, 리뷰 02-01 대안 A) — 결과 불명. 같은 Idempotency-Key로 Retry-After 뒤 재전송. 새 키 금지. # 2초 = 기본 processing lease(30s)보다 짧은 고정값. 이전 "client는 polling"(retryable=false)은 조회 엔드포인트가 없어 폐기. # 값 하나: IdempotencyExecutor.IN_FLIGHT_RETRY_AFTER. processing-lease가 이 값 이하면 기동 실패(재리뷰 RR-02-05). diff --git a/docs/runbooks/server-busy.md b/docs/runbooks/server-busy.md new file mode 100644 index 00000000..cc3e2513 --- /dev/null +++ b/docs/runbooks/server-busy.md @@ -0,0 +1,46 @@ +--- +title: Runbook — 서버 포화(SERVER_BUSY) +category: TRANSIENT_DEPENDENCY +error_codes: [SERVER_BUSY] +severity: P2 +owner: oncall +last_updated: 2026-11-13 +status: 운영 +--- + +# Runbook: 서버 포화 (`runbook://server/busy`) + +이 instance가 요청을 더 받을 자리가 없어 바로 거절했다. 호출자의 빈도 초과(`RATE_LIMIT_EXCEEDED` 429)와 다르다. 이 +코드는 서버 쪽 상한이 찼다는 뜻이다. + +## 증상 + +- HTTP `503`과 `error.code = SERVER_BUSY`, `Retry-After: 1`. gRPC는 `UNAVAILABLE`와 trailer `error-code: SERVER_BUSY`. +- `http_server_requests_seconds_count{status="503"}`이 오르고, 같은 때 p95 지연은 오르지 않는다. 거절은 일을 하기 전에 + 나기 때문이다. +- 이 코드를 내는 곳은 셋이다. + +| 내는 곳 | 찬 상한 | 설정 | +| --- | --- | --- | +| web-advanced `VirtualThreadAdmissionFilter` | 동시에 처리하는 요청 수 | `app.web.advanced.mvc-virtual-threads.admission-limit` | +| fileserver 전송 허가 | instance 전체 업로드·직접 다운로드 permit, 전송 worker pool | `app.fileserver-platform.quota.instance-upload-permits`, `…direct-download-permits` | +| gRPC `GrpcCallAdmission` | feature 호출 worker와 대기열 | `app.grpc.server.executor-max-pool-size`, `executor-queue-capacity` | + +## 진단 + +1. 거절이 한 instance에만 몰리는지 본다. 한 instance만이면 부하 분산이 고르지 않거나 그 instance의 하류가 느리다. +2. 같은 때 하류 지연(DB pool 대기, 외부 HTTP 지연)을 본다. 하류가 느리면 허가를 쥔 요청이 오래 머물러 상한이 찬다. + 이때 상한을 올리면 하류 pool 앞의 줄만 길어진다. +3. 정상 부하에서 상한 자체가 작은지 본다. 거절률이 트래픽에 비례하고 하류가 건강하면 상한이 작다. + +## 조치 + +- 하류가 느린 경우: 하류 runbook(`db-unavailable`, `dependency-timeout`)을 먼저 따른다. 상한은 그대로 둔다. +- 상한이 작은 경우: instance를 늘리거나 위 표의 설정을 올린다. virtual thread `admission-limit`은 DB pool과 외부 + bulkhead를 넘지 않게 둔다(`docs/web/virtual-thread-profile.md`). +- client는 `Retry-After` 뒤에 같은 요청을 다시 보낸다. 고칠 것이 없으면 거절은 1초 안에 풀린다. + +## 에스컬레이션 + +- 5분 넘게 거절률이 전체 요청의 5%를 넘으면 P1으로 올리고 인프라 당번을 부른다. +- 모든 instance가 같이 거절하고 하류가 건강하면 용량 문제다. 배포 소유자에게 scale-out을 요청한다. diff --git a/docs/upgrade/2026-11-round9-contracts.md b/docs/upgrade/2026-11-round9-contracts.md index 8f89e0c3..5731f75c 100644 --- a/docs/upgrade/2026-11-round9-contracts.md +++ b/docs/upgrade/2026-11-round9-contracts.md @@ -16,4 +16,13 @@ | 2 | patch 거부를 읽는 client는 `PATCH_REJECTED`(400) 하나로 본다. 전에는 JSON이 아닌 본문이 `500 INTERNAL_ERROR`, 빈 문서·연산 모양 오류·merge 깊이 초과가 `400 BAD_PARAMETER`, 너무 깊은 pointer가 `403 PATCH_PATH_FORBIDDEN`이었다 | client가 옛 코드로 분기하던 곳이 다른 코드를 받는다 | patch route를 둔 서비스와 그 client | | 3 | patch 결과가 Bean Validation 제약을 어기면 `400 VALIDATION_FAILED`와 필드별 `details`가 나간다. patch 대상 타입의 제약을 확인한다 | 전에는 제약을 어긴 객체가 그대로 저장 경로로 갔다 | web-advanced patch를 쓰는 서비스 | | 4 | 스트림 `Failure` 기록의 `code`가 실패의 코드(예: `STREAM_ITEM_TOO_LARGE`) 또는 `INTERNAL_ERROR`다. `DEPENDENCY_FAILURE`로 분기하던 client를 고친다 | 그 분기가 다시는 맞지 않는다 | NDJSON·JSON-seq stream을 읽는 client | -| 5 | virtual thread profile을 켠 배포는 이제 실제로 허가 상한을 건다. `admission-limit`이 부하에 맞는지 본다 | 전에는 상한이 없어 모든 요청이 들어왔다. 이제 넘치면 `503 ADMISSION_REFUSED`와 `Retry-After: 1`이다 | `app.web.advanced.mvc-virtual-threads.enabled=true`인 배포 | +| 5 | virtual thread profile을 켠 배포는 이제 실제로 허가 상한을 건다. `admission-limit`이 부하에 맞는지 본다 | 전에는 상한이 없어 모든 요청이 들어왔다. 이제 넘치면 `503 SERVER_BUSY`와 `Retry-After: 1`이다 | `app.web.advanced.mvc-virtual-threads.enabled=true`인 배포 | +| 6 | fileserver batch(`POST /v1/files:batch`)를 읽는 client는 `success`로 가른다. 모두 성공하면 `success: true`와 최상위 `results[]`, 하나라도 실패하면 `success: false`, `error.code = BATCH_PARTIAL_FAILURE`, part별 `error.details[]`(`index`, `status`, `id`, `code`, `message`)다 | 전에는 실패가 있어도 `success: true`이고 결과가 `data.results[]`에 있었다. 옛 경로로 읽던 client는 결과를 못 찾는다 | fileserver를 쓰는 서비스와 그 client | +| 7 | `BulkItemResult.error(index, code, message)`를 부르던 코드를 `BulkItemResult.error(index, ApiErrorCode)`로 바꾼다. 문구는 code의 category 고정 문구가 된다 | 컴파일이 실패한다 | bulk endpoint를 둔 서비스 | +| 8 | 서버 포화 거절을 읽는 client는 `SERVER_BUSY`(503, retryable, `Retry-After: 1`)로 본다. web-advanced `ADMISSION_REFUSED`, fileserver instance·전송 pool 포화의 `TRANSFER_ADMISSION_REJECTED`(429), gRPC 포화의 `RESOURCE_EXHAUSTED`/`RATE_LIMIT_EXCEEDED`가 이 코드로 바뀌었다(gRPC status는 `UNAVAILABLE`). 호출자 자신의 동시 전송 초과만 `TRANSFER_ADMISSION_REJECTED` 429로 남는다 | 옛 코드로 분기하던 재시도 로직이 맞지 않는다. 자바에서 `WebAdvancedError.ADMISSION_REFUSED`를 참조하던 코드는 컴파일이 실패한다 | 세 전송의 client, web-advanced 코드를 참조하던 서비스 | +| 9 | 컨테이너가 넘긴 503(필터의 `sendError(503)` 등)은 `SERVER_BUSY`다. 비동기 요청 시간 초과만 `REQUEST_TIMEOUT`이다 | 전에는 모든 503이 `REQUEST_TIMEOUT`이었다 | web을 쓰는 모든 서비스의 client | +| 10 | 알림 template 게시 경로를 `/notification-templates`에서 `/notificationTemplates`로 바꾼다(prefix 뒤) | 옛 경로는 404다 | 알림 template을 게시하는 client와 배포 스크립트 | +| 11 | 알림 제출·template 게시의 모르는 `channel`·`slot`·`locale`·`fallbackLocale`은 `400 VALIDATION_FAILED`와 그 필드를 짚는 `details`다(전에는 details 없는 `BAD_PARAMETER`) | `BAD_PARAMETER`로 분기하던 client가 다른 코드를 받는다 | 알림 HTTP client | +| 12 | 이름 없이 올린 파일의 표시 이름이 `upload.bin`에서 `file`이 된다(모든 전송에서 같다) | 표시 이름이 바뀐다. 동작은 같다 | fileserver를 쓰는 서비스 | +| 13 | fileserver `ATOMIC_PUBLISH_UNSUPPORTED`(재시도 불가 code)는 더 이상 `Retry-After`를 싣지 않는다. 본문 `retryable=false`와 맞춘 것이다 | 그 헤더로 재시도하던 client가 재시도하지 않는다(원래 재시도해도 소용없는 실패다) | fileserver client | +| 14 | 브라우저 client는 `Content-Range`를 읽을 수 있다(CORS 노출 목록에 더함) | — 더하기만 한 변경 | fileserver 다운로드를 이어 받는 브라우저 client | diff --git a/docs/web/http-contract.md b/docs/web/http-contract.md index 1f66a154..4b324818 100644 --- a/docs/web/http-contract.md +++ b/docs/web/http-contract.md @@ -177,7 +177,7 @@ CORS가 켜져 있으면(`app.cors.enabled=true`) cross-origin 실제 응답의 `traceparent`, `Location`, `ETag`, `X-RateLimit-Limit`, `X-RateLimit-Remaining`, `X-RateLimit-Reset` - fileserver를 실은 배포: `Tus-Resumable`, `Tus-Version`, `Tus-Extension`, `Tus-Max-Size`, `Upload-Offset`, `Upload-Length`, `Upload-Defer-Length`, `Upload-Expires`, `Upload-Complete`, - `Upload-Limit`, `Content-Disposition`, `Accept-Ranges` + `Upload-Limit`, `Content-Disposition`, `Accept-Ranges`, `Content-Range` 새 응답 헤더를 쓰는 모듈은 `CorsExposedHeaders` bean으로 자기 헤더를 기여한다. @@ -187,15 +187,18 @@ CORS가 켜져 있으면(`app.cors.enabled=true`) cross-origin 실제 응답의 | 코드 | status | category | `retryable` | `Retry-After` | 언제 | | --- | --- | --- | --- | --- | --- | -| `REQUEST_TIMEOUT` | 503 | `TRANSIENT_DEPENDENCY` | `true` | 없음 | 비동기 요청(`Callable`·`DeferredResult`·스트리밍)이 시간 안에 끝나지 않았다. GraphQL도 같은 이름을 쓰지만 그쪽 `category`는 GraphQL 전용 어휘 `TIMEOUT`이다(`src/adapter/inbound/graphql/README.md`) | +| `REQUEST_TIMEOUT` | 503 | `TRANSIENT_DEPENDENCY` | `true` | 없음 | 비동기 요청(`Callable`·`DeferredResult`·스트리밍)이 시간 안에 끝나지 않았다. 그 밖의 503(필터의 `sendError(503)` 등)은 공용 `SERVER_BUSY`다. GraphQL도 같은 이름을 쓰지만 그쪽 `category`는 GraphQL 전용 어휘 `TIMEOUT`이다(`src/adapter/inbound/graphql/README.md`) | | `IDEMPOTENCY_KEY_INVALID` | 400 | `VALIDATION` | `false` | 없음 | 2절 | | `RATE_LIMIT_UNAVAILABLE` | 503 | `TRANSIENT_DEPENDENCY` | `true` | 있음 | rate-limit 저장소 응답 없음(`docs/web/runbook.md`) | | `RATE_LIMIT_INCOMPATIBLE` | 503 | `INTERNAL` | `false` | 없음 | rate-limit 제공자가 계약과 맞지 않음 | +| `OPERATION_NOT_FOUND` | 404 | `NOT_FOUND` | `false` | 없음 | durable operation 조회(`DurableOperationController`)에서 그 operation이 없거나 다른 호출자의 것이다. 둘을 가르지 않는다 | - 컨트롤러 파라미터 제약 위반(`@RequestParam @Max(10)`)은 요청 본문 위반과 같은 `400 VALIDATION_FAILED`이고 `details`는 `[{field, code, message}]`다. 거절된 값은 싣지 않는다. -- web-advanced를 실은 서비스의 예외 코드(`ADMISSION_REFUSED` 503 등 10종)는 +- web-advanced를 실은 서비스의 예외 코드(`PATCH_REJECTED` 400 등)는 `src/adapter/inbound/web-advanced/README.md` 「오류 응답」이 정본이다. +- 서버 포화(허가 상한, 꽉 찬 work pool)는 모든 전송이 공용 `SERVER_BUSY`(503, retryable, `Retry-After: 1`) + 하나로 답한다. 호출자 자신의 빈도 초과만 `RATE_LIMIT_EXCEEDED` 429다(`docs/registries/error-codes.yaml`). - 직접 쓰는 거부 응답(401·403·429·503)도 컨트롤러 응답과 같이 `application/json;charset=UTF-8`이다. ## 5. 바꿀 때 diff --git a/docs/web/virtual-thread-profile.md b/docs/web/virtual-thread-profile.md index 610ea717..94522c31 100644 --- a/docs/web/virtual-thread-profile.md +++ b/docs/web/virtual-thread-profile.md @@ -40,7 +40,7 @@ thread로 얻은 것을 버리게 된다. virtual thread 만 개가 있어도 아무 일도 하지 않는다. async 요청은 async 일이 끝날 때까지 허가를 쥔다. 넘겨준 순간 돌려주면 넘기는 일만 제한하고 정작 하류 pool을 쓰는 일은 제한하지 못한다. -거부는 **`Retry-After`가 붙은 503**(`ADMISSION_REFUSED`)이고, 상한이 만들려는 결과가 바로 그것이다. 1밀리초 만에 거부된 요청이, 받아들여진 뒤 +거부는 **`Retry-After`가 붙은 503**(공용 코드 `SERVER_BUSY`)이고, 상한이 만들려는 결과가 바로 그것이다. 1밀리초 만에 거부된 요청이, 받아들여진 뒤 가득 찬 pool 뒤에서 30초 뒤에 시간 초과가 난 같은 요청보다 client에게 확실히 낫다. semaphore는 일부러 공정하다. 불공정한 것이 빠르지만, 새로 온 요청이 기다리던 요청을 앞지르게 두고, 앞지름당한 diff --git a/docs/work/reference/decisions.md b/docs/work/reference/decisions.md index db1f8aa0..3345f10a 100644 --- a/docs/work/reference/decisions.md +++ b/docs/work/reference/decisions.md @@ -116,6 +116,9 @@ | GraphQL 인자 변환 실패 | Spring GraphQL 인자 binder의 `BindException`(`@Argument` 값이 파라미터 타입으로 바뀌지 않음)은 `BAD_PARAMETER`(`REQUEST`)로 답하고, WARN 한 줄(예외 타입만, 입력값 없음)을 남긴다. web이 바뀌지 않는 경로 변수에 답하는 것과 같다 | 소유자 | 2026-11-07 같은 전달, r8-f1 보고 「범위 밖」 2 | | 약속한 응답으로 맞추기 | 레지스트리·`docs/web/http-contract.md`·Javadoc이 이미 약속한 응답으로 코드를 맞추는 것은 결함 수정이다. 고치기 전에 묻지 않고, 응답이 바뀌니 업그레이드 안내 행은 적는다. 약속한 곳끼리 다르면 묻는다 | 소유자(2026-11-07, 기본값 확인) | `docs/work/improve.md` 7절, `docs/work/evidence.md` 11절 | | STOMP 필수 header 누락 | 필수 header·destination 변수가 없을 때 Spring이 던지는 `MessageHandlingException`은 ERROR 그대로 둔다. 메시지에 입력값이 없고, Spring이 같은 타입으로 서버 실패(handler의 `Error` 등)도 감싸서 타입만으로는 클라이언트 탓을 가를 수 없다 | 기본값(2026-11-07) | 2026-11-07 총괄 지시, r8-f1 보고 「범위 밖」 3 | +| 서버 포화 코드 | 서버 쪽 상한(요청 허가, 꽉 찬 work pool)이 찬 거절은 모든 전송이 공용 `SERVER_BUSY`(503, `TRANSIENT_DEPENDENCY`, retryable, `Retry-After: 1`) 하나다. 호출자 자신의 빈도·동시성 초과만 429(`RATE_LIMIT_EXCEEDED`, fileserver `TRANSFER_ADMISSION_REJECTED`)다. 상태만 아는 503도 `SERVER_BUSY`이고, 비동기 요청 시간 초과만 `REQUEST_TIMEOUT`이다 | 기본값(2026-11-12, 소유자가 뒤집을 수 있다) | 2026-11-12 전체 리뷰 기록 README 6절 ② | +| batch 부분 실패 | fileserver batch도 공용 bulk envelope(web AGENTS B8)다. 하나라도 실패하면 `success:false` + `BATCH_PARTIAL_FAILURE` + 항목별 `details` | 기본값(2026-11-12) | 같은 README 6절 ① | +| PATCH 정본 | `docs/web/patch-contract.md`가 merge patch·JSON Patch의 정본이다. web AGENTS B2는 `application/json` PATCH DTO만 다루고 그쪽을 가리킨다 | 기본값(2026-11-12) | 같은 README 6절 ③ | ### 2.4 멱등성 diff --git a/src/adapter/inbound/graphql/build.gradle b/src/adapter/inbound/graphql/build.gradle index 53bbe3a2..1f2b622b 100644 --- a/src/adapter/inbound/graphql/build.gradle +++ b/src/adapter/inbound/graphql/build.gradle @@ -139,17 +139,17 @@ strictTestLanes { // be. // // A snapshot does not shrink the surface. It makes each addition visible in review, which is the -// prerequisite for shrinking it: the `api` and `spi` packages are the surface an adopter is meant -// to use, and everything else in this file is a candidate for becoming internal when the leaf is -// split into capability artifacts. Until then the number cannot grow by accident. +// prerequisite for shrinking it: the `api` package is the surface an adopter is meant to use, and +// everything else in this file is a candidate for becoming internal when the leaf is split into +// capability artifacts. Until then the number cannot grow by accident. apiSurface { label = 'GraphQl' baseline = rootProject.file('docs/architecture/graphql-api-surface.txt') description = 'GraphQL leaf public API surface — every public top-level type in src/main/java.' rationale = [ 'A public type in a single-jar leaf is reachable from every adopter\'s code, so', - 'additions are reviewed rather than discovered. `api` and `spi` are the intended', - 'external surface; the rest are candidates to become internal when this leaf is', - 'split into capability artifacts.', + 'additions are reviewed rather than discovered. `api` is the intended external', + 'surface; the rest are candidates to become internal when this leaf is split into', + 'capability artifacts.', ] } diff --git a/src/adapter/inbound/grpc/README.md b/src/adapter/inbound/grpc/README.md index e713d0cf..19eccf2a 100644 --- a/src/adapter/inbound/grpc/README.md +++ b/src/adapter/inbound/grpc/README.md @@ -94,7 +94,8 @@ executor, 서버 deadline 없음, 연결 수명 무제한이다. 어댑터 서 health/reflection 에는 적용하지 않는다(`Health/Watch` 는 원래 오래 사는 스트림이다). - **포화 시 동작(`GrpcCallAdmission`).** feature 호출은 `executor-max-pool-size + executor-queue-capacity` 개까지만 받고, 넘친 호출은 handler 를 시작하지 않고 즉시 - `RESOURCE_EXHAUSTED`(`RATE_LIMIT_EXCEEDED`, retryable)로 닫는다. 구현: io.grpc + `UNAVAILABLE`(`SERVER_BUSY`, retryable)로 닫는다. 서버 포화는 모든 전송이 이 공용 코드 하나다(2026-11-12 전에는 + `RESOURCE_EXHAUSTED`/`RATE_LIMIT_EXCEEDED`였다). 구현: io.grpc `ServerCallExecutorSupplier`(`@ExperimentalApi`, 1.39부터)가 호출 시작 전에 permit 을 확인해 받은 호출만 고정 크기 work pool 로 옮기고, 못 받은 호출은 가장 바깥 인터셉터가 닫는다. 메서드 조회·context 정리·health·reflection 은 작은 dispatch pool 에서 돈다. 버린 대안: diff --git a/src/adapter/inbound/grpc/src/main/java/dev/caskeleton/adapter/inbound/grpc/GrpcCallAdmission.java b/src/adapter/inbound/grpc/src/main/java/dev/caskeleton/adapter/inbound/grpc/GrpcCallAdmission.java index 1a8c8b77..f86c4012 100644 --- a/src/adapter/inbound/grpc/src/main/java/dev/caskeleton/adapter/inbound/grpc/GrpcCallAdmission.java +++ b/src/adapter/inbound/grpc/src/main/java/dev/caskeleton/adapter/inbound/grpc/GrpcCallAdmission.java @@ -28,9 +28,11 @@ import java.util.concurrent.atomic.AtomicBoolean; * dispatch executor, before the call starts. A feature call that gets a permit is moved to the * bounded work pool; one that does not stays on the dispatch executor and is marked, and this same * object, installed as the outermost feature interceptor, closes the marked call with {@code - * RATE_LIMIT_EXCEEDED} without starting the handler. The permit is returned once, when the admitted - * call completes or is cancelled. Health and reflection are not feature calls: they take no permit - * and run on the dispatch executor, so a saturated pool cannot make health lie. + * SERVER_BUSY} ({@code UNAVAILABLE}) without starting the handler: the shared code every transport + * answers server saturation with, not the caller-rate {@code RATE_LIMIT_EXCEEDED} it used to be. + * The permit is returned once, when the admitted call completes or is cancelled. Health and + * reflection are not feature calls: they take no permit and run on the dispatch executor, so a + * saturated pool cannot make health lie. * *

Each admitted call has at most one task in the work pool at a time (io.grpc serializes a * call's callbacks), so a work queue as deep as the permit count can never reject. @@ -79,8 +81,8 @@ final class GrpcCallAdmission implements ServerCallExecutorSupplier, ServerInter ServerCall call, Metadata headers, ServerCallHandler next) { if (refused.remove(headers)) { call.close( - statusMapper.statusFor(OperationalError.RATE_LIMIT_EXCEEDED), - statusMapper.trailersFor(OperationalError.RATE_LIMIT_EXCEEDED)); + statusMapper.statusFor(OperationalError.SERVER_BUSY), + statusMapper.trailersFor(OperationalError.SERVER_BUSY)); return new ServerCall.Listener<>() {}; } AtomicBoolean released = new AtomicBoolean(); diff --git a/src/adapter/inbound/grpc/src/main/java/dev/caskeleton/adapter/inbound/grpc/GrpcStatusMapper.java b/src/adapter/inbound/grpc/src/main/java/dev/caskeleton/adapter/inbound/grpc/GrpcStatusMapper.java index 1fa5c57a..0e7b1b36 100644 --- a/src/adapter/inbound/grpc/src/main/java/dev/caskeleton/adapter/inbound/grpc/GrpcStatusMapper.java +++ b/src/adapter/inbound/grpc/src/main/java/dev/caskeleton/adapter/inbound/grpc/GrpcStatusMapper.java @@ -17,9 +17,7 @@ import io.grpc.Status; */ public class GrpcStatusMapper { - /** - * Trailer key carrying the stable {@link ApiErrorCode#code()} (e.g. {@code WORKLOG_NOT_FOUND}). - */ + /** Trailer key carrying the stable {@link ApiErrorCode#code()} (e.g. {@code SERVER_BUSY}). */ static final Metadata.Key CODE_KEY = Metadata.Key.of("error-code", Metadata.ASCII_STRING_MARSHALLER); diff --git a/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcP1BoundaryWireTest.java b/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcP1BoundaryWireTest.java index 97fd4735..cbd20d75 100644 --- a/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcP1BoundaryWireTest.java +++ b/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcP1BoundaryWireTest.java @@ -329,8 +329,8 @@ class GrpcP1BoundaryWireTest { } @Test - @DisplayName("스레드와 대기열이 모두 찬 서버는 넘친 호출을 기다리게 두지 않고 즉시 RESOURCE_EXHAUSTED로 거절한다") - void aCallBeyondTheExecutorAndQueueIsRefusedAtOnceAsResourceExhausted() { + @DisplayName("스레드와 대기열이 모두 찬 서버는 넘친 호출을 기다리게 두지 않고 즉시 UNAVAILABLE(SERVER_BUSY)로 거절한다") + void aCallBeyondTheExecutorAndQueueIsRefusedAtOnceAsServerBusy() { withChannel( contextRunner.withPropertyValues( "app.grpc.server.executor-max-pool-size=1", @@ -351,12 +351,11 @@ class GrpcP1BoundaryWireTest { // then assertThat(outcomes) .extracting(Status::getCode) - .containsExactlyInAnyOrder( - Status.Code.OK, Status.Code.OK, Status.Code.RESOURCE_EXHAUSTED); + .containsExactlyInAnyOrder(Status.Code.OK, Status.Code.OK, Status.Code.UNAVAILABLE); assertThat(outcomes) .filteredOn(status -> !status.isOk()) .extracting(Status::getDescription) - .containsExactly(ClientErrorMessages.of(Category.RATE_LIMIT)); + .containsExactly(ClientErrorMessages.of(Category.TRANSIENT_DEPENDENCY)); }); } diff --git a/src/adapter/inbound/web-advanced/README.md b/src/adapter/inbound/web-advanced/README.md index 34f7c59c..9ae11896 100644 --- a/src/adapter/inbound/web-advanced/README.md +++ b/src/adapter/inbound/web-advanced/README.md @@ -51,12 +51,12 @@ virtual thread 실행기는 `TaskDecorator` bean 을 요구한다(요청 상관 이 leaf 의 예외는 모두 `ApiErrorCarrier` 다. 코드는 `error.WebAdvancedError` 한 enum 이 정하고, web 의 `GlobalExceptionHandler.handleCarrier` 한 분기가 Envelope 로 답한다(별도 `@RestControllerAdvice` 없음). -예전에는 10종 모두 `500 INTERNAL_ERROR` 였다(리뷰 2026-11-01 02-07). 아래 표와 enum 은 같은 값을 말하고, +예전에는 10종 모두 `500 INTERNAL_ERROR` 였다(리뷰 2026-11-01 02-07). admission 거절은 2026-11-12부터 공용 `SERVER_BUSY`다. 아래 표와 enum 은 같은 값을 말하고, `WebAdvancedErrorMappingTest` 가 실제 응답으로 확인한다. | 예외 | `error.code` | status | category | retryable | 응답에 더 실리는 것 | 클라이언트 행동 | | --- | --- | --- | --- | --- | --- | --- | -| `AdmissionRefusedException` | `ADMISSION_REFUSED` | 503 | `TRANSIENT_DEPENDENCY` | true | `Retry-After: 1` | 잠시 뒤 같은 요청 | +| `AdmissionRefusedException` | `SERVER_BUSY`(공용 `OperationalError`) | 503 | `TRANSIENT_DEPENDENCY` | true | `Retry-After: 1` | 잠시 뒤 같은 요청. 서버 포화는 모든 전송이 이 코드 하나다 | | `JsonPatchTestFailedException` | `PATCH_TEST_FAILED` | 409 | `CONFLICT` | false | `details.pointer`(서버 값은 없음) | 다시 읽고 새 patch | | `JsonPatchRejectedException` | `PATCH_REJECTED` | 400 | `VALIDATION` | false | — | patch 문서를 고친다. JSON이 아닌 본문, 빈 문서, 한도·깊이 초과, 연산 모양 오류, 결과가 자원 타입으로 읽히지 않는 patch가 모두 이 코드다 | | `PatchAuthorizationException` | `PATCH_PATH_FORBIDDEN` | 403 | `AUTHZ` | false | `details.refusedPaths` | 해당 경로를 빼거나 권한 확보 | diff --git a/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/error/WebAdvancedError.java b/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/error/WebAdvancedError.java index 7729b0f0..d6ef5439 100644 --- a/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/error/WebAdvancedError.java +++ b/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/error/WebAdvancedError.java @@ -11,16 +11,14 @@ import dev.caskeleton.shared.error.Category; * leaf README's "오류 응답" section; it states the same values. * *

Before this existed every one of these reached the client as {@code 500 INTERNAL_ERROR} with - * an ERROR log line, including an admission refusal the profile documents as a 503 (review - * 2026-11-01 02-07). + * an ERROR log line (review 2026-11-01 02-07). The admission refusal it used to name is now the + * shared {@code OperationalError.SERVER_BUSY} (review 2026-11-12). * *

Like {@code RateLimitTransportError}, these are not in {@code * docs/registries/error-codes.yaml}, which is the {@code OperationalError} SSOT. */ public enum WebAdvancedError implements ApiErrorCode, RetryAfterHint { - /** The virtual-thread admission limit was full. A refusal in a millisecond; come back shortly. */ - ADMISSION_REFUSED(Category.TRANSIENT_DEPENDENCY, 503, true, 1), /** A JSON Patch {@code test} did not hold; nothing was applied. Re-read, then retry. */ PATCH_TEST_FAILED(Category.CONFLICT, 409, false, 0), /** The patch document is malformed or over a limit; nothing was applied. */ diff --git a/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/mvc/MvcStreamWriter.java b/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/mvc/MvcStreamWriter.java index 49092b59..d0634a5e 100644 --- a/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/mvc/MvcStreamWriter.java +++ b/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/mvc/MvcStreamWriter.java @@ -29,8 +29,8 @@ import java.util.Objects; * or a rewind it would apply. * *

What this writer does not do: it writes synchronously from an iterator on the request thread, - * so it sends no heartbeat while the source is blocked, enforces no idle timeout or maximum age, and - * keeps no buffer (a slow consumer blocks the source instead). Those bounds are in {@code + * so it sends no heartbeat while the source is blocked, enforces no idle timeout or maximum age, + * and keeps no buffer (a slow consumer blocks the source instead). Those bounds are in {@code * WebStreamPolicy} and the drain in {@code WebStreamDrainCoordinator} for a service that runs its * streams through {@code WebStreamRegistry}; {@code docs/web/streaming-contract.md} says which. * diff --git a/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/mvc/VirtualThreadAdmissionFilter.java b/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/mvc/VirtualThreadAdmissionFilter.java index ee5c9c17..9a4c6914 100644 --- a/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/mvc/VirtualThreadAdmissionFilter.java +++ b/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/mvc/VirtualThreadAdmissionFilter.java @@ -4,7 +4,7 @@ import dev.caskeleton.adapter.inbound.web.advanced.virtualthread.AdmissionRefuse import dev.caskeleton.adapter.inbound.web.advanced.virtualthread.VirtualThreadAdmissionGuard; import dev.caskeleton.adapter.inbound.web.error.EnvelopeResponseWriter; import dev.caskeleton.adapter.inbound.web.http.ApiHeaders; -import dev.caskeleton.adapter.inbound.web.observability.RetryAfterHint; +import dev.caskeleton.adapter.inbound.web.observability.RetryAfterAdvisor; import jakarta.servlet.AsyncEvent; import jakarta.servlet.AsyncListener; import jakarta.servlet.FilterChain; @@ -26,8 +26,8 @@ import tools.jackson.databind.ObjectMapper; * security chain, and refuses before any of it runs. * *

A request that goes async keeps its permit until the async work ends: releasing when this - * thread returns would bound only the hand-off, and the work the limit protects the downstream pools - * from happens after it. + * thread returns would bound only the hand-off, and the work the limit protects the downstream + * pools from happens after it. */ public final class VirtualThreadAdmissionFilter extends OncePerRequestFilter implements Ordered { @@ -37,7 +37,8 @@ public final class VirtualThreadAdmissionFilter extends OncePerRequestFilter imp private final VirtualThreadAdmissionGuard guard; private final ObjectMapper objectMapper; - public VirtualThreadAdmissionFilter(VirtualThreadAdmissionGuard guard, ObjectMapper objectMapper) { + public VirtualThreadAdmissionFilter( + VirtualThreadAdmissionGuard guard, ObjectMapper objectMapper) { this.guard = Objects.requireNonNull(guard, "guard"); this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper"); } @@ -55,9 +56,9 @@ public final class VirtualThreadAdmissionFilter extends OncePerRequestFilter imp try { permit = guard.enter(); } catch (AdmissionRefusedException refused) { - if (refused.errorCode() instanceof RetryAfterHint hint) { - response.setHeader(ApiHeaders.RETRY_AFTER, Integer.toString(hint.retryAfterSeconds())); - } + RetryAfterAdvisor.retryAfterSeconds(refused.errorCode()) + .ifPresent( + seconds -> response.setHeader(ApiHeaders.RETRY_AFTER, Integer.toString(seconds))); EnvelopeResponseWriter.write(response, refused.errorCode(), objectMapper); return; } catch (InterruptedException interrupted) { diff --git a/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/virtualthread/AdmissionRefusedException.java b/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/virtualthread/AdmissionRefusedException.java index 36912a06..aa75b2d0 100644 --- a/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/virtualthread/AdmissionRefusedException.java +++ b/src/adapter/inbound/web-advanced/src/main/java/dev/caskeleton/adapter/inbound/web/advanced/virtualthread/AdmissionRefusedException.java @@ -1,8 +1,8 @@ package dev.caskeleton.adapter.inbound.web.advanced.virtualthread; -import dev.caskeleton.adapter.inbound.web.advanced.error.WebAdvancedError; import dev.caskeleton.shared.error.ApiErrorCarrier; import dev.caskeleton.shared.error.ApiErrorCode; +import dev.caskeleton.shared.error.OperationalError; /** * An arrival was refused because the concurrency limit was full. @@ -27,9 +27,12 @@ public final class AdmissionRefusedException extends RuntimeException implements return limit; } - /** {@link WebAdvancedError#ADMISSION_REFUSED}; see the leaf README for the client contract. */ + /** + * {@link OperationalError#SERVER_BUSY}, the one code every transport answers server saturation + * with; see the leaf README for the client contract. + */ @Override public ApiErrorCode errorCode() { - return WebAdvancedError.ADMISSION_REFUSED; + return OperationalError.SERVER_BUSY; } } diff --git a/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/error/WebAdvancedErrorMappingTest.java b/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/error/WebAdvancedErrorMappingTest.java index 6e810ebf..f2e85e17 100644 --- a/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/error/WebAdvancedErrorMappingTest.java +++ b/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/error/WebAdvancedErrorMappingTest.java @@ -62,7 +62,7 @@ class WebAdvancedErrorMappingTest { static Stream documentedMappings() { return Stream.of( - Arguments.of("admission", 503, "ADMISSION_REFUSED", "TRANSIENT_DEPENDENCY", true), + Arguments.of("admission", 503, "SERVER_BUSY", "TRANSIENT_DEPENDENCY", true), Arguments.of("patch-test", 409, "PATCH_TEST_FAILED", "CONFLICT", false), Arguments.of("patch-rejected", 400, "PATCH_REJECTED", "VALIDATION", false), Arguments.of("patch-forbidden", 403, "PATCH_PATH_FORBIDDEN", "AUTHZ", false), diff --git a/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/mvc/VirtualThreadAdmissionFilterTest.java b/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/mvc/VirtualThreadAdmissionFilterTest.java index 47b1434a..a5c10cd1 100644 --- a/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/mvc/VirtualThreadAdmissionFilterTest.java +++ b/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/mvc/VirtualThreadAdmissionFilterTest.java @@ -12,12 +12,17 @@ import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; import tools.jackson.databind.json.JsonMapper; -/** The request path through the admission guard. */ +/** + * The request path through the admission guard. + * + *

required_test: {@code contract-verification:server-saturation} — a full limit answers the + * shared {@code SERVER_BUSY} 503 with {@code Retry-After}, before any handler runs. + */ @Tag("web-advanced") class VirtualThreadAdmissionFilterTest { @Test - @DisplayName("한도가 차면 요청은 controller에 닿지 않고 503 ADMISSION_REFUSED와 Retry-After를 받는다") + @DisplayName("한도가 차면 요청은 controller에 닿지 않고 503 SERVER_BUSY와 Retry-After를 받는다") void aFullLimitRefusesWith503AndRetryAfter() throws Exception { // given VirtualThreadAdmissionGuard guard = new VirtualThreadAdmissionGuard(1, Duration.ZERO); @@ -38,7 +43,7 @@ class VirtualThreadAdmissionFilterTest { assertThat(reached).as("the guard was a bean nothing called").hasValue(0); assertThat(response.getStatus()).isEqualTo(503); assertThat(response.getHeader("Retry-After")).isEqualTo("1"); - assertThat(response.getContentAsString()).contains("ADMISSION_REFUSED"); + assertThat(response.getContentAsString()).contains("SERVER_BUSY"); assertThat(guard.rejectedCount()).isEqualTo(1); } diff --git a/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/patch/JsonPatchApplierTest.java b/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/patch/JsonPatchApplierTest.java index c50838c0..55b02e60 100644 --- a/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/patch/JsonPatchApplierTest.java +++ b/src/adapter/inbound/web-advanced/src/test/java/dev/caskeleton/adapter/inbound/web/advanced/patch/JsonPatchApplierTest.java @@ -269,8 +269,7 @@ class JsonPatchApplierTest { // when Throwable thrown = - catchThrowable( - () -> applier("/displayName").apply(profile(), document, Profile.class)); + catchThrowable(() -> applier("/displayName").apply(profile(), document, Profile.class)); // then assertThat(thrown).isInstanceOf(JsonPatchRejectedException.class); @@ -285,8 +284,7 @@ class JsonPatchApplierTest { // when Throwable thrown = - catchThrowable( - () -> applier("/version").apply(profile(), document, Profile.class)); + catchThrowable(() -> applier("/version").apply(profile(), document, Profile.class)); // then assertThat(thrown).isInstanceOf(JsonPatchRejectedException.class); diff --git a/src/adapter/inbound/web-fileserver/src/main/java/dev/caskeleton/adapter/inbound/web/fileserver/config/BlockingTransferExecutor.java b/src/adapter/inbound/web-fileserver/src/main/java/dev/caskeleton/adapter/inbound/web/fileserver/config/BlockingTransferExecutor.java index 8b3516b3..17eb4410 100644 --- a/src/adapter/inbound/web-fileserver/src/main/java/dev/caskeleton/adapter/inbound/web/fileserver/config/BlockingTransferExecutor.java +++ b/src/adapter/inbound/web-fileserver/src/main/java/dev/caskeleton/adapter/inbound/web/fileserver/config/BlockingTransferExecutor.java @@ -58,7 +58,7 @@ public final class BlockingTransferExecutor { throw new FileserverException( "transfer pool is saturated", rejected, - FileserverFailureContext.of(FileserverErrorCode.TRANSFER_ADMISSION_REJECTED, true)); + FileserverFailureContext.of(FileserverErrorCode.SERVER_BUSY, true)); } try { return future.get(awaitSeconds, TimeUnit.SECONDS); diff --git a/src/adapter/inbound/web-fileserver/src/main/java/dev/caskeleton/adapter/inbound/web/fileserver/problem/FileserverProblemHeaders.java b/src/adapter/inbound/web-fileserver/src/main/java/dev/caskeleton/adapter/inbound/web/fileserver/problem/FileserverProblemHeaders.java index bbc0d182..1a6ad304 100644 --- a/src/adapter/inbound/web-fileserver/src/main/java/dev/caskeleton/adapter/inbound/web/fileserver/problem/FileserverProblemHeaders.java +++ b/src/adapter/inbound/web-fileserver/src/main/java/dev/caskeleton/adapter/inbound/web/fileserver/problem/FileserverProblemHeaders.java @@ -30,8 +30,8 @@ public final class FileserverProblemHeaders { private static final Map RETRY_AFTER_SECONDS = Map.of( FileserverErrorCode.TRANSFER_ADMISSION_REJECTED, 1, + FileserverErrorCode.SERVER_BUSY, 1, FileserverErrorCode.STORAGE_UNAVAILABLE, 30, - FileserverErrorCode.ATOMIC_PUBLISH_UNSUPPORTED, 30, FileserverErrorCode.TRANSFER_TIMEOUT, 5, FileserverErrorCode.FILE_NOT_READY, 2, FileserverErrorCode.CONCURRENT_MODIFICATION, 1); @@ -47,8 +47,10 @@ public final class FileserverProblemHeaders { Map headers = new LinkedHashMap<>(); Integer retryAfter = RETRY_AFTER_SECONDS.get(failure.code()); // Only a failure the server itself called retryable advertises a retry: telling a client to - // come back after a permanent rejection is worse than saying nothing. - if (retryAfter != null && failure.context().retryable()) { + // come back after a permanent rejection is worse than saying nothing. The same narrowing as the + // body (ErrorResponseFactory): the occurrence can only narrow its code, so a code that is never + // retryable never carries the header, whatever the occurrence said (review 2026-11-12). + if (retryAfter != null && failure.code().retryable() && failure.context().retryable()) { headers.put("Retry-After", String.valueOf(retryAfter)); } if (failure instanceof HttpRangeNotSatisfiableException unsatisfiable) { diff --git a/src/adapter/inbound/web-fileserver/src/test/java/dev/caskeleton/adapter/inbound/web/fileserver/config/MvcUploadExecutorSaturationTest.java b/src/adapter/inbound/web-fileserver/src/test/java/dev/caskeleton/adapter/inbound/web/fileserver/config/MvcUploadExecutorSaturationTest.java index 0a65bcca..4ded8b41 100644 --- a/src/adapter/inbound/web-fileserver/src/test/java/dev/caskeleton/adapter/inbound/web/fileserver/config/MvcUploadExecutorSaturationTest.java +++ b/src/adapter/inbound/web-fileserver/src/test/java/dev/caskeleton/adapter/inbound/web/fileserver/config/MvcUploadExecutorSaturationTest.java @@ -23,6 +23,7 @@ class MvcUploadExecutorSaturationTest { BlockingTransferExecutor executor = new BlockingTransferExecutor(pool, bounds.awaitSeconds()); CountDownLatch release = new CountDownLatch(1); AtomicInteger rejections = new AtomicInteger(); + CountDownLatch firstRejection = new CountDownLatch(1); List callers = new ArrayList<>(); for (int index = 0; index < 8; index++) { @@ -34,21 +35,25 @@ class MvcUploadExecutorSaturationTest { } catch (FileserverException failure) { // A caller that outlived the budget (TRANSFER_TIMEOUT) is not a saturation // signal; only an admission rejection is. - if (failure.code() == FileserverErrorCode.TRANSFER_ADMISSION_REJECTED) { + if (failure.code() == FileserverErrorCode.SERVER_BUSY) { rejections.incrementAndGet(); + firstRejection.countDown(); } } }); callers.add(caller); caller.start(); } - Thread.sleep(200); + // Held until the pool has refused someone, rather than for a guessed 200 ms: under load the + // callers may not have arrived yet, and then the release lets every one of them through. + boolean refused = firstRejection.await(10, TimeUnit.SECONDS); release.countDown(); for (Thread caller : callers) { caller.join(TimeUnit.SECONDS.toMillis(10)); } pool.shutdown(); + assertThat(refused).as("a pool of 1 with a queue of 1 refused within the wait").isTrue(); assertThat(rejections.get()) .as("a pool of 1 with a queue of 1 cannot absorb 8 concurrent transfers") .isPositive(); @@ -102,8 +107,7 @@ class MvcUploadExecutorSaturationTest { .satisfies( failure -> { FileserverException rejected = (FileserverException) failure; - assertThat(rejected.code()) - .isEqualTo(FileserverErrorCode.TRANSFER_ADMISSION_REJECTED); + assertThat(rejected.code()).isEqualTo(FileserverErrorCode.SERVER_BUSY); assertThat(rejected.context().retryable()).isTrue(); }); } @@ -115,7 +119,10 @@ class MvcUploadExecutorSaturationTest { BlockingTransferExecutor executor = new BlockingTransferExecutor(pool, bounds.awaitSeconds()); assertThatThrownBy(() -> executor.call(() -> await(new CountDownLatch(1)))) - .isInstanceOf(FileserverException.class); + .isInstanceOf(FileserverException.class) + .extracting(failure -> ((FileserverException) failure).code()) + .as("the overrun is answered as the timeout it is, not as some other transfer failure") + .isEqualTo(FileserverErrorCode.TRANSFER_TIMEOUT); pool.shutdown(); } diff --git a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/GlobalExceptionHandler.java b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/GlobalExceptionHandler.java index 22563880..ee70d0fe 100644 --- a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/GlobalExceptionHandler.java +++ b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/GlobalExceptionHandler.java @@ -412,7 +412,7 @@ public class GlobalExceptionHandler extends ResponseEntityExceptionHandler { return null; } } - ApiErrorCode code = HttpStatusErrorCodes.forStatus(statusCode); + ApiErrorCode code = HttpStatusErrorCodes.forException(ex, statusCode); if (code.httpStatus() >= 500) { log.error("spring mvc failure classified as {}", code.code(), ex); spanErrorRecorder.recordException(ex, code.code()); diff --git a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/HttpStatusErrorCodes.java b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/HttpStatusErrorCodes.java index e37d6948..47ae06a8 100644 --- a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/HttpStatusErrorCodes.java +++ b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/error/HttpStatusErrorCodes.java @@ -3,6 +3,7 @@ package dev.caskeleton.adapter.inbound.web.error; import dev.caskeleton.shared.error.ApiErrorCode; import dev.caskeleton.shared.error.OperationalError; import org.springframework.http.HttpStatusCode; +import org.springframework.web.context.request.async.AsyncRequestTimeoutException; /** * The operational code a bare HTTP status stands for, when a status is all that is known. @@ -15,6 +16,17 @@ final class HttpStatusErrorCodes { private HttpStatusErrorCodes() {} + /** + * The code for a Spring MVC exception that carries only a status. An async request timeout is + * {@code REQUEST_TIMEOUT}; anything else is classified by its status. + */ + static ApiErrorCode forException(Exception failure, HttpStatusCode status) { + if (failure instanceof AsyncRequestTimeoutException) { + return WebTransportError.REQUEST_TIMEOUT; + } + return forStatus(status); + } + static ApiErrorCode forStatus(HttpStatusCode status) { return switch (status.value()) { case 401 -> OperationalError.UNAUTHENTICATED; @@ -27,8 +39,10 @@ final class HttpStatusErrorCodes { case 414 -> OperationalError.URI_TOO_LONG; case 415 -> OperationalError.UNSUPPORTED_MEDIA_TYPE; case 429 -> OperationalError.RATE_LIMIT_EXCEEDED; - // Spring answers its async request timeout with 503: transient, not an internal fault. - case 503 -> WebTransportError.REQUEST_TIMEOUT; + // A bare 503 is the server saying it cannot serve now (RFC 9110 15.6.4). An async request + // timeout is also a 503, and forException tells it apart; every other one used to be + // labelled REQUEST_TIMEOUT whatever it was (review 2026-11-12). + case 503 -> OperationalError.SERVER_BUSY; default -> status.is4xxClientError() ? OperationalError.BAD_PARAMETER diff --git a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/observability/RetryAfterAdvisor.java b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/observability/RetryAfterAdvisor.java index cdc4aed4..f1a14374 100644 --- a/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/observability/RetryAfterAdvisor.java +++ b/src/adapter/inbound/web/src/main/java/dev/caskeleton/adapter/inbound/web/observability/RetryAfterAdvisor.java @@ -19,6 +19,7 @@ public final class RetryAfterAdvisor { private static final Map RETRY_AFTER_SECONDS = Map.ofEntries( Map.entry("RATE_LIMIT_EXCEEDED", 1), + Map.entry("SERVER_BUSY", 1), // The application's value, which the processing-lease check also reads: the resend // arrives while the first attempt can still finish. docs/web/http-contract.md §2. Map.entry( diff --git a/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/error/EnvelopeErrorControllerTest.java b/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/error/EnvelopeErrorControllerTest.java index 074f8f8d..436a3bb6 100644 --- a/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/error/EnvelopeErrorControllerTest.java +++ b/src/adapter/inbound/web/src/test/java/dev/caskeleton/adapter/inbound/web/error/EnvelopeErrorControllerTest.java @@ -28,6 +28,7 @@ class EnvelopeErrorControllerTest { "401, 401, UNAUTHENTICATED", "404, 404, ROUTE_NOT_FOUND", "429, 429, RATE_LIMIT_EXCEEDED", + "503, 503, SERVER_BUSY", "409, 400, BAD_PARAMETER", "422, 400, BAD_PARAMETER", "500, 500, INTERNAL_ERROR", diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/error/FileserverErrorCode.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/error/FileserverErrorCode.java index 24c64ea0..1fe66959 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/error/FileserverErrorCode.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/error/FileserverErrorCode.java @@ -17,10 +17,10 @@ import dev.caskeleton.shared.error.Category; * servlet transport, the reactive transport and the Nginx delegation path cannot answer the same * failure three different ways. * - *

{@code UNAUTHENTICATED}, {@code PRECONDITION_FAILED} and {@code UNSUPPORTED_MEDIA_TYPE} share - * their name with an {@code OperationalError}; they are kept here so the failure context stays one - * enum, and {@code FileserverExceptionTest} pins that each means exactly what the operational code - * means on the wire (category, status, retryability). + *

{@code UNAUTHENTICATED}, {@code PRECONDITION_FAILED}, {@code UNSUPPORTED_MEDIA_TYPE} and + * {@code SERVER_BUSY} share their name with an {@code OperationalError}; they are kept here so the + * failure context stays one enum, and {@code FileserverExceptionTest} pins that each means exactly + * what the operational code means on the wire (category, status, retryability). */ public enum FileserverErrorCode implements ApiErrorCode { BAD_REQUEST(400, Category.VALIDATION, false), @@ -42,6 +42,7 @@ public enum FileserverErrorCode implements ApiErrorCode { MALWARE_DETECTED(422, Category.DATA_INTEGRITY, false), INVALID_PATH(422, Category.VALIDATION, false), PATH_OUTSIDE_NAMESPACE(422, Category.VALIDATION, false), + // The caller's own transfers fill its scope's permits: the caller's rate, so 429. TRANSFER_ADMISSION_REJECTED(429, Category.RATE_LIMIT, true), // tus checksum extension status: one request's bytes did not match the checksum the client sent // for them. The chunk is rolled back before its offset commits, so the upload stays resumable. @@ -52,6 +53,9 @@ public enum FileserverErrorCode implements ApiErrorCode { AMBIGUOUS_COMPLETION(500, Category.DATA_INTEGRITY, false), ATOMIC_PUBLISH_UNSUPPORTED(503, Category.PERMANENT_DEPENDENCY, false), STORAGE_UNAVAILABLE(503, Category.TRANSIENT_DEPENDENCY, true), + // The instance's permits or its transfer pool are full: server saturation, the shared code every + // transport answers it with (decisions 2.3, 2026-11-12). + SERVER_BUSY(503, Category.TRANSIENT_DEPENDENCY, true), TRANSFER_TIMEOUT(504, Category.TRANSIENT_DEPENDENCY, true), STORAGE_FULL(507, Category.TRANSIENT_DEPENDENCY, false); diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/quota/TransferAdmissionController.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/quota/TransferAdmissionController.java index d9969dba..b4c28d67 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/quota/TransferAdmissionController.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/quota/TransferAdmissionController.java @@ -15,9 +15,11 @@ import java.util.concurrent.atomic.AtomicBoolean; * In-process admission control backed by bounded semaphores. * *

The failure vocabulary is deliberately distinct so a client can tell a permanent policy denial - * from a transient one: an exhausted pool is {@code STORAGE_FULL}, and running out of concurrency - * permits — for the caller's scope or for the whole instance — is a retryable admission rejection. - * Stored bytes over a quota ({@code QUOTA_EXCEEDED}) are decided by the quota service, not here. + * from a transient one: an exhausted pool is {@code STORAGE_FULL}; running out of the caller's own + * scope permits is a retryable {@code TRANSFER_ADMISSION_REJECTED} (429, the caller's rate); + * running out of the whole instance's permits is the shared server-saturation {@code SERVER_BUSY} + * (503). Stored bytes over a quota ({@code QUOTA_EXCEEDED}) are decided by the quota service, not + * here. */ public final class TransferAdmissionController { @@ -74,7 +76,7 @@ public final class TransferAdmissionController { scopePermits.release(); throw new FileserverException( "instance upload permits are exhausted", - FileserverFailureContext.of(FileserverErrorCode.TRANSFER_ADMISSION_REJECTED, true)); + FileserverFailureContext.of(FileserverErrorCode.SERVER_BUSY, true)); } return new SemaphorePermit(scopePermits, instanceUploads); } @@ -83,7 +85,7 @@ public final class TransferAdmissionController { if (!directDownloads.tryAcquire()) { throw new FileserverException( "instance direct download permits are exhausted", - FileserverFailureContext.of(FileserverErrorCode.TRANSFER_ADMISSION_REJECTED, true)); + FileserverFailureContext.of(FileserverErrorCode.SERVER_BUSY, true)); } return new SemaphorePermit(directDownloads); } diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/quota/TransferAdmissionControllerTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/quota/TransferAdmissionControllerTest.java index 74a675dc..9c31a867 100644 --- a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/quota/TransferAdmissionControllerTest.java +++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/quota/TransferAdmissionControllerTest.java @@ -119,7 +119,12 @@ class TransferAdmissionControllerTest { assertThatThrownBy(() -> narrow.acquireUpload(scope("tenant-b"), 10)) .isInstanceOf(FileserverException.class) .satisfies( - failure -> assertThat(((FileserverException) failure).context().retryable()).isTrue()); + failure -> { + assertThat(((FileserverException) failure).context().retryable()).isTrue(); + assertThat(((FileserverException) failure).context().code()) + .as("the whole instance is full: server saturation, not the caller's rate") + .isEqualTo(FileserverErrorCode.SERVER_BUSY); + }); held.close(); TransferPermit afterRelease = narrow.acquireUpload(scope("tenant-b"), 10); @@ -197,7 +202,9 @@ class TransferAdmissionControllerTest { TransferPermit download = narrow.acquireDirectDownload(scope("tenant-a")); assertThatThrownBy(() -> narrow.acquireDirectDownload(scope("tenant-a"))) - .isInstanceOf(FileserverException.class); + .isInstanceOf(FileserverException.class) + .extracting(failure -> ((FileserverException) failure).context().code()) + .isEqualTo(FileserverErrorCode.SERVER_BUSY); assertThat(narrow.acquireUpload(scope("tenant-a"), 10).isHeld()).isTrue(); download.close(); diff --git a/src/shared-contract/src/main/java/dev/caskeleton/shared/error/OperationalError.java b/src/shared-contract/src/main/java/dev/caskeleton/shared/error/OperationalError.java index 37d5ed02..70270688 100644 --- a/src/shared-contract/src/main/java/dev/caskeleton/shared/error/OperationalError.java +++ b/src/shared-contract/src/main/java/dev/caskeleton/shared/error/OperationalError.java @@ -44,6 +44,12 @@ public enum OperationalError implements ApiErrorCode { ROUTE_NOT_FOUND(Category.NOT_FOUND, 404, false), // rate-limit / idempotency RATE_LIMIT_EXCEEDED(Category.RATE_LIMIT, 429, true), + /** + * This instance has no capacity for another request right now: an admission limit, a full work + * pool. Every transport answers server saturation with this one code; {@link + * #RATE_LIMIT_EXCEEDED} is the caller's own rate (decisions 2.3, 2026-11-12). + */ + SERVER_BUSY(Category.TRANSIENT_DEPENDENCY, 503, true), IDEMPOTENT_IN_FLIGHT(Category.CONFLICT, 409, true), IDEMPOTENT_REQUEST_MISMATCH(Category.VALIDATION, 422, false), IDEMPOTENCY_UNAVAILABLE(Category.TRANSIENT_DEPENDENCY, 503, true),