From e7235bacf2b490d633a4bda970292023fd47616f Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 13 Nov 2026 18:43:16 +0900 Subject: [PATCH] =?UTF-8?q?fix(objectstorage)!:=20=ED=8F=AC=ED=8A=B8=20?= =?UTF-8?q?=EB=B0=96=EC=9C=BC=EB=A1=9C=20adapter=20=EC=98=88=EC=99=B8?= =?UTF-8?q?=EB=A5=BC=20=EB=82=B4=EB=B3=B4=EB=82=B4=EC=A7=80=20=EC=95=8A?= =?UTF-8?q?=EA=B3=A0=20=ED=8F=AC=ED=8A=B8=20=EC=8B=A4=ED=8C=A8=20=ED=95=98?= =?UTF-8?q?=EB=82=98=EB=A1=9C=20=EC=95=8C=EB=A6=B0=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit objectstorage 포트가 저장소 실패, 내용 불일치, producer·consumer 실패를 adapter의 ObjectStorageProviderException·ObjectControlConflictException으로 그대로 내보냈다. 유스케이스는 그 타입을 import할 수 없어 실패를 가르지 못했고, 포트 계약도 그 경우를 "어떤 unchecked 예외든"으로 비워 뒀다. 저장소 장애·손상 일부는 요청 거절 타입(IAE·ISE)으로 나가 거절로 읽혔다. - 포트 소유 예외 ObjectStorageFailureException과 분류 error()를 둔다. ObjectOperationError에 CONTENT_MISMATCH·CONTENT_PRODUCTION_FAILED· CONTENT_CONSUMPTION_FAILED를 더하고 포트 Javadoc이 @throws로 선언한다. - routing adapter의 포트 메서드와 handoff 포트(RoutingObjectPublicationHandoffAdapter)가 모두 ObjectStoragePortBoundary 한 곳을 지나 거기서만 바뀐다. ArchUnit이 지킨다. - ContentCallbackFailures가 sink·source가 낸 실패, 요청 취소, callback 자신의 실패를 가른다. sink·source는 callback에 선언된 chunk 예외만 던진다. - 저장소 장애·손상은 거절 타입이 아니다: 제어 store IO, 저장 디렉터리의 symlink, 결과 모르는 create, 기록과 다른 저장 객체, 제어 레코드 누락. resolveCreate는 결과를 모르는 실패만 INDETERMINATE로 답한다. - 요청 call budget 소진은 CANCELLED, provider·SDK 시한 초과는 PROVIDER_UNAVAILABLE, 설정 거절(권한·owner·region·보존)은 UNSUPPORTED_CAPABILITY다. - S3는 SDK가 감싼 실패 대신 bridge가 기록한 호출자 쪽 분류를 쓴다(PUT·GET). 정확히 다 읽은 consumer는 EOF를 읽지 않아도 된다(local·fake와 같게). - 겹친 managed 게시·finalize에서 진 호출은 상대가 게시를 끝냈으면 REPLAYED다. - 게시 중(pointer만 있고 lifecycle 없음)인 reference는 inspect가 빈 값이다. - 중단한 stage의 scan 전송을 adapter도 거절한다(fake와 같게). - 포트 계약·provider 계약(MinIO lane 포함)이 실패 분류까지 단언한다. BREAKING CHANGE: 포트 호출의 실패가 ObjectStorageFailureException이다. ObjectOperationError에 값 셋이 늘었다. ObjectPublicationHandoffService는 포트를 구현하지 않는다. 할 일은 docs/upgrade/2026-11-round9-objectstorage.md 5~14행이다. Refs: RR9-03-34 --- docs/testing/TESTING_STRATEGY.md | 8 +- docs/upgrade/2026-11-round9-objectstorage.md | 19 +- src/adapter/outbound/objectstorage/AGENTS.md | 5 +- src/adapter/outbound/objectstorage/README.md | 14 +- .../config/ObjectStorageCapabilityConfig.java | 6 +- .../LocalDevObjectControlStore.java | 16 +- .../filesystem/LocalDevObjectDataStore.java | 6 +- .../LocalDevObjectStorageProvider.java | 7 +- .../filesystem/LocalObjectPathGuard.java | 39 ++- .../filesystem/LocalObjectStreamTransfer.java | 60 +++-- .../ManagedObjectPublicationKernel.java | 43 ++- .../ObjectIntegrityVerificationService.java | 11 +- .../ObjectPublicationHandoffService.java | 18 +- .../kernel/StagedObjectPublicationKernel.java | 58 ++++- .../provider/ContentCallbackFailures.java | 114 ++++++++ .../ObjectStorageProviderException.java | 17 +- .../routing/ObjectStoragePortBoundary.java | 66 +++++ .../routing/RoutingObjectMutationAdapter.java | 32 +++ ...outingObjectPublicationHandoffAdapter.java | 41 +++ .../routing/RoutingObjectReadAdapter.java | 21 +- .../RoutingObjectScanMaintenanceAdapter.java | 37 ++- .../s3/S3AsyncRequestBodyBridge.java | 56 ++-- .../s3/S3AsyncResponseBodyBridge.java | 65 +++-- .../objectstorage/s3/S3ChecksumPolicy.java | 2 +- .../s3/S3ConditionalObjectControlStore.java | 4 +- .../s3/S3ManagedObjectProvider.java | 125 ++++++++- .../ObjectStorageArchitectureTest.java | 58 +++++ .../LocalDevCorruptStoredDataTest.java | 101 ++++++++ .../contract/LocalDevObjectStorage.java | 7 +- .../LocalDevObjectStorageConfinementTest.java | 36 ++- .../LocalDevObjectStorageRecoveryTest.java | 2 +- .../LocalObjectStreamTransferFailureTest.java | 119 +++++++++ .../kernel/ManagedPublicationOutcomeTest.java | 71 ++++- .../StagedObjectPublicationLifecycleTest.java | 76 +++++- .../ObjectStoragePortBoundaryTest.java | 136 ++++++++++ .../s3/S3AsyncRequestBodyBridgeTest.java | 104 ++++++++ .../s3/S3AsyncResponseBodyBridgeTest.java | 116 ++++++++- .../s3/S3ChecksumPolicyTest.java | 39 +++ .../s3/S3ManagedObjectProviderTest.java | 244 ++++++++++++++++++ .../ObjectStorageProviderContract.java | 136 +++++++++- src/application/objectstorage/AGENTS.md | 8 + src/application/objectstorage/README.md | 2 +- .../content/ObjectChunkSink.java | 8 +- .../content/ObjectChunkSource.java | 9 +- .../model/ObjectOperationError.java | 16 +- .../model/ObjectStorageFailureException.java | 63 +++++ .../port/ManagedObjectPublicationPort.java | 13 +- .../port/ObjectScanMaintenancePort.java | 9 +- .../port/ObjectTransferPort.java | 6 +- .../port/StagedObjectPublicationPort.java | 6 +- .../testkit/InMemoryObjectStorage.java | 98 ++++++- .../ManagedObjectStoragePortContract.java | 183 ++++++++++++- .../ObjectScanMaintenancePortContract.java | 26 +- .../ObjectStoragePortContractSupport.java | 117 ++++++++- .../StagedObjectPublicationPortContract.java | 32 ++- 55 files changed, 2539 insertions(+), 192 deletions(-) create mode 100644 src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/provider/ContentCallbackFailures.java create mode 100644 src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/ObjectStoragePortBoundary.java create mode 100644 src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectPublicationHandoffAdapter.java create mode 100644 src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/contract/LocalDevCorruptStoredDataTest.java create mode 100644 src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalObjectStreamTransferFailureTest.java create mode 100644 src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/routing/ObjectStoragePortBoundaryTest.java create mode 100644 src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ChecksumPolicyTest.java create mode 100644 src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/model/ObjectStorageFailureException.java diff --git a/docs/testing/TESTING_STRATEGY.md b/docs/testing/TESTING_STRATEGY.md index 1eb8c081..5aeab809 100644 --- a/docs/testing/TESTING_STRATEGY.md +++ b/docs/testing/TESTING_STRATEGY.md @@ -376,7 +376,8 @@ testFixtures(`dev.caskeleton.application.fileserver.testkit`)에 있고, 실제 포트 8개. 계약 셋과 fake는 `application-objectstorage` testFixtures(`dev.caskeleton.application.objectstorage.testkit`)에 있다. 실제 구현 쪽은 adapter `test`의 `contract` 패키지가 local-dev provider 위에서 돌린다. 계약은 각 포트 -Javadoc이 선언한 결과와 예외 타입만 단언한다. 예외 메시지는 구현의 것이라 단언하지 않는다. +Javadoc이 선언한 결과와 예외 타입, 그리고 `ObjectStorageFailureException`이면 그 분류(`error()`)만 단언한다. 예외 +메시지는 구현의 것이라 단언하지 않는다. | 포트 | 계약 | fake | 상속하는 실제 구현 테스트 | | --- | --- | --- | --- | @@ -386,7 +387,10 @@ Javadoc이 선언한 결과와 예외 타입만 단언한다. 예외 메시지 - 실제 벤더: S3 provider는 readiness registry(`docs/registries/object-storage-readiness.yaml`)에서 아직 배포 대상이 아니다. MinIO에서는 provider 계약(`ObjectStorageProviderContract`, `objectStorageMinioContractTest`)이 - 돈다. 포트 계약을 MinIO로 묶는 일은 S3를 배포 대상으로 올릴 때 같이 한다. + 돈다. 그 계약은 producer·consumer 자신의 실패, 내용 불일치, 취소, 범위를 다 읽지 않은 consumer의 실패 + 종류(`ObjectStorageProviderException.Failure`)를 보므로, 그 분류가 S3 경로에서도 같은지 MinIO에서 확인된다. + 손상·설정 거절 분류는 S3 단위 시험(SDK mock)이 본다. 포트 계약을 MinIO로 묶는 일은 S3를 배포 대상으로 올릴 때 + 같이 한다. - 소비자 없는 fake(9.2, 결정 2026-11-03)의 예외: 이 fake의 소비자는 객체 저장소를 쓰는 서비스의 유스케이스 시험이다. 템플릿 안에는 그런 유스케이스가 없지만 capability 기준 5(`docs/manual/10-capabilities.md`)가 fake를 요구한다(round9 리뷰 6절 ⑧). diff --git a/docs/upgrade/2026-11-round9-objectstorage.md b/docs/upgrade/2026-11-round9-objectstorage.md index 284c3e0a..452c15a7 100644 --- a/docs/upgrade/2026-11-round9-objectstorage.md +++ b/docs/upgrade/2026-11-round9-objectstorage.md @@ -1,7 +1,8 @@ # 업그레이드 안내 — 2026-11 objectstorage 포트 계약 - 범위: `application-objectstorage` 포트의 결과와 거절을 포트 Javadoc에 적고, fake와 local-dev adapter가 같은 계약 - 시험을 통과하게 맞춘 변경. 결과가 바뀐 호출이 셋 있다. 아래 표가 그 목록이다. + 시험을 통과하게 맞춘 변경. 그리고 저장소·producer·consumer 실패를 포트 소유 예외 하나로 바꾼 변경. 결과나 예외가 + 바뀐 호출이 있다. 아래 표가 그 목록이다. - 근거: 2026-11-12 전체 리뷰(`docs/reviews/2026-11-12-round9-review/`)의 objectstorage 포트 계약 발견 ## 1. 누가 읽나 @@ -17,3 +18,19 @@ | 2 | 중단한 stage를 다시 검증하지 않는다. 이제 `verifyIntegrity`가 `IllegalStateException`으로 거절한다(전에는 검증 뒤 중단한 stage를 다시 검증하면 `REPLAYED`) | 그 호출이 예외로 끝난다 | `StagedObjectPublicationPort`를 쓰는 서비스 | | 3 | 불변 `Set`(`Set.of`)으로 capability를 넘겨도 된다. 전에는 요청을 만들 때 `NullPointerException`이 났다 | — 고치기만 한 변경 | 게시·stage 요청을 만드는 서비스 | | 4 | 유스케이스 시험에서 포트 대역이 필요하면 `testImplementation testFixtures(project(':application-objectstorage'))`로 `InMemoryObjectStorage`와 포트 계약을 쓴다 | — 더하기만 한 변경 | 객체 저장소를 쓰는 유스케이스를 시험하는 서비스 | +| 5 | 포트 호출을 감싼 `catch`를 `ObjectStorageFailureException`(`dev.caskeleton.application.objectstorage.model`)으로 바꾸고 `error()`로 가른다. 전에는 저장소 실패, 내용 불일치, producer·consumer 실패가 adapter의 `ObjectStorageProviderException`이나 `ObjectControlConflictException`, 또는 그냥 `RuntimeException`으로 나왔다. 저장소 장애·손상 일부(제어 레코드 IO 실패, 증거 불일치, 저장 디렉터리의 symlink, 결과를 모르는 create)는 `IllegalStateException`·`IllegalArgumentException`으로 나와 거절로 읽혔다. 이제 그것도 이 예외다 | adapter 타입을 잡던 코드는 더 잡지 못한다(application 코드가 adapter 타입을 import했다면 원래 계층 규칙 위반이다). `RuntimeException`으로만 다루던 코드는 그대로 돌지만 실패를 가르지 못한다 | 객체 저장소 포트를 쓰는 서비스 | +| 6 | `ObjectOperationError`를 `default` 없는 `switch` 식(`->`)이나 pattern `switch`로 다루는 코드에 `CONTENT_MISMATCH`·`CONTENT_PRODUCTION_FAILED`·`CONTENT_CONSUMPTION_FAILED`를 더한다. receipt는 새 값을 싣지 않는다 | 그 `switch`의 컴파일이 실패한다(옛 `switch` 문은 컴파일되지만 새 값을 놓친다) | 그 enum을 `switch`로 다루는 서비스 | +| 7 | 같은 key로 겹친 managed 게시나 finalize에서 진 호출을 다시 확인한다. 다른 호출이 이미 게시를 끝냈으면 이제 `REPLAYED`다(전에는 adapter 충돌 예외). 아직 끝내지 않았으면 `CONFLICT` 분류의 `ObjectStorageFailureException`이고 같은 요청으로 다시 하면 된다 | 충돌 예외를 기대하던 재시도 코드가 `REPLAYED`를 받는다 | 같은 operation key로 동시에 게시하는 서비스 | +| 8 | 중단한 stage를 scanner에게 넘기지 않는다. 이제 `transferForScan`이 `IllegalStateException`으로 거절한다(전에는 abort가 지운 데이터를 읽으려다 adapter의 `ObjectStorageProviderException`이 났다) | 그 거절을 잡던 코드의 예외 타입이 바뀐다 | scan 유지보수 포트를 쓰는 scanner 조립 | +| 9 | `ObjectPublicationHandoffService`를 직접 주입하던 코드는 `ObjectPublicationHandoffPort`를 주입한다. 그 클래스는 이제 포트를 구현하지 않고, 포트 bean은 `RoutingObjectPublicationHandoffAdapter`다 | 주입이 실패한다(그런 bean이 없다) | adapter 클래스를 직접 주입하던 서비스 | +| 10 | 자기 provider를 더한 서비스는 다음 넷을 맞춘다. `ObjectStorageProviderException.Failure`를 `switch`로 다루면 새 두 값을 더한다 | `ObjectStorageProviderContract`의 실패 종류 단언이 실패한다 | provider SPI를 구현한 서비스 | +| | ① producer·consumer 자신의 실패는 `PRODUCER_FAILED`·`CONSUMER_FAILED`다. 범위를 다 읽지 않은 consumer도 `CONSUMER_FAILED`다(`provider.ContentCallbackFailures`가 sink·source 실패와 가른다) | | | +| | ② sink·source는 adapter 예외를 callback에 던지지 않고 `ObjectChunkWriteException`·`ObjectChunkReadException`으로 던진다 | | | +| | ③ 요청의 call budget 소진은 `CANCELLED`다. `DEADLINE_EXCEEDED`는 provider 시한 초과이고 포트에서는 `PROVIDER_UNAVAILABLE`이다 | | | +| | ④ 기록된 증거와 다른 저장 객체를 읽는 것은 `CORRUPT_DATA`다(전에는 `PRECONDITION_FAILED`) | | | +| | ⑤ `resolveCreate`는 결과를 모르는 실패(`Failure.leavesTheOutcomeUnknown()`)만 `INDETERMINATE`로 답하고, 설정 거절·손상·취소는 그대로 던진다 | | | +| | ⑥ 범위를 정확히 다 읽고 EOF 없이 돌아온 consumer는 성공이다. 선언한 바이트를 다 쓰고 스스로 실패한 producer는 `PRODUCER_FAILED`다. 취소를 삼키고 돌아온 callback은 `CANCELLED`다 | | | +| 11 | 게시가 manifest와 pointer까지 쓰고 멈춘 reference를 확인한다. 이제 `inspect`는 빈 값이고 `transfer`는 「게시되지 않음」 `IllegalArgumentException`이다(전에는 둘 다 `IllegalArgumentException`「lifecycle 없음」). 같은 operation key로 다시 게시하면 끝난다 | 그 예외를 잡던 코드가 빈 값을 받는다 | 게시 중 장애를 다루는 서비스 | +| 12 | 권한·bucket owner·region·보존 설정을 provider가 거절하면 `UNSUPPORTED_CAPABILITY`다(다시 해도 소용없다). provider 시한 초과·throttle·IO 실패와 결과를 모르는 실패(`INDETERMINATE`·`UNKNOWN`)만 `PROVIDER_UNAVAILABLE`이다 | 설정 오류를 재시도하던 코드가 멈추지 않는다 | 포트 실패로 재시도를 정하는 서비스 | +| 13 | 취소를 삼키고 정상으로 돌아오는 producer·consumer를 확인한다. 이제 그 호출은 `CANCELLED`로 끝난다(전에는 local과 fake에서 `CONTENT_MISMATCH`, S3에서 `CANCELLED`) | 그 분류로 분기하던 코드가 다른 값을 받는다 | 자기 producer·consumer가 취소를 잡는 서비스 | +| 14 | consumer는 요청 범위를 정확히 다 읽으면 EOF를 읽지 않고 돌아와도 된다(전에는 S3에서만 `CONTENT_CONSUMPTION_FAILED`) | — 풀기만 한 변경 | S3에서 transfer를 쓰는 서비스 | diff --git a/src/adapter/outbound/objectstorage/AGENTS.md b/src/adapter/outbound/objectstorage/AGENTS.md index cfa70497..200767a7 100644 --- a/src/adapter/outbound/objectstorage/AGENTS.md +++ b/src/adapter/outbound/objectstorage/AGENTS.md @@ -13,7 +13,7 @@ ## 책임 -- application-core의 semantic object 게시·읽기·폐기·operation 해석 포트를 provider 중립의 한계 있는 streaming과 +- application-objectstorage의 semantic object 게시·읽기·폐기·operation 해석 포트를 provider 중립의 한계 있는 streaming과 엄격한 durable 제어 레코드 뒤에서 구현한다. - 기본 꺼짐인 `app.object-storage` 설정을, 고른 자원을 만들기 전에 정확하고 바뀌지 않는 provider·destination·route·policy 바인딩으로 컴파일한다. @@ -34,7 +34,8 @@ ## 금지 - inbound 어댑터, 이웃 outbound 어댑터, 영속, app-bootstrap 의존. -- provider 키·경로·locator·SDK 타입·Spring 타입·제어 레코드 타입이 application-core로 새는 것. +- provider 키·경로·locator·SDK 타입·Spring 타입·제어 레코드 타입이 application-objectstorage·application-core로 새는 것. +- 포트 메서드가 `routing.ObjectStoragePortBoundary`를 거치지 않는 것. adapter 예외(provider 실패, 제어 레코드 충돌·손상)는 그 한 곳에서만 포트의 `ObjectStorageFailureException`으로 바뀐다(`ObjectStorageArchitectureTest`가 본다). 저장소 장애나 손상을 요청 거절 타입(`IllegalArgumentException`·`IllegalStateException`)으로 던지지 않는다. 그 타입은 포트가 선언한 거절에만 쓴다. - 기본 provider·destination·namespace, 운영에서 local-dev, capability가 꺼져 있을 때 기동 중 변경. - 보통 라우팅 타입이 scanner 유지보수 포트를 구현하기. - `docs/registries/object-storage-readiness.yaml`의 정확한 행보다 높은 주장. diff --git a/src/adapter/outbound/objectstorage/README.md b/src/adapter/outbound/objectstorage/README.md index b2d2bc63..d3093a49 100644 --- a/src/adapter/outbound/objectstorage/README.md +++ b/src/adapter/outbound/objectstorage/README.md @@ -86,10 +86,16 @@ Spring Boot가 `software.amazon.awssdk:*`를 관리하지 않으니 AWS SDK v2 B ### 포트 계약 -이 leaf의 포트 구현(`RoutingObjectMutationAdapter`, `RoutingObjectReadAdapter`, `ObjectPublicationHandoffService`, -`RoutingObjectScanMaintenanceAdapter`)은 application-objectstorage testFixtures의 포트 계약 셋을 local-dev -provider 위에서 상속한다(`contract/LocalDev*ContractTest`). in-memory fake도 같은 계약을 상속한다. 포트 동작을 -바꾸면 두 쪽이 함께 통과해야 한다. +이 leaf의 포트 구현은 `routing` 패키지의 넷이다(`RoutingObjectMutationAdapter`, `RoutingObjectReadAdapter`, +`RoutingObjectPublicationHandoffAdapter`, `RoutingObjectScanMaintenanceAdapter`). 이들은 application-objectstorage +testFixtures의 포트 계약 셋을 local-dev provider 위에서 상속한다(`contract/LocalDev*ContractTest`). in-memory fake도 +같은 계약을 상속한다. 포트 동작을 바꾸면 두 쪽이 함께 통과해야 한다. + +포트 메서드는 모두 `ObjectStoragePortBoundary`를 지난다. adapter 예외는 이 한 곳에서 포트의 +`ObjectStorageFailureException`으로 바뀐다. 바뀌는 것은 provider 실패(`ObjectStorageProviderException`)와 제어 +레코드 충돌·손상이고, 분류표는 `ObjectStoragePortBoundaryTest`다. 요청 거절(`IllegalArgumentException`· +`IllegalStateException`)은 그대로 나간다. producer·consumer 자신의 실패는 sink·source가 낸 실패와 구분한다 +(`provider.ContentCallbackFailures`). 이 구분은 그 예외를 원인으로 싣고 `PRODUCER_FAILED`·`CONSUMER_FAILED`가 된다. ### 새 provider를 위한 provider 계약 diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/config/ObjectStorageCapabilityConfig.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/config/ObjectStorageCapabilityConfig.java index 3738c9ad..9b056705 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/config/ObjectStorageCapabilityConfig.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/config/ObjectStorageCapabilityConfig.java @@ -3,6 +3,7 @@ package dev.caskeleton.adapter.outbound.objectstorage.config; import dev.caskeleton.adapter.outbound.objectstorage.filesystem.FilesystemLocalDevProviderContribution; import dev.caskeleton.adapter.outbound.objectstorage.kernel.ObjectPublicationHandoffService; import dev.caskeleton.adapter.outbound.objectstorage.routing.RoutingObjectMutationAdapter; +import dev.caskeleton.adapter.outbound.objectstorage.routing.RoutingObjectPublicationHandoffAdapter; import dev.caskeleton.adapter.outbound.objectstorage.routing.RoutingObjectReadAdapter; import dev.caskeleton.adapter.outbound.objectstorage.s3.S3ObjectStorageProviderContribution; import dev.caskeleton.application.secret.SecretSourcePort; @@ -127,10 +128,11 @@ public class ObjectStorageCapabilityConfig { name = "enabled", havingValue = "true", matchIfMissing = true) - public ObjectPublicationHandoffService objectPublicationHandoffService( + public RoutingObjectPublicationHandoffAdapter routingObjectPublicationHandoffAdapter( ObjectStorageCapabilityAssembler.AssembledCapability capability, ObjectProvider clocks) { - return new ObjectPublicationHandoffService(capability, clock(clocks)); + return new RoutingObjectPublicationHandoffAdapter( + new ObjectPublicationHandoffService(capability, clock(clocks))); } @Bean diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectControlStore.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectControlStore.java index 27d86964..8696e092 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectControlStore.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectControlStore.java @@ -6,6 +6,7 @@ import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlMutati import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlRecord; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlStore; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlVersion; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; import java.io.IOException; import java.nio.channels.Channels; import java.nio.channels.FileChannel; @@ -45,7 +46,10 @@ public final class LocalDevObjectControlStore implements ObjectControlStore { return Optional.of( new StoredControlRecord(ObjectControlVersion.of(codec.sha256Hex(encoded)), record)); } catch (IOException exception) { - throw new IllegalStateException("local control read failed", exception); + throw new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.IO_FAILURE, + "local control read failed", + exception); } } @@ -68,7 +72,10 @@ public final class LocalDevObjectControlStore implements ObjectControlStore { } catch (FileAlreadyExistsException exception) { throw new ObjectControlConflictException("local control record already exists"); } catch (IOException exception) { - throw new IllegalStateException("local control create failed", exception); + throw new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.IO_FAILURE, + "local control create failed", + exception); } finally { pathGuard.deleteQuietly(temporary); } @@ -94,7 +101,10 @@ public final class LocalDevObjectControlStore implements ObjectControlStore { return new StoredControlRecord( ObjectControlVersion.of(codec.sha256Hex(replacement)), mutation.replacement()); } catch (IOException exception) { - throw new IllegalStateException("local control CAS failed", exception); + throw new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.IO_FAILURE, + "local control CAS failed", + exception); } finally { pathGuard.deleteQuietly(temporary); } diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectDataStore.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectDataStore.java index aa4f6df6..341aefb2 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectDataStore.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectDataStore.java @@ -125,9 +125,11 @@ public final class LocalDevObjectDataStore { stored(operation.canonicalDataKey(), identityOf(channel)); if (!actual.exactVersion().equals(operation.exactVersion()) || !actual.contentIdentity().equals(operation.contentIdentity())) { + // The read carries the manifest's recorded evidence; a different stored object is + // corruption, not a concurrent change (data keys are create-once). throw new ObjectStorageProviderException( - ObjectStorageProviderException.Failure.PRECONDITION_FAILED, - "local read precondition failed"); + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "local stored object does not match its recorded evidence"); } streamTransfer.read(channel, operation, consumer); } catch (ObjectStorageProviderException exception) { diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectStorageProvider.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectStorageProvider.java index e559bd8b..418451c8 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectStorageProvider.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectStorageProvider.java @@ -77,7 +77,12 @@ public final class LocalDevObjectStorageProvider implements ObjectStorageProvide ? Resolution.APPLIED : Resolution.PRECONDITION_FAILED; } catch (ObjectStorageProviderException exception) { - return Resolution.INDETERMINATE; + // Only a failure that leaves the outcome unknown is "cannot resolve": a refused bucket or a + // corrupt data key is an answer the caller must see, not something a retry settles. + if (exception.failure().leavesTheOutcomeUnknown()) { + return Resolution.INDETERMINATE; + } + throw exception; } } diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalObjectPathGuard.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalObjectPathGuard.java index 480d47c2..6cfe84d3 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalObjectPathGuard.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalObjectPathGuard.java @@ -1,5 +1,6 @@ package dev.caskeleton.adapter.outbound.objectstorage.filesystem; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderOperation; import dev.caskeleton.adapter.outbound.support.filesystem.SecureDirectoryWalk; import dev.caskeleton.adapter.outbound.support.filesystem.SecureFileAccess; @@ -38,8 +39,10 @@ import java.util.UUID; * {@code MULTI_NODE_LINEARIZABLE_CAS} unsupported and its control store says so), and exclusivity * between two processes sharing one local root is not claimed. * - *

Refusals surface as {@link IllegalArgumentException}, as the pathname checks did: they are - * permanent rejections of what is on disk, not outages. + *

A key that is not canonical or escapes the root is an {@link IllegalArgumentException}: keys + * come from the adapter's own codecs, so that is a defect, as are root checks at assembly. A link + * or a non-directory found in the tree the store writes is {@code CORRUPT_DATA}: a permanent + * rejection of what is on disk, which the port reports as corrupt evidence, not an outage. */ public final class LocalObjectPathGuard { @@ -98,7 +101,8 @@ public final class LocalObjectPathGuard { * Opens a regular file for reading. * * @return empty when nothing, or something other than a regular file, has that name - * @throws IllegalArgumentException when a component or the leaf is a symbolic link + * @throws ObjectStorageProviderException {@code CORRUPT_DATA} when a component or the leaf is a + * symbolic link */ public Optional openForRead(Path target) throws IOException { try { @@ -111,14 +115,15 @@ public final class LocalObjectPathGuard { /** * True when something already has the name, without following a link. * - * @throws IllegalArgumentException when a component or the leaf is a symbolic link + * @throws ObjectStorageProviderException {@code CORRUPT_DATA} when a component or the leaf is a + * symbolic link */ public boolean exists(Path target) throws IOException { try { BasicFileAttributes attributes = refusing(() -> SecureFileAccess.readAttributes(root, target)); if (attributes.isSymbolicLink()) { - throw new IllegalArgumentException("local object target cannot be a symbolic link"); + throw tampered("local object target cannot be a symbolic link", null); } return true; } catch (NoSuchFileException absent) { @@ -223,8 +228,10 @@ public final class LocalObjectPathGuard { try { createParentsWithoutLinks(target.getParent()); } catch (IOException exception) { - throw new IllegalArgumentException( - "local object path cannot be resolved safely", exception); + throw new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.IO_FAILURE, + "local object path cannot be resolved safely", + exception); } } return target; @@ -242,7 +249,7 @@ public final class LocalObjectPathGuard { for (Path segment : root.relativize(parent)) { current = current.resolve(segment); if (Files.isSymbolicLink(current)) { - throw new IllegalArgumentException("local object parent cannot be a symbolic link"); + throw tampered("local object parent cannot be a symbolic link", null); } if (!Files.exists(current, LinkOption.NOFOLLOW_LINKS)) { try { @@ -251,11 +258,11 @@ public final class LocalObjectPathGuard { } catch (FileAlreadyExistsException race) { if (!Files.isDirectory(current, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(current)) { - throw new IllegalArgumentException("local object parent must be a directory", race); + throw tampered("local object parent must be a directory", race); } } } else if (!Files.isDirectory(current, LinkOption.NOFOLLOW_LINKS)) { - throw new IllegalArgumentException("local object parent must be a directory"); + throw tampered("local object parent must be a directory", null); } } } @@ -277,13 +284,21 @@ public final class LocalObjectPathGuard { try { return operation.run(); } catch (SecureDirectoryWalk.SymbolicComponentException refused) { - throw new IllegalArgumentException( - "local object path cannot contain a symbolic link", refused); + throw tampered("local object path cannot contain a symbolic link", refused); } catch (SecureDirectoryWalk.PathEscapeException refused) { throw new IllegalArgumentException("local object key escapes the configured root", refused); } } + /** + * A link or a non-directory where the store keeps its own tree: the stored layout is not what the + * store wrote, so this is corruption the operator must see, not a refusal of the request. + */ + private static ObjectStorageProviderException tampered(String message, Throwable cause) { + return new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.CORRUPT_DATA, message, cause); + } + @FunctionalInterface private interface DescriptorOperation { T run() throws IOException; diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalObjectStreamTransfer.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalObjectStreamTransfer.java index 2c5eb06f..fdf8dbe6 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalObjectStreamTransfer.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalObjectStreamTransfer.java @@ -1,12 +1,11 @@ package dev.caskeleton.adapter.outbound.objectstorage.filesystem; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ContentCallbackFailures; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProvider; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderOperation; -import dev.caskeleton.application.objectstorage.content.ObjectChunkReadException; import dev.caskeleton.application.objectstorage.content.ObjectChunkSink; import dev.caskeleton.application.objectstorage.content.ObjectChunkSource; -import dev.caskeleton.application.objectstorage.content.ObjectChunkWriteException; import dev.caskeleton.application.objectstorage.content.ObjectContentConsumer; import dev.caskeleton.application.objectstorage.content.ObjectContentConsumptionException; import dev.caskeleton.application.objectstorage.content.ObjectContentProducer; @@ -50,20 +49,31 @@ public final class LocalObjectStreamTransfer { ObjectContentProductionContext context = ObjectContentProductionContext.open( operation.budget(), operation.cancellation(), operation.maximumChunkBytes()); + ContentCallbackFailures failures = ContentCallbackFailures.forProducer(); ObjectChunkSink sink = ObjectChunkSink.scoped( context, (bytes, offset, length) -> { - requireActive(operation.budget(), operation.cancellation().isCancelled()); + try { + requireActive(operation.budget(), operation.cancellation().isCancelled()); + } catch (ObjectStorageProviderException exception) { + throw failures.writeFailure(exception); + } long next; try { next = Math.addExact(written[0], length); } catch (ArithmeticException exception) { - throw new ObjectChunkWriteException("content length overflows", exception); + throw failures.writeFailure( + ObjectStorageProviderException.Failure.CONTENT_MISMATCH, + "content length overflows", + exception); } if (next > operation.maximumObjectBytes() || next > operation.expectedContentIdentity().exactLength()) { - throw new ObjectChunkWriteException("producer exceeded the bounded content size"); + throw failures.writeFailure( + ObjectStorageProviderException.Failure.CONTENT_MISMATCH, + "producer exceeded the bounded content size", + null); } ByteBuffer buffer = ByteBuffer.wrap(bytes, offset, length); try { @@ -74,7 +84,10 @@ public final class LocalObjectStreamTransfer { } } } catch (IOException exception) { - throw new ObjectChunkWriteException("local data write failed", exception); + throw failures.writeFailure( + ObjectStorageProviderException.Failure.IO_FAILURE, + "local data write failed", + exception); } digest.update(bytes, offset, length); written[0] = next; @@ -82,13 +95,11 @@ public final class LocalObjectStreamTransfer { try { producer.produce(context, sink); } catch (ObjectContentProductionException | RuntimeException exception) { - throw new ObjectStorageProviderException( - ObjectStorageProviderException.Failure.IO_FAILURE, - "bounded content production failed", - exception); + throw failures.classify(exception, operation.cancellation().isCancelled()); } finally { context.invalidate(); } + failures.throwRaised(operation.cancellation().isCancelled()); channel.force(true); faultInjector.afterWrite(written[0], temporary); ObjectContentIdentity observed = @@ -120,6 +131,7 @@ public final class LocalObjectStreamTransfer { operation.contentIdentity(), operation.exactVersion(), operation.range()); + ContentCallbackFailures failures = ContentCallbackFailures.forConsumer(); try { channel.position(operation.range().offset()); long[] remaining = {operation.range().length()}; @@ -127,7 +139,11 @@ public final class LocalObjectStreamTransfer { ObjectChunkSource.scoped( context, (destination, offset, length) -> { - requireActive(operation.budget(), operation.cancellation().isCancelled()); + try { + requireActive(operation.budget(), operation.cancellation().isCancelled()); + } catch (ObjectStorageProviderException exception) { + throw failures.readFailure(exception); + } if (remaining[0] == 0) { return ObjectChunkSource.EOF; } @@ -135,27 +151,31 @@ public final class LocalObjectStreamTransfer { try { int count = channel.read(ByteBuffer.wrap(destination, offset, allowed)); if (count < 0) { - throw new ObjectChunkReadException("local data ended before the exact range"); + throw failures.readFailure( + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "local data ended before the exact range", + null); } remaining[0] -= count; return count; } catch (IOException exception) { - throw new ObjectChunkReadException("local data read failed", exception); + throw failures.readFailure( + ObjectStorageProviderException.Failure.IO_FAILURE, + "local data read failed", + exception); } }); try { consumer.consume(context, chunkSource); } catch (ObjectContentConsumptionException | RuntimeException exception) { - throw new ObjectStorageProviderException( - ObjectStorageProviderException.Failure.IO_FAILURE, - "bounded content consumption failed", - exception); + throw failures.classify(exception, operation.cancellation().isCancelled()); } finally { context.invalidate(); } + failures.throwRaised(operation.cancellation().isCancelled()); if (remaining[0] != 0) { throw new ObjectStorageProviderException( - ObjectStorageProviderException.Failure.PRECONDITION_FAILED, + ObjectStorageProviderException.Failure.CONSUMER_FAILED, "consumer did not read the exact requested range"); } } catch (IOException exception) { @@ -172,9 +192,9 @@ public final class LocalObjectStreamTransfer { ObjectStorageProviderException.Failure.CANCELLED, "object transfer is cancelled"); } if (budget.isExpiredAt(System.nanoTime())) { + // The request's own call budget, not a provider timeout: the caller's limit. throw new ObjectStorageProviderException( - ObjectStorageProviderException.Failure.DEADLINE_EXCEEDED, - "object transfer deadline is exceeded"); + ObjectStorageProviderException.Failure.CANCELLED, "object transfer call budget ran out"); } } diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ManagedObjectPublicationKernel.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ManagedObjectPublicationKernel.java index f81b9ff7..8aabe3a5 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ManagedObjectPublicationKernel.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ManagedObjectPublicationKernel.java @@ -7,9 +7,12 @@ import dev.caskeleton.adapter.outbound.objectstorage.codec.ObjectReferenceCodec; import dev.caskeleton.adapter.outbound.objectstorage.codec.ObjectRequestFingerprintCodec; import dev.caskeleton.adapter.outbound.objectstorage.config.CompiledObjectStorageDestination; import dev.caskeleton.adapter.outbound.objectstorage.config.ObjectStorageCapabilityAssembler; +import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlConflictException; +import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlCorruptionException; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlStore; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectOperationRecord; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProvider; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderOperation; import dev.caskeleton.application.idempotency.RequestFingerprint; import dev.caskeleton.application.objectstorage.content.ObjectContentProducer; @@ -45,6 +48,23 @@ public final class ManagedObjectPublicationKernel { public ObjectPublishReceipt publish( ObjectPublishRequest request, ObjectContentProducer producer) { + try { + return publishOnce(request, producer); + } catch (ObjectControlConflictException conflict) { + // Another call under the same key moved the operation first. Once it has published, this + // call is a replay of it: one more pass finds PUBLISHED, so it neither calls the producer nor + // writes, and answers REPLAYED. Before that the conflict stands; the caller retries later. + if (loadOperation(request) + .filter(operation -> operation.stablePhase() == PublicationOperationState.PUBLISHED) + .isEmpty()) { + throw conflict; + } + return publishOnce(request, producer); + } + } + + private ObjectPublishReceipt publishOnce( + ObjectPublishRequest request, ObjectContentProducer producer) { requirePolicy(request, runtime.destination()); String fingerprint = fingerprint(request); ObjectId objectId = ObjectIdentityCodec.objectId(request.operationKey(), fingerprint); @@ -140,7 +160,11 @@ public final class ManagedObjectPublicationKernel { if (operation.stablePhase() != PublicationOperationState.DATA_UPLOAD_IN_PROGRESS) { return provider .inspect(providerOperation.canonicalDataKey()) - .orElseThrow(() -> new IllegalStateException("published operation data is absent")); + .orElseThrow( + () -> + new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "published operation data is absent")); } if (operation.pendingEffect() == null) { operation = @@ -162,9 +186,20 @@ public final class ManagedObjectPublicationKernel { stored = provider .inspect(providerOperation.canonicalDataKey()) - .orElseThrow(() -> new IllegalStateException("resolved local data disappeared")); + .orElseThrow( + () -> + new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "resolved local data disappeared")); + } else if (resolution == ObjectStorageProvider.Resolution.PRECONDITION_FAILED) { + // The data key comes from the request's fingerprint: different bytes under it are corruption. + throw new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "a different object occupies the operation's data key"); } else { - throw new IllegalStateException("provider create cannot be resolved safely"); + throw new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.INDETERMINATE, + "provider create cannot be resolved safely"); } if (operation.effectCertainty() != ObjectEffectCertainty.CONFIRMED) { operation = kernel.confirmEffect(operation, clock.instant()); @@ -201,7 +236,7 @@ public final class ManagedObjectPublicationKernel { private ObjectOperationRecord requireOperation(ObjectPublishRequest request) { return loadOperation(request) - .orElseThrow(() -> new IllegalStateException("operation evidence disappeared")); + .orElseThrow(() -> new ObjectControlCorruptionException("operation evidence disappeared")); } private Optional loadOperation(ObjectPublishRequest request) { diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ObjectIntegrityVerificationService.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ObjectIntegrityVerificationService.java index d2c91ffe..ea7e96aa 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ObjectIntegrityVerificationService.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ObjectIntegrityVerificationService.java @@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.objectstorage.kernel; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectStagedObjectRecord; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProvider; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; import java.util.Objects; /** Exact-version provider evidence verifier for one unpublished stage. */ @@ -14,10 +15,16 @@ public final class ObjectIntegrityVerificationService { ObjectStorageProvider.StoredData stored = provider .inspect(stage.canonicalDataKey()) - .orElseThrow(() -> new IllegalStateException("staged provider object is absent")); + .orElseThrow( + () -> + new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "staged provider object is absent")); if (!stored.exactVersion().equals(stage.exactVersion()) || !stored.contentIdentity().equals(stage.contentIdentity())) { - throw new IllegalStateException("staged provider evidence conflicts with exact identity"); + throw new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "staged provider evidence conflicts with exact identity"); } return stage.verified(now); } diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ObjectPublicationHandoffService.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ObjectPublicationHandoffService.java index 5dd840ae..04171d05 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ObjectPublicationHandoffService.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ObjectPublicationHandoffService.java @@ -3,6 +3,7 @@ package dev.caskeleton.adapter.outbound.objectstorage.kernel; import dev.caskeleton.adapter.outbound.objectstorage.codec.ObjectControlKeyCodec; import dev.caskeleton.adapter.outbound.objectstorage.config.ObjectStorageCapabilityAssembler; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlConflictException; +import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlCorruptionException; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlMutation; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlStore; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectOperationRecord; @@ -12,7 +13,6 @@ import dev.caskeleton.application.objectstorage.model.ObjectHandoffReceipt; import dev.caskeleton.application.objectstorage.model.ObjectMutationOutcome; import dev.caskeleton.application.objectstorage.model.ObjectMutationReceipt; import dev.caskeleton.application.objectstorage.model.ObjectOperationError; -import dev.caskeleton.application.objectstorage.port.ObjectPublicationHandoffPort; import dev.caskeleton.application.objectstorage.request.ObjectAbortAuthorization; import dev.caskeleton.application.objectstorage.request.ObjectHandoffClaimRequest; import dev.caskeleton.application.objectstorage.request.ObjectHandoffReleaseRequest; @@ -21,8 +21,11 @@ import java.time.Clock; import java.time.Instant; import java.util.Objects; -/** Exact stage/version handoff lease with monotonic CAS fences. */ -public final class ObjectPublicationHandoffService implements ObjectPublicationHandoffPort { +/** + * Exact stage/version handoff lease with monotonic CAS fences. The port is {@code + * RoutingObjectPublicationHandoffAdapter}, which reports this kernel's failures as the port's. + */ +public final class ObjectPublicationHandoffService { private final ObjectStorageCapabilityAssembler.AssembledCapability capability; private final Clock clock; @@ -33,7 +36,6 @@ public final class ObjectPublicationHandoffService implements ObjectPublicationH this.clock = Objects.requireNonNull(clock, "clock must be non-null"); } - @Override public ObjectHandoffReceipt claimForPublication(ObjectHandoffClaimRequest request) { RuntimeState state = state(request.operationKey(), request.stageHandle()); if (!state.stage().exactVersion().equals(request.exactVersion())) { @@ -72,7 +74,6 @@ public final class ObjectPublicationHandoffService implements ObjectPublicationH return receipt(claimed, outcome); } - @Override public ObjectHandoffReceipt renewClaim(ObjectHandoffRenewRequest request) { RuntimeState state = state(request.operationKey(), request.stageHandle()); ObjectControlStore.StoredControlRecord stored = loadHandoff(state); @@ -88,7 +89,6 @@ public final class ObjectPublicationHandoffService implements ObjectPublicationH return receipt(renewed, ObjectMutationOutcome.APPLIED); } - @Override public ObjectMutationReceipt releaseClaim(ObjectHandoffReleaseRequest request) { RuntimeState state = state(request.operationKey(), request.stageHandle()); ObjectControlStore.StoredControlRecord stored = loadHandoff(state); @@ -97,7 +97,6 @@ public final class ObjectPublicationHandoffService implements ObjectPublicationH return mutation(request.operationKey(), writeUnlessReplayed(state, stored, released)); } - @Override public ObjectMutationReceipt authorizeAbort(ObjectAbortAuthorization authorization) { RuntimeState state = state(authorization.operationKey(), authorization.stageHandle()); if (!state.stage().exactVersion().equals(authorization.exactVersion())) { @@ -110,7 +109,8 @@ public final class ObjectPublicationHandoffService implements ObjectPublicationH .map(ObjectControlStore.StoredControlRecord::record) .filter(ObjectOperationRecord.class::isInstance) .map(ObjectOperationRecord.class::cast) - .orElseThrow(() -> new IllegalStateException("publication operation is absent")) + .orElseThrow( + () -> new ObjectControlCorruptionException("publication operation is absent")) .stablePhase(); if (phase == PublicationOperationState.REFERENCE_PUBLISH_IN_PROGRESS || phase == PublicationOperationState.PUBLISHED) { @@ -175,7 +175,7 @@ public final class ObjectPublicationHandoffService implements ObjectPublicationH private static ObjectPublicationHandoffRecord handoff( ObjectControlStore.StoredControlRecord stored) { if (!(stored.record() instanceof ObjectPublicationHandoffRecord handoff)) { - throw new IllegalStateException("publication handoff family is corrupt"); + throw new ObjectControlCorruptionException("publication handoff family is corrupt"); } return handoff; } diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/StagedObjectPublicationKernel.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/StagedObjectPublicationKernel.java index 88226aac..9175e0b2 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/StagedObjectPublicationKernel.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/StagedObjectPublicationKernel.java @@ -7,12 +7,14 @@ import dev.caskeleton.adapter.outbound.objectstorage.codec.ObjectIdentityCodec; import dev.caskeleton.adapter.outbound.objectstorage.codec.ObjectRequestFingerprintCodec; import dev.caskeleton.adapter.outbound.objectstorage.config.ObjectStorageCapabilityAssembler; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlConflictException; +import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlCorruptionException; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlMutation; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlStore; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectOperationRecord; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectPublicationHandoffRecord; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectStagedObjectRecord; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProvider; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderOperation; import dev.caskeleton.application.idempotency.RequestFingerprint; import dev.caskeleton.application.objectstorage.content.ObjectContentProducer; @@ -90,12 +92,16 @@ public final class StagedObjectPublicationKernel { .providerFactory() .provider() .inspect(dataKey) - .orElseThrow(() -> new IllegalStateException("uploaded stage data is absent")); + .orElseThrow( + () -> + new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "uploaded stage data is absent")); } else { throw new IllegalStateException("stage operation is not uploadable"); } if (!stored.contentIdentity().equals(request.contentIdentity())) { - throw new IllegalStateException("staged content conflicts with declared identity"); + throw storedContentConflicts(); } ObjectStagedObjectRecord staged = ObjectStagedObjectRecord.staged( @@ -153,7 +159,7 @@ public final class StagedObjectPublicationKernel { } ObjectStorageProvider.StoredData stored = resolveOrCreate(providerOperation, producer); if (!stored.contentIdentity().equals(request.contentIdentity())) { - throw new IllegalStateException("staged content conflicts with declared identity"); + throw storedContentConflicts(); } if (operation.effectCertainty() != ObjectEffectCertainty.CONFIRMED) { operation = operations.confirmEffect(operation, clock.instant()); @@ -229,6 +235,19 @@ public final class StagedObjectPublicationKernel { } public ObjectPublishReceipt finalizePublication(ObjectFinalizeRequest request) { + try { + return finalizeOnce(request); + } catch (ObjectControlConflictException conflict) { + // The same claimant's overlapping finalize published first: answer as its replay. + if (loadOperation(request.operationKey()).stablePhase() + != PublicationOperationState.PUBLISHED) { + throw conflict; + } + return finalizeOnce(request); + } + } + + private ObjectPublishReceipt finalizeOnce(ObjectFinalizeRequest request) { requireDestination(request.operationKey().destination()); ObjectStagedObjectRecord stage = requireStage(storedStage(request.stageHandle()), request.operationKey()); @@ -326,7 +345,10 @@ public final class StagedObjectPublicationKernel { new ObjectOperationKernel(control()) .terminate(operation, PublicationOperationState.ABORTED, clock.instant()); if (!runtime.providerFactory().provider().retire(retirement)) { - throw new IllegalStateException("staged provider object could not be retired"); + // ABORTED is already recorded; a repeated abort retries the retirement. + throw new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.IO_FAILURE, + "staged provider object could not be retired"); } return abortReceipt(request, ObjectMutationOutcome.APPLIED); } @@ -344,12 +366,34 @@ public final class StagedObjectPublicationKernel { case APPLIED -> provider .inspect(operation.canonicalDataKey()) - .orElseThrow(() -> new IllegalStateException("resolved staged data is absent")); + .orElseThrow( + () -> + new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "resolved staged data is absent")); case NOT_APPLIED -> provider.create(operation, producer); - default -> throw new IllegalStateException("staged create remains indeterminate"); + // The data key comes from the request's fingerprint: different bytes under it are corruption. + case PRECONDITION_FAILED -> + throw new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "a different object occupies the stage's data key"); + default -> + throw new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.INDETERMINATE, + "staged create remains indeterminate"); }; } + /** + * The stored object under the data key is not the declared content. The key is derived from the + * request's fingerprint, so this is stored evidence gone wrong, not a different request. + */ + private static ObjectStorageProviderException storedContentConflicts() { + return new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "staged content conflicts with declared identity"); + } + private ObjectControlStore control() { return runtime.providerFactory().controlStore(); } @@ -386,7 +430,7 @@ public final class StagedObjectPublicationKernel { .map(ObjectControlStore.StoredControlRecord::record) .filter(ObjectOperationRecord.class::isInstance) .map(ObjectOperationRecord.class::cast) - .orElseThrow(() -> new IllegalStateException("publication operation is absent")); + .orElseThrow(() -> new ObjectControlCorruptionException("publication operation is absent")); } private static ObjectStagedObjectRecord requireStage( diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/provider/ContentCallbackFailures.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/provider/ContentCallbackFailures.java new file mode 100644 index 00000000..4f4473bf --- /dev/null +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/provider/ContentCallbackFailures.java @@ -0,0 +1,114 @@ +package dev.caskeleton.adapter.outbound.objectstorage.provider; + +import dev.caskeleton.application.objectstorage.content.ObjectChunkReadException; +import dev.caskeleton.application.objectstorage.content.ObjectChunkWriteException; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Tells a producer's or consumer's own failure apart from a failure the adapter raised inside the + * sink or source it lent that callback. + * + *

The sink and source never throw an adapter exception into the callback: what they raise + * reaches it as the declared {@link ObjectChunkWriteException} or {@link ObjectChunkReadException}, + * and its classification is recorded here. + * + *

A callback may catch what the sink or source threw and wrap it in its own exception, or + * swallow it. So the sink and source record what they raised here, and the transfer asks afterwards + * what to report: what the adapter raised first wins, then cancellation of the request, and only + * then the callback's own failure ({@link ObjectStorageProviderException.Failure#PRODUCER_FAILED} + * or {@link ObjectStorageProviderException.Failure#CONSUMER_FAILED}, the callback's exception as + * the cause). + * + *

One instance serves one callback run. The sink or source may run on another thread than the + * one that asks, so the first recorded failure is kept atomically. + */ +public final class ContentCallbackFailures { + + private final ObjectStorageProviderException.Failure callbackFailure; + private final String callbackMessage; + private final AtomicReference raised = new AtomicReference<>(); + + private ContentCallbackFailures( + ObjectStorageProviderException.Failure callbackFailure, String callbackMessage) { + this.callbackFailure = callbackFailure; + this.callbackMessage = callbackMessage; + } + + /** For one run of a producer. */ + public static ContentCallbackFailures forProducer() { + return new ContentCallbackFailures( + ObjectStorageProviderException.Failure.PRODUCER_FAILED, "content producer failed"); + } + + /** For one run of a consumer. */ + public static ContentCallbackFailures forConsumer() { + return new ContentCallbackFailures( + ObjectStorageProviderException.Failure.CONSUMER_FAILED, "content consumer failed"); + } + + /** Records a failure the sink raised and returns the chunk-write exception it throws for it. */ + public ObjectChunkWriteException writeFailure(ObjectStorageProviderException failure) { + return new ObjectChunkWriteException(failure.getMessage(), raise(failure)); + } + + /** Records a failure the source raised and returns the chunk-read exception it throws for it. */ + public ObjectChunkReadException readFailure(ObjectStorageProviderException failure) { + return new ObjectChunkReadException(failure.getMessage(), raise(failure)); + } + + private ObjectStorageProviderException raise(ObjectStorageProviderException failure) { + raised.compareAndSet(null, failure); + return failure; + } + + /** Records a sink failure and returns the chunk-write exception the sink throws for it. */ + public ObjectChunkWriteException writeFailure( + ObjectStorageProviderException.Failure failure, String message, Throwable cause) { + return writeFailure(new ObjectStorageProviderException(failure, message, cause)); + } + + /** Records a source failure and returns the chunk-read exception the source throws for it. */ + public ObjectChunkReadException readFailure( + ObjectStorageProviderException.Failure failure, String message, Throwable cause) { + return readFailure(new ObjectStorageProviderException(failure, message, cause)); + } + + /** + * For a callback that returned normally: throws what the adapter raised inside it, else the + * request's cancellation when it is set. A callback that swallowed either still ends with it. + * + * @param cancelled whether the request's cancellation is set now + */ + public void throwRaised(boolean cancelled) { + ObjectStorageProviderException first = raised.get(); + if (first != null) { + throw first; + } + if (cancelled) { + throw new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.CANCELLED, "content transfer is cancelled"); + } + } + + /** + * What to report for a callback that ended with {@code thrown}. + * + * @param cancelled whether the request's cancellation is set now + */ + public ObjectStorageProviderException classify(Throwable thrown, boolean cancelled) { + ObjectStorageProviderException first = raised.get(); + if (first != null) { + return first; + } + if (thrown instanceof ObjectStorageProviderException providerFailure) { + return providerFailure; + } + if (cancelled) { + return new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.CANCELLED, + "content transfer is cancelled", + thrown); + } + return new ObjectStorageProviderException(callbackFailure, callbackMessage, thrown); + } +} diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/provider/ObjectStorageProviderException.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/provider/ObjectStorageProviderException.java index 82afb574..00e10ba6 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/provider/ObjectStorageProviderException.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/provider/ObjectStorageProviderException.java @@ -38,6 +38,21 @@ public final class ObjectStorageProviderException extends RuntimeException { DEADLINE_EXCEEDED, IO_FAILURE, CORRUPT_DATA, - UNSUPPORTED + UNSUPPORTED, + /** The caller's producer failed by itself; its exception is the cause. */ + PRODUCER_FAILED, + /** The caller's consumer failed by itself (the cause) or did not read the exact range. */ + CONSUMER_FAILED; + + /** + * Whether the store could not say what happened: a transient or unexplained failure. Anything + * else (a refused configuration, corruption, the caller's own cancellation) is an answer. + */ + public boolean leavesTheOutcomeUnknown() { + return switch (this) { + case IO_FAILURE, DEADLINE_EXCEEDED, THROTTLED, UNKNOWN, INDETERMINATE -> true; + default -> false; + }; + } } } diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/ObjectStoragePortBoundary.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/ObjectStoragePortBoundary.java new file mode 100644 index 00000000..039bd04e --- /dev/null +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/ObjectStoragePortBoundary.java @@ -0,0 +1,66 @@ +package dev.caskeleton.adapter.outbound.objectstorage.routing; + +import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlConflictException; +import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlCorruptionException; +import dev.caskeleton.adapter.outbound.objectstorage.control.UnsupportedObjectControlSchemaException; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; +import dev.caskeleton.application.objectstorage.model.ObjectOperationError; +import dev.caskeleton.application.objectstorage.model.ObjectStorageFailureException; +import java.util.function.Supplier; + +/** + * The one place an adapter failure becomes the port's {@link ObjectStorageFailureException}. Every + * method of a routing adapter runs through {@link #call}; refusals ({@link + * IllegalArgumentException}, {@link IllegalStateException}) pass through unchanged. + * + *

The message names only the classification. The adapter exception stays as the cause, for + * diagnosis only, except for a producer's or consumer's own failure, whose cause is the callback's + * exception as the port declares. + */ +final class ObjectStoragePortBoundary { + + private ObjectStoragePortBoundary() {} + + static T call(Supplier operation) { + try { + return operation.get(); + } catch (ObjectStorageProviderException failure) { + throw translate(failure); + } catch (ObjectControlConflictException failure) { + throw new ObjectStorageFailureException( + ObjectOperationError.CONFLICT, "object storage failed: CONFLICT", failure); + } catch (ObjectControlCorruptionException | UnsupportedObjectControlSchemaException failure) { + throw new ObjectStorageFailureException( + ObjectOperationError.CORRUPT_EVIDENCE, + "object storage failed: CORRUPT_EVIDENCE", + failure); + } + } + + static ObjectStorageFailureException translate(ObjectStorageProviderException failure) { + boolean callbackOwn = + failure.failure() == ObjectStorageProviderException.Failure.PRODUCER_FAILED + || failure.failure() == ObjectStorageProviderException.Failure.CONSUMER_FAILED; + Throwable cause = callbackOwn && failure.getCause() != null ? failure.getCause() : failure; + ObjectOperationError error = error(failure.failure()); + return new ObjectStorageFailureException(error, "object storage failed: " + error, cause); + } + + static ObjectOperationError error(ObjectStorageProviderException.Failure failure) { + return switch (failure) { + case CONFLICT, PRECONDITION_FAILED -> ObjectOperationError.CONFLICT; + case NOT_FOUND -> ObjectOperationError.NOT_FOUND; + case CONTENT_MISMATCH -> ObjectOperationError.CONTENT_MISMATCH; + case CORRUPT_DATA -> ObjectOperationError.CORRUPT_EVIDENCE; + // The request's own cancellation or call budget; a provider timeout is DEADLINE_EXCEEDED. + case CANCELLED -> ObjectOperationError.CANCELLED; + case PRODUCER_FAILED -> ObjectOperationError.CONTENT_PRODUCTION_FAILED; + case CONSUMER_FAILED -> ObjectOperationError.CONTENT_CONSUMPTION_FAILED; + // Configuration the provider refuses: repeating the request cannot help. + case UNSUPPORTED, PERMISSION_DENIED, OWNER_MISMATCH, REGION_MISMATCH, RETENTION_BLOCKED -> + ObjectOperationError.UNSUPPORTED_CAPABILITY; + case DEADLINE_EXCEEDED, THROTTLED, INDETERMINATE, UNKNOWN, IO_FAILURE -> + ObjectOperationError.PROVIDER_UNAVAILABLE; + }; + } +} diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectMutationAdapter.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectMutationAdapter.java index 9db65609..feb9d313 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectMutationAdapter.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectMutationAdapter.java @@ -51,28 +51,51 @@ public final class RoutingObjectMutationAdapter @Override public dev.caskeleton.application.objectstorage.model.ObjectStageReceipt stage( ObjectStageRequest request, ObjectContentProducer producer) { + return ObjectStoragePortBoundary.call(() -> stageUnguarded(request, producer)); + } + + private dev.caskeleton.application.objectstorage.model.ObjectStageReceipt stageUnguarded( + ObjectStageRequest request, ObjectContentProducer producer) { return staged(request.operationKey()).stage(request, producer); } @Override public dev.caskeleton.application.objectstorage.model.ObjectVerificationReceipt verifyIntegrity( ObjectVerifyRequest request) { + return ObjectStoragePortBoundary.call(() -> verifyIntegrityUnguarded(request)); + } + + private dev.caskeleton.application.objectstorage.model.ObjectVerificationReceipt + verifyIntegrityUnguarded(ObjectVerifyRequest request) { return staged(request.operationKey()).verifyIntegrity(request); } @Override public ObjectPublishReceipt finalizePublication(ObjectFinalizeRequest request) { + return ObjectStoragePortBoundary.call(() -> finalizePublicationUnguarded(request)); + } + + private ObjectPublishReceipt finalizePublicationUnguarded(ObjectFinalizeRequest request) { return staged(request.operationKey()).finalizePublication(request); } @Override public ObjectMutationReceipt abort(ObjectAbortRequest request) { + return ObjectStoragePortBoundary.call(() -> abortUnguarded(request)); + } + + private ObjectMutationReceipt abortUnguarded(ObjectAbortRequest request) { return staged(request.operationKey()).abort(request); } @Override public ObjectPublishReceipt publish( ObjectPublishRequest request, ObjectContentProducer producer) { + return ObjectStoragePortBoundary.call(() -> publishUnguarded(request, producer)); + } + + private ObjectPublishReceipt publishUnguarded( + ObjectPublishRequest request, ObjectContentProducer producer) { return new ManagedObjectPublicationKernel( capability.destination(request.operationKey().destination()), clock) .publish(request, producer); @@ -80,6 +103,10 @@ public final class RoutingObjectMutationAdapter @Override public ObjectMutationReceipt retire(ObjectRetireRequest request) { + return ObjectStoragePortBoundary.call(() -> retireUnguarded(request)); + } + + private ObjectMutationReceipt retireUnguarded(ObjectRetireRequest request) { String routeToken = ObjectReferenceCodec.routeToken(request.reference()); ObjectStorageCapabilityAssembler.DestinationRuntime runtime = capability.route(routeToken); if (!runtime.destination().destinationId().equals(request.operationKey().destination())) { @@ -143,6 +170,11 @@ public final class RoutingObjectMutationAdapter @Override public ObjectOperationResolution resolve( dev.caskeleton.application.objectstorage.identity.ObjectOperationKey operationKey) { + return ObjectStoragePortBoundary.call(() -> resolveUnguarded(operationKey)); + } + + private ObjectOperationResolution resolveUnguarded( + dev.caskeleton.application.objectstorage.identity.ObjectOperationKey operationKey) { ObjectStorageCapabilityAssembler.DestinationRuntime runtime = capability.destination(operationKey.destination()); String key = ObjectControlKeyCodec.operation(runtime.destination().route(), operationKey); diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectPublicationHandoffAdapter.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectPublicationHandoffAdapter.java new file mode 100644 index 00000000..04ce2de3 --- /dev/null +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectPublicationHandoffAdapter.java @@ -0,0 +1,41 @@ +package dev.caskeleton.adapter.outbound.objectstorage.routing; + +import dev.caskeleton.adapter.outbound.objectstorage.kernel.ObjectPublicationHandoffService; +import dev.caskeleton.application.objectstorage.model.ObjectHandoffReceipt; +import dev.caskeleton.application.objectstorage.model.ObjectMutationReceipt; +import dev.caskeleton.application.objectstorage.port.ObjectPublicationHandoffPort; +import dev.caskeleton.application.objectstorage.request.ObjectAbortAuthorization; +import dev.caskeleton.application.objectstorage.request.ObjectHandoffClaimRequest; +import dev.caskeleton.application.objectstorage.request.ObjectHandoffReleaseRequest; +import dev.caskeleton.application.objectstorage.request.ObjectHandoffRenewRequest; +import java.util.Objects; + +/** The publication handoff port: the kernel's fenced handoff behind the port boundary. */ +public final class RoutingObjectPublicationHandoffAdapter implements ObjectPublicationHandoffPort { + + private final ObjectPublicationHandoffService handoff; + + public RoutingObjectPublicationHandoffAdapter(ObjectPublicationHandoffService handoff) { + this.handoff = Objects.requireNonNull(handoff, "handoff must be non-null"); + } + + @Override + public ObjectHandoffReceipt claimForPublication(ObjectHandoffClaimRequest request) { + return ObjectStoragePortBoundary.call(() -> handoff.claimForPublication(request)); + } + + @Override + public ObjectHandoffReceipt renewClaim(ObjectHandoffRenewRequest request) { + return ObjectStoragePortBoundary.call(() -> handoff.renewClaim(request)); + } + + @Override + public ObjectMutationReceipt releaseClaim(ObjectHandoffReleaseRequest request) { + return ObjectStoragePortBoundary.call(() -> handoff.releaseClaim(request)); + } + + @Override + public ObjectMutationReceipt authorizeAbort(ObjectAbortAuthorization authorization) { + return ObjectStoragePortBoundary.call(() -> handoff.authorizeAbort(authorization)); + } +} diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectReadAdapter.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectReadAdapter.java index 86992e74..991d71e1 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectReadAdapter.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectReadAdapter.java @@ -4,6 +4,7 @@ import dev.caskeleton.adapter.outbound.objectstorage.codec.ObjectControlKeyCodec import dev.caskeleton.adapter.outbound.objectstorage.codec.ObjectDataKeyCodec; import dev.caskeleton.adapter.outbound.objectstorage.codec.ObjectReferenceCodec; import dev.caskeleton.adapter.outbound.objectstorage.config.ObjectStorageCapabilityAssembler; +import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlCorruptionException; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlStore; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectManifestRecord; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectReferencePointerRecord; @@ -35,6 +36,10 @@ public final class RoutingObjectReadAdapter implements ObjectInspectionPort, Obj @Override public Optional inspect(ObjectReference reference) { + return ObjectStoragePortBoundary.call(() -> inspectUnguarded(reference)); + } + + private Optional inspectUnguarded(ObjectReference reference) { Optional candidate = load(reference); if (candidate.isEmpty()) { return Optional.empty(); @@ -67,6 +72,11 @@ public final class RoutingObjectReadAdapter implements ObjectInspectionPort, Obj @Override public ObjectReadReceipt transfer(ObjectReadRequest request, ObjectContentConsumer consumer) { + return ObjectStoragePortBoundary.call(() -> transferUnguarded(request, consumer)); + } + + private ObjectReadReceipt transferUnguarded( + ObjectReadRequest request, ObjectContentConsumer consumer) { LoadedReference loaded = load(request.reference()) .orElseThrow(() -> new IllegalArgumentException("object reference is absent")); @@ -127,7 +137,12 @@ public final class RoutingObjectReadAdapter implements ObjectInspectionPort, Obj .map(ObjectControlStore.StoredControlRecord::record) .filter(ObjectReferenceRecord.class::isInstance) .map(ObjectReferenceRecord.class::cast) - .orElseThrow(() -> new IllegalArgumentException("object lifecycle is absent")); + .orElse(null); + if (lifecycle == null) { + // Publication writes the manifest, then the pointer, then the lifecycle: a pointer without + // its lifecycle is a publication still in progress (a retry finishes it), not published yet. + return Optional.empty(); + } ObjectManifestRecord manifest = control .read( @@ -138,10 +153,10 @@ public final class RoutingObjectReadAdapter implements ObjectInspectionPort, Obj .map(ObjectControlStore.StoredControlRecord::record) .filter(ObjectManifestRecord.class::isInstance) .map(ObjectManifestRecord.class::cast) - .orElseThrow(() -> new IllegalArgumentException("object manifest is absent")); + .orElseThrow(() -> new ObjectControlCorruptionException("object manifest is absent")); if (!pointer.immutableDataVersion().equals(lifecycle.immutableDataVersion()) || !pointer.immutableDataVersion().equals(manifest.immutableDataVersion())) { - throw new IllegalStateException("object reference evidence is corrupt"); + throw new ObjectControlCorruptionException("object reference evidence is corrupt"); } return Optional.of(new LoadedReference(runtime, pointer, lifecycle, manifest)); } diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectScanMaintenanceAdapter.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectScanMaintenanceAdapter.java index 2e974510..2f2c786a 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectScanMaintenanceAdapter.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/routing/RoutingObjectScanMaintenanceAdapter.java @@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.objectstorage.routing; import dev.caskeleton.adapter.outbound.objectstorage.codec.ObjectControlKeyCodec; import dev.caskeleton.adapter.outbound.objectstorage.config.ObjectStorageCapabilityAssembler; +import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlCorruptionException; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlMutation; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlStore; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectOperationRecord; @@ -37,12 +38,22 @@ public final class RoutingObjectScanMaintenanceAdapter implements ObjectScanMain @Override public ObjectScanReadReceipt transferForScan( ObjectScanReadRequest request, ObjectContentConsumer consumer) { + return ObjectStoragePortBoundary.call(() -> transferForScanUnguarded(request, consumer)); + } + + private ObjectScanReadReceipt transferForScanUnguarded( + ObjectScanReadRequest request, ObjectContentConsumer consumer) { Loaded loaded = load(request.operationKey(), request.stageHandle()); requireVersion(loaded.stage(), request.exactVersion()); if (!loaded.stage().integrityVerified() || loaded.stage().scanState() != ObjectScanState.PENDING) { throw new IllegalStateException("staged object is not awaiting scan"); } + // Abort leaves the stage record as it was; the operation says the stage is gone. + if (operation(loaded, request.operationKey()).stablePhase() + == PublicationOperationState.ABORTED) { + throw new IllegalStateException("staged operation was aborted"); + } loaded .runtime() .providerFactory() @@ -71,6 +82,10 @@ public final class RoutingObjectScanMaintenanceAdapter implements ObjectScanMain @Override public ObjectMutationReceipt recordScanVerdict(ObjectScanVerdictRequest request) { + return ObjectStoragePortBoundary.call(() -> recordScanVerdictUnguarded(request)); + } + + private ObjectMutationReceipt recordScanVerdictUnguarded(ObjectScanVerdictRequest request) { Loaded loaded = load(request.operationKey(), request.stageHandle()); requireVersion(loaded.stage(), request.exactVersion()); ObjectStagedObjectRecord verdict = @@ -91,16 +106,7 @@ public final class RoutingObjectScanMaintenanceAdapter implements ObjectScanMain .compareAndSet( loaded.stageKey(), new ObjectControlMutation(loaded.stored().version(), verdict)); } - ObjectOperationRecord operation = - loaded - .store() - .read( - ObjectControlKeyCodec.operation( - loaded.runtime().destination().route(), request.operationKey())) - .map(ObjectControlStore.StoredControlRecord::record) - .filter(ObjectOperationRecord.class::isInstance) - .map(ObjectOperationRecord.class::cast) - .orElseThrow(() -> new IllegalStateException("scan operation is absent")); + ObjectOperationRecord operation = operation(loaded, request.operationKey()); // Also on a replayed verdict: a crash after the stage CAS left the operation SCAN_PENDING. if (operation.stablePhase() == PublicationOperationState.SCAN_PENDING) { ObjectOperationKernel kernel = new ObjectOperationKernel(loaded.store()); @@ -118,6 +124,17 @@ public final class RoutingObjectScanMaintenanceAdapter implements ObjectScanMain request.operationKey(), outcome, ObjectOperationError.NONE, clock.instant()); } + private static ObjectOperationRecord operation( + Loaded loaded, dev.caskeleton.application.objectstorage.identity.ObjectOperationKey key) { + return loaded + .store() + .read(ObjectControlKeyCodec.operation(loaded.runtime().destination().route(), key)) + .map(ObjectControlStore.StoredControlRecord::record) + .filter(ObjectOperationRecord.class::isInstance) + .map(ObjectOperationRecord.class::cast) + .orElseThrow(() -> new ObjectControlCorruptionException("scan operation is absent")); + } + private Loaded load( dev.caskeleton.application.objectstorage.identity.ObjectOperationKey operation, dev.caskeleton.application.objectstorage.identity.ObjectStageHandle handle) { diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncRequestBodyBridge.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncRequestBodyBridge.java index dd301e6c..bfd91227 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncRequestBodyBridge.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncRequestBodyBridge.java @@ -1,10 +1,10 @@ package dev.caskeleton.adapter.outbound.objectstorage.s3; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ContentCallbackFailures; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderOperation; import dev.caskeleton.application.objectstorage.content.CancellationView; import dev.caskeleton.application.objectstorage.content.ObjectChunkSink; -import dev.caskeleton.application.objectstorage.content.ObjectChunkWriteException; import dev.caskeleton.application.objectstorage.content.ObjectContentProducer; import dev.caskeleton.application.objectstorage.content.ObjectContentProductionContext; import dev.caskeleton.application.objectstorage.model.ObjectContentIdentity; @@ -123,28 +123,46 @@ public final class S3AsyncRequestBodyBridge implements AsyncRequestBody { ObjectContentProductionContext context = ObjectContentProductionContext.open( operation.budget(), cancellation, operation.maximumChunkBytes()); + ContentCallbackFailures failures = ContentCallbackFailures.forProducer(); ObjectChunkSink sink = ObjectChunkSink.scoped( context, (bytes, offset, length) -> { - requireActive(); + try { + requireActive(); + } catch (ObjectStorageProviderException exception) { + throw failures.writeFailure(exception); + } long next; try { next = Math.addExact(written[0], length); } catch (ArithmeticException exception) { - throw new ObjectChunkWriteException("S3 content length overflows", exception); + throw failures.writeFailure( + ObjectStorageProviderException.Failure.CONTENT_MISMATCH, + "S3 content length overflows", + exception); } if (next > operation.expectedContentIdentity().exactLength() || next > operation.maximumObjectBytes()) { - throw new ObjectChunkWriteException("S3 producer exceeded its exact bound"); + throw failures.writeFailure( + ObjectStorageProviderException.Failure.CONTENT_MISMATCH, + "S3 producer exceeded its exact bound", + null); + } + try { + awaitDemand(); + } catch (ObjectStorageProviderException exception) { + throw failures.writeFailure(exception); } - awaitDemand(); byte[] copy = Arrays.copyOfRange(bytes, offset, offset + length); maximumBufferedBytesObserved.accumulateAndGet(copy.length, Math::max); try { subscriber.onNext(ByteBuffer.wrap(copy).asReadOnlyBuffer()); } catch (RuntimeException exception) { - throw new ObjectChunkWriteException("S3 subscriber rejected content", exception); + throw failures.writeFailure( + ObjectStorageProviderException.Failure.IO_FAILURE, + "S3 subscriber rejected content", + exception); } digest.update(copy); written[0] = next; @@ -155,6 +173,7 @@ public final class S3AsyncRequestBodyBridge implements AsyncRequestBody { try { requireActive(); producer.produce(context, sink); + failures.throwRaised(cancellation.isCancelled()); requireActive(); ObjectContentIdentity observed = new ObjectContentIdentity( @@ -178,22 +197,15 @@ public final class S3AsyncRequestBodyBridge implements AsyncRequestBody { subscriber.onComplete(); } } catch (Exception exception) { - ObjectStorageProviderException mapped = - exception instanceof ObjectStorageProviderException providerException - ? providerException - : providerFailure( - isCancelled() - ? ObjectStorageProviderException.Failure.CANCELLED - : ObjectStorageProviderException.Failure.IO_FAILURE, - "bounded S3 content production failed", - exception); - fail(mapped); + // The same view the producer saw: the SDK's own cancel once it holds every byte is the + // upload finishing, not a cancellation. + fail(failures.classify(exception, cancellation.isCancelled())); } finally { context.invalidate(); } } - private void awaitDemand() throws ObjectChunkWriteException { + private void awaitDemand() { synchronized (monitor) { while (demand == 0 && !cancelled && !operation.cancellation().isCancelled()) { requireDeadline(); @@ -203,7 +215,10 @@ public final class S3AsyncRequestBodyBridge implements AsyncRequestBody { monitor.wait(waitMillis); } catch (InterruptedException exception) { Thread.currentThread().interrupt(); - throw new ObjectChunkWriteException("S3 producer was interrupted", exception); + throw providerFailure( + ObjectStorageProviderException.Failure.CANCELLED, + "S3 producer was interrupted", + exception); } } requireActive(); @@ -225,9 +240,10 @@ public final class S3AsyncRequestBodyBridge implements AsyncRequestBody { private void requireDeadline() { if (operation.budget().isExpiredAt(System.nanoTime())) { + // The request's own call budget, not an SDK timeout: the caller's limit. throw providerFailure( - ObjectStorageProviderException.Failure.DEADLINE_EXCEEDED, - "S3 content production deadline elapsed", + ObjectStorageProviderException.Failure.CANCELLED, + "S3 content production call budget ran out", null); } } diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncResponseBodyBridge.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncResponseBodyBridge.java index 7a86ecf0..1330bd63 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncResponseBodyBridge.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncResponseBodyBridge.java @@ -1,5 +1,6 @@ package dev.caskeleton.adapter.outbound.objectstorage.s3; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ContentCallbackFailures; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProvider; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; import dev.caskeleton.application.objectstorage.content.ObjectChunkReadException; @@ -11,6 +12,7 @@ import java.nio.ByteBuffer; import java.security.MessageDigest; import java.util.Arrays; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; @@ -106,6 +108,20 @@ public final class S3AsyncResponseBodyBridge implements AsyncResponseTransfor throwable)); } + /** + * The failure this bridge itself completed its result with, once it has. The SDK hands it back + * wrapped in its own client exception, which says only that the request failed. + */ + Optional ownFailure() { + CompletableFuture target = result; + if (target == null + || target.state() != java.util.concurrent.Future.State.FAILED + || !(target.exceptionNow() instanceof ObjectStorageProviderException failure)) { + return Optional.empty(); + } + return Optional.of(failure); + } + public int maximumBufferedBytesObserved() { return maximumBufferedBytesObserved.get(); } @@ -120,14 +136,18 @@ public final class S3AsyncResponseBodyBridge implements AsyncResponseTransfor operation.exactVersion(), operation.range()); MessageDigest digest = Sha256.newMessageDigest(); - ChunkCursor cursor = new ChunkCursor(subscriber, digest); + ContentCallbackFailures failures = ContentCallbackFailures.forConsumer(); + ChunkCursor cursor = new ChunkCursor(subscriber, digest, failures); ObjectChunkSource source = ObjectChunkSource.scoped(context, cursor::read); Throwable failure = null; try { consumer.consume(context, source); - if (!cursor.eof || cursor.delivered != operation.range().length()) { + failures.throwRaised(operation.cancellation().isCancelled()); + // The exact range read is enough; the consumer need not read once more to see EOF (the + // response length was already checked against the range, as local-dev and the fake do). + if (cursor.delivered != operation.range().length()) { throw new ObjectStorageProviderException( - ObjectStorageProviderException.Failure.PRECONDITION_FAILED, + ObjectStorageProviderException.Failure.CONSUMER_FAILED, "S3 consumer did not read the exact response range"); } if (operation.range().offset() == 0 @@ -135,18 +155,13 @@ public final class S3AsyncResponseBodyBridge implements AsyncResponseTransfor && !MessageDigest.isEqual( digest.digest(), operation.contentIdentity().fullDigest().decodedValue())) { throw new ObjectStorageProviderException( - ObjectStorageProviderException.Failure.CONTENT_MISMATCH, + ObjectStorageProviderException.Failure.CORRUPT_DATA, "S3 response logical digest does not match"); } } catch (Exception exception) { - subscriber.cancel(); - failure = - exception instanceof ObjectStorageProviderException - ? exception - : new ObjectStorageProviderException( - ObjectStorageProviderException.Failure.IO_FAILURE, - "bounded S3 content consumption failed", - exception); + // fail() below completes the result before it cancels the subscriber. Cancelling here first + // would let the SDK report the cancellation as a stream failure ahead of this one. + failure = failures.classify(exception, operation.cancellation().isCancelled()); } finally { context.invalidate(); } @@ -294,9 +309,13 @@ public final class S3AsyncResponseBodyBridge implements AsyncResponseTransfor private long delivered; private boolean eof; - private ChunkCursor(BodySubscriber subscriber, MessageDigest digest) { + private final ContentCallbackFailures failures; + + private ChunkCursor( + BodySubscriber subscriber, MessageDigest digest, ContentCallbackFailures failures) { this.subscriber = subscriber; this.digest = digest; + this.failures = failures; } private int read(byte[] destination, int destinationOffset, int length) @@ -305,11 +324,22 @@ public final class S3AsyncResponseBodyBridge implements AsyncResponseTransfor return ObjectChunkSource.EOF; } if (current == null || offset == current.length) { - Chunk chunk = subscriber.take(); + Chunk chunk; + try { + chunk = subscriber.take(); + } catch (ObjectChunkReadException exception) { + // take() stops only on cancellation, interruption or the request's own call budget. + throw failures.readFailure( + ObjectStorageProviderException.Failure.CANCELLED, exception.getMessage(), exception); + } if (chunk.end()) { eof = true; if (delivered != operation.range().length()) { - throw new ObjectChunkReadException("S3 response body is truncated"); + // A body shorter than its Content-Length is the transport's, not stored corruption. + throw failures.readFailure( + ObjectStorageProviderException.Failure.IO_FAILURE, + "S3 response body is truncated", + null); } return ObjectChunkSource.EOF; } @@ -319,7 +349,10 @@ public final class S3AsyncResponseBodyBridge implements AsyncResponseTransfor int count = Math.min(length, current.length - offset); long next = delivered + count; if (next > operation.range().length()) { - throw new ObjectChunkReadException("S3 response exceeds the exact requested range"); + throw failures.readFailure( + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "S3 response exceeds the exact requested range", + null); } System.arraycopy(current, offset, destination, destinationOffset, count); digest.update(current, offset, count); diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ChecksumPolicy.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ChecksumPolicy.java index f1b13054..e5f84e40 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ChecksumPolicy.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ChecksumPolicy.java @@ -36,7 +36,7 @@ public final class S3ChecksumPolicy { throw new dev.caskeleton.adapter.outbound.objectstorage.provider .ObjectStorageProviderException( dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException - .Failure.CONTENT_MISMATCH, + .Failure.CORRUPT_DATA, "S3 logical/provider checksum evidence does not match"); } } diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ConditionalObjectControlStore.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ConditionalObjectControlStore.java index f7a57a87..684a28f3 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ConditionalObjectControlStore.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ConditionalObjectControlStore.java @@ -173,7 +173,9 @@ public final class S3ConditionalObjectControlStore implements ObjectControlStore private ControlPayload(String eTag, byte[] bytes) { if (eTag == null || eTag.isBlank()) { - throw new IllegalArgumentException("S3 control response ETag is absent"); + throw new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.IO_FAILURE, + "S3 control response ETag is absent"); } this.eTag = eTag; this.bytes = bytes.clone(); diff --git a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ManagedObjectProvider.java b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ManagedObjectProvider.java index 16715aef..bf84c3d7 100644 --- a/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ManagedObjectProvider.java +++ b/src/adapter/outbound/objectstorage/src/main/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ManagedObjectProvider.java @@ -6,6 +6,7 @@ import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProvi import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderOperation; import dev.caskeleton.application.objectstorage.content.ObjectContentConsumer; import dev.caskeleton.application.objectstorage.content.ObjectContentProducer; +import dev.caskeleton.application.objectstorage.model.ObjectContentIdentity; import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -59,16 +60,27 @@ public final class S3ManagedObjectProvider implements ObjectStorageProvider { if (binding.expectedOwner() != null) { request.expectedBucketOwner(binding.expectedOwner()); } - PutObjectResponse response = await(client.putObject(request.build(), body), true); + PutObjectResponse response; + try { + response = await(client.putObject(request.build(), body), true); + } catch (ObjectStorageProviderException failure) { + throw callerSideBodyFailure(body, operation).orElseGet(() -> providerSide(failure, body)); + } await(body.observedIdentity(), true); - if (response.serverSideEncryption() != ServerSideEncryption.AES256 - || (response.checksumSHA256() != null - && !response - .checksumSHA256() - .equals(operation.expectedContentIdentity().fullDigest().base64Value()))) { + // The produced bytes already matched the declared identity (observedIdentity above), so a + // mismatch here is the provider's, not the caller's content. + if (response.serverSideEncryption() != ServerSideEncryption.AES256) { throw new ObjectStorageProviderException( - ObjectStorageProviderException.Failure.CONTENT_MISMATCH, - "S3 PUT evidence does not match the managed profile"); + ObjectStorageProviderException.Failure.UNSUPPORTED, + "S3 PUT did not apply the managed encryption profile"); + } + if (response.checksumSHA256() != null + && !response + .checksumSHA256() + .equals(operation.expectedContentIdentity().fullDigest().base64Value())) { + throw new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "S3 PUT checksum evidence does not match the produced content"); } return new StoredData( operation.canonicalDataKey(), @@ -98,9 +110,11 @@ public final class S3ManagedObjectProvider implements ObjectStorageProvider { operation.canonicalDataKey(), operation.contentIdentity(), operation.exactVersion()))) { + // The read operation carries the manifest's recorded evidence: a different stored object is + // corruption, not a concurrent change (data keys are create-once). throw new ObjectStorageProviderException( - ObjectStorageProviderException.Failure.PRECONDITION_FAILED, - "S3 exact read precondition failed"); + ObjectStorageProviderException.Failure.CORRUPT_DATA, + "S3 stored object does not match its recorded evidence"); } S3ConditionalRequestMapper.ObjectStorageProviderReadEvidence readEvidence = new S3ConditionalRequestMapper.ObjectStorageProviderReadEvidence( @@ -114,7 +128,33 @@ public final class S3ManagedObjectProvider implements ObjectStorageProvider { consumer, contentWorker, response -> validateGetResponse(response, head, operation)); - await(client.getObject(request, bridge), false); + try { + await(client.getObject(request, bridge), false); + } catch (ObjectStorageProviderException failure) { + throw callerSideReadFailure(bridge, operation).orElse(failure); + } + } + + /** + * The failure the response bridge recorded, when it is the caller's side or the body itself that + * ended the GET: the consumer failed, the body was not the recorded object, or the caller's own + * cancellation or budget stopped it. The SDK reports such a GET as a failed request; the bridge + * knows why. A failure the bridge saw only because the transport gave up is not that, and the + * GET's own classification stands. + */ + private static Optional callerSideReadFailure( + S3AsyncResponseBodyBridge bridge, ReadOperation operation) { + return bridge + .ownFailure() + .filter( + failure -> + switch (failure.failure()) { + case CONSUMER_FAILED, CORRUPT_DATA -> true; + case CANCELLED -> + operation.cancellation().isCancelled() + || operation.budget().isExpiredAt(System.nanoTime()); + default -> false; + }); } /** @@ -167,7 +207,12 @@ public final class S3ManagedObjectProvider implements ObjectStorageProvider { ? Resolution.APPLIED : Resolution.PRECONDITION_FAILED; } catch (ObjectStorageProviderException exception) { - return Resolution.INDETERMINATE; + // Only a failure that leaves the outcome unknown is "cannot resolve": a refused bucket or a + // corrupt data key is an answer the caller must see, not something a retry settles. + if (exception.failure().leavesTheOutcomeUnknown()) { + return Resolution.INDETERMINATE; + } + throw exception; } } @@ -188,7 +233,7 @@ public final class S3ManagedObjectProvider implements ObjectStorageProvider { || (head.providerVersionId() != null && !Objects.equals(response.versionId(), head.providerVersionId()))) { throw new ObjectStorageProviderException( - ObjectStorageProviderException.Failure.PRECONDITION_FAILED, + ObjectStorageProviderException.Failure.CORRUPT_DATA, "S3 GET response does not match exact HEAD evidence"); } boolean range = @@ -204,7 +249,7 @@ public final class S3ManagedObjectProvider implements ObjectStorageProvider { + operation.contentIdentity().exactLength(); if (!expected.equals(response.contentRange())) { throw new ObjectStorageProviderException( - ObjectStorageProviderException.Failure.PRECONDITION_FAILED, + ObjectStorageProviderException.Failure.CORRUPT_DATA, "S3 GET content range is not exact"); } } @@ -228,9 +273,61 @@ public final class S3ManagedObjectProvider implements ObjectStorageProvider { } } + /** + * The failure the request body recorded, when it is the caller's side that ended the PUT: its + * producer failed, its bytes were not the declared content, or its own cancellation or budget + * stopped it. The SDK reports such a PUT as an indeterminate mutation; the body knows why. A + * failure the body saw only because the transport gave up (the SDK cancelled it) is not the + * caller's, and the PUT's own classification stands. + */ + private static Optional callerSideBodyFailure( + S3AsyncRequestBodyBridge body, ObjectStorageProviderOperation operation) { + CompletableFuture observed = body.observedIdentity(); + if (observed.state() != java.util.concurrent.Future.State.FAILED + || !(observed.exceptionNow() instanceof ObjectStorageProviderException bodyFailure)) { + return Optional.empty(); + } + boolean callerSide = + switch (bodyFailure.failure()) { + case PRODUCER_FAILED, CONTENT_MISMATCH -> true; + case CANCELLED -> + operation.cancellation().isCancelled() + || operation.budget().isExpiredAt(System.nanoTime()); + default -> false; + }; + return callerSide ? Optional.of(bodyFailure) : Optional.empty(); + } + + /** + * The PUT's own failure. When S3 reports a checksum mismatch although the body delivered exactly + * the declared bytes, the mismatch is not the caller's content: the bytes changed on the way. + */ + private static ObjectStorageProviderException providerSide( + ObjectStorageProviderException failure, S3AsyncRequestBodyBridge body) { + CompletableFuture observed = body.observedIdentity(); + if (failure.failure() == ObjectStorageProviderException.Failure.CONTENT_MISMATCH + && observed.state() == java.util.concurrent.Future.State.SUCCESS) { + // Nothing was stored and the bytes were right when they left: damage in transit, which a + // retry can settle. + return new ObjectStorageProviderException( + ObjectStorageProviderException.Failure.IO_FAILURE, + "S3 rejected a body that matched its declared identity", + failure); + } + return failure; + } + private T await(CompletableFuture future, boolean mutation) { try { return future.join(); + } catch (java.util.concurrent.CancellationException cancelled) { + // Nothing here cancels an SDK future; a cancelled one is the SDK giving up, not the caller. + throw new ObjectStorageProviderException( + mutation + ? ObjectStorageProviderException.Failure.INDETERMINATE + : ObjectStorageProviderException.Failure.IO_FAILURE, + "S3 request was cancelled", + cancelled); } catch (CompletionException exception) { Throwable cause = exception.getCause(); if (cause instanceof ObjectStorageProviderException providerException) { diff --git a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/ObjectStorageArchitectureTest.java b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/ObjectStorageArchitectureTest.java index 8df21a1f..24503d70 100644 --- a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/ObjectStorageArchitectureTest.java +++ b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/ObjectStorageArchitectureTest.java @@ -1,11 +1,16 @@ package dev.caskeleton.adapter.outbound.objectstorage; +import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.classes; import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.methods; import com.tngtech.archunit.core.domain.JavaClass; +import com.tngtech.archunit.core.domain.JavaMethod; import com.tngtech.archunit.junit.AnalyzeClasses; import com.tngtech.archunit.junit.ArchTest; +import com.tngtech.archunit.lang.ArchCondition; import com.tngtech.archunit.lang.ArchRule; +import com.tngtech.archunit.lang.ConditionEvents; +import com.tngtech.archunit.lang.SimpleConditionEvent; @AnalyzeClasses(packages = "dev.caskeleton.adapter.outbound.objectstorage") class ObjectStorageArchitectureTest { @@ -30,4 +35,57 @@ class ObjectStorageArchitectureTest { "..adapter.inbound.web..", "..adapter.outbound.persistence..")) .allowEmptyShould(false); + + @ArchTest + static final ArchRule ONLY_ROUTING_ADAPTERS_IMPLEMENT_OBJECT_STORAGE_PORTS = + classes() + .that() + .implement( + JavaClass.Predicates.resideInAPackage( + "dev.caskeleton.application.objectstorage.port")) + .should() + .resideInAPackage("..adapter.outbound.objectstorage.routing") + .allowEmptyShould(false); + + /** + * Every port method runs through the one place adapter failures become the port's own failure; a + * method that skips it would let a provider or control exception leave the port. + */ + @ArchTest + static final ArchRule EVERY_PORT_METHOD_RUNS_THROUGH_THE_PORT_BOUNDARY = + methods() + .that() + .areDeclaredInClassesThat() + .implement( + JavaClass.Predicates.resideInAPackage( + "dev.caskeleton.application.objectstorage.port")) + .and() + .arePublic() + .and() + .areNotStatic() + .should(callThePortBoundary()) + .allowEmptyShould(false); + + private static ArchCondition callThePortBoundary() { + String boundary = + "dev.caskeleton.adapter.outbound.objectstorage.routing.ObjectStoragePortBoundary"; + return new ArchCondition<>("call " + boundary + ".call") { + @Override + public void check(JavaMethod method, ConditionEvents events) { + boolean calls = + method.getMethodCallsFromSelf().stream() + .anyMatch( + call -> + call.getTarget().getOwner().getFullName().equals(boundary) + && call.getTarget().getName().equals("call")); + events.add( + new SimpleConditionEvent( + method, + calls, + method.getFullName() + + (calls ? " calls" : " does not call") + + " the port boundary")); + } + }; + } } diff --git a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/contract/LocalDevCorruptStoredDataTest.java b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/contract/LocalDevCorruptStoredDataTest.java new file mode 100644 index 00000000..a54eda09 --- /dev/null +++ b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/contract/LocalDevCorruptStoredDataTest.java @@ -0,0 +1,101 @@ +package dev.caskeleton.adapter.outbound.objectstorage.contract; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.objectstorage.content.CancellationView; +import dev.caskeleton.application.objectstorage.model.ObjectDigestVerification; +import dev.caskeleton.application.objectstorage.model.ObjectOperationError; +import dev.caskeleton.application.objectstorage.model.ObjectPublishReceipt; +import dev.caskeleton.application.objectstorage.model.ObjectReadRange; +import dev.caskeleton.application.objectstorage.request.ObjectReadRequest; +import dev.caskeleton.application.objectstorage.testkit.ObjectStorageDestinationPolicy; +import dev.caskeleton.application.objectstorage.testkit.ObjectStoragePortContractSupport; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Stored bytes that are no longer what the store recorded reach the port as corrupt evidence, not + * as a conflict to retry. The fake keeps no files, so this lives beside the local-dev contracts + * rather than in them. + */ +class LocalDevCorruptStoredDataTest extends ObjectStoragePortContractSupport { + + @TempDir Path root; + + private LocalDevObjectStorage storage; + + @BeforeEach + void assemble() { + storage = new LocalDevObjectStorage(root); + } + + @AfterEach + void close() { + storage.close(); + } + + @Override + protected ObjectStorageDestinationPolicy destination() { + return storage.destination(LocalDevObjectStorage.POSTER); + } + + @Test + @DisplayName("저장된 바이트가 기록과 다르면 전송은 CORRUPT_EVIDENCE다") + void aTransferOfStoredBytesThatDifferFromTheRecordIsCorruptEvidence() throws IOException { + // given + byte[] content = content(200, 90); + ObjectPublishReceipt published = + storage.mutation.publish( + publishRequest(key("corrupt-transfer"), content), producing(content)); + Path stored = onlyDataFile(); + Files.setPosixFilePermissions(stored, PosixFilePermissions.fromString("rw-------")); + Files.write(stored, Arrays.copyOf(content, content.length - 1)); + ObjectReadRequest read = + new ObjectReadRequest( + published.reference(), + Optional.empty(), + ObjectReadRange.of(0, content.length), + ObjectDigestVerification.FULL_CONTENT, + content.length, + budget(), + CancellationView.never()); + + // when + Throwable failed = + catchThrowable( + () -> + storage.read.transfer( + read, + (context, source) -> { + byte[] buffer = new byte[16]; + while (source.read(buffer, 0, buffer.length) != -1) {} + })); + + // then + assertPortFailure(failed, ObjectOperationError.CORRUPT_EVIDENCE); + } + + private Path onlyDataFile() throws IOException { + try (Stream files = Files.walk(root)) { + List data = + files + .filter(Files::isRegularFile) + .filter(path -> root.relativize(path).toString().contains("/data/")) + .toList(); + assertThat(data).as("the one published object's data file").hasSize(1); + return data.get(0); + } + } +} diff --git a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/contract/LocalDevObjectStorage.java b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/contract/LocalDevObjectStorage.java index 4e14adfd..b6ccb52e 100644 --- a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/contract/LocalDevObjectStorage.java +++ b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/contract/LocalDevObjectStorage.java @@ -9,6 +9,7 @@ import dev.caskeleton.adapter.outbound.objectstorage.filesystem.FilesystemLocalD import dev.caskeleton.adapter.outbound.objectstorage.kernel.ObjectPolicySnapshot; import dev.caskeleton.adapter.outbound.objectstorage.kernel.ObjectPublicationHandoffService; import dev.caskeleton.adapter.outbound.objectstorage.routing.RoutingObjectMutationAdapter; +import dev.caskeleton.adapter.outbound.objectstorage.routing.RoutingObjectPublicationHandoffAdapter; import dev.caskeleton.adapter.outbound.objectstorage.routing.RoutingObjectReadAdapter; import dev.caskeleton.adapter.outbound.objectstorage.routing.RoutingObjectScanMaintenanceAdapter; import dev.caskeleton.application.objectstorage.identity.ObjectDestinationId; @@ -37,7 +38,7 @@ final class LocalDevObjectStorage implements AutoCloseable { final RoutingObjectMutationAdapter mutation; final RoutingObjectReadAdapter read; - final ObjectPublicationHandoffService handoff; + final RoutingObjectPublicationHandoffAdapter handoff; final RoutingObjectScanMaintenanceAdapter scan; private final ObjectStorageCapabilityAssembler.AssembledCapability capability; @@ -51,7 +52,9 @@ final class LocalDevObjectStorage implements AutoCloseable { Clock clock = Clock.systemUTC(); mutation = new RoutingObjectMutationAdapter(capability, clock); read = new RoutingObjectReadAdapter(capability); - handoff = new ObjectPublicationHandoffService(capability, clock); + handoff = + new RoutingObjectPublicationHandoffAdapter( + new ObjectPublicationHandoffService(capability, clock)); scan = new RoutingObjectScanMaintenanceAdapter(capability, clock); } diff --git a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectStorageConfinementTest.java b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectStorageConfinementTest.java index 20fe5fb0..d7735540 100644 --- a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectStorageConfinementTest.java +++ b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectStorageConfinementTest.java @@ -8,6 +8,8 @@ import dev.caskeleton.adapter.outbound.objectstorage.control.CanonicalJsonObject import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectReferenceRecord; import dev.caskeleton.adapter.outbound.objectstorage.kernel.PublishedReferenceState; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProvider; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException.Failure; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderOperation; import dev.caskeleton.application.objectstorage.identity.ObjectId; import dev.caskeleton.application.objectstorage.identity.ObjectReference; @@ -72,10 +74,34 @@ class LocalDevObjectStorageConfinementTest { // then assertThat(refused) - .isInstanceOf(IllegalArgumentException.class) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + tampered -> assertThat(tampered.failure()).isEqualTo(Failure.CORRUPT_DATA)) .hasMessageContaining("symbolic link"); } + @Test + @org.junit.jupiter.api.DisplayName( + "data key 자리의 symlink는 resolveCreate가 결과 모름이 아니라 CORRUPT_DATA로 던진다") + void aSymbolicLinkAtTheDataKeyIsCorruptionForResolveCreate() throws IOException { + // given + LocalDevObjectStorageProvider provider = + provider(LocalObjectStreamTransfer.FaultInjector.none()); + ObjectStorageProvider.StoredData stored = provider.create(operation(4), producer()); + Path leaf = physical(stored.canonicalDataKey()); + Files.delete(leaf); + Files.createSymbolicLink(leaf, Files.write(outside.resolve("planted"), OUTSIDE_CONTENT)); + + // when + Throwable refused = catchThrowable(() -> provider.resolveCreate(operation(4))); + + // then + assertThat(refused) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + tampered -> assertThat(tampered.failure()).isEqualTo(Failure.CORRUPT_DATA)); + } + @Test void aSymbolicLinkLeafIsNeverFollowedOnRead() throws IOException { // given @@ -91,7 +117,9 @@ class LocalDevObjectStorageConfinementTest { // then assertThat(refused) - .isInstanceOf(IllegalArgumentException.class) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + tampered -> assertThat(tampered.failure()).isEqualTo(Failure.CORRUPT_DATA)) .hasMessageContaining("symbolic link"); } @@ -155,7 +183,9 @@ class LocalDevObjectStorageConfinementTest { // then assertThat(refused) - .isInstanceOf(IllegalArgumentException.class) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + tampered -> assertThat(tampered.failure()).isEqualTo(Failure.CORRUPT_DATA)) .hasMessageContaining("symbolic link"); } diff --git a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectStorageRecoveryTest.java b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectStorageRecoveryTest.java index 7d2aad27..57af5d84 100644 --- a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectStorageRecoveryTest.java +++ b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalDevObjectStorageRecoveryTest.java @@ -70,7 +70,7 @@ class LocalDevObjectStorageRecoveryTest { () -> guard.resolveData( "data/v1/0123456789ab/aa/0123456789abcdefghjkmnpqrs/0000000000000001", true)) - .isInstanceOf(IllegalArgumentException.class); + .isInstanceOf(ObjectStorageProviderException.class); } catch (UnsupportedOperationException exception) { assertThat(exception).isNotNull(); } diff --git a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalObjectStreamTransferFailureTest.java b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalObjectStreamTransferFailureTest.java new file mode 100644 index 00000000..52f764fe --- /dev/null +++ b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/filesystem/LocalObjectStreamTransferFailureTest.java @@ -0,0 +1,119 @@ +package dev.caskeleton.adapter.outbound.objectstorage.filesystem; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.adapter.outbound.objectstorage.contract.ObjectStorageProviderContract; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProvider; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException.Failure; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderOperation; +import dev.caskeleton.adapter.outbound.objectstorage.s3.S3ObjectEvidenceMapper; +import dev.caskeleton.application.objectstorage.content.CallBudget; +import dev.caskeleton.application.objectstorage.content.CancellationView; +import dev.caskeleton.application.objectstorage.content.ObjectChunkReadException; +import dev.caskeleton.application.objectstorage.content.ObjectChunkWriteException; +import dev.caskeleton.application.objectstorage.content.ObjectContentConsumer; +import dev.caskeleton.application.objectstorage.content.ObjectContentProducer; +import dev.caskeleton.application.objectstorage.model.ObjectContentIdentity; +import dev.caskeleton.application.objectstorage.model.ObjectDigest; +import dev.caskeleton.application.objectstorage.model.ObjectReadRange; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class LocalObjectStreamTransferFailureTest { + + @TempDir Path directory; + + @Test + @DisplayName("sink의 쓰기 실패를 삼키고 돌아온 producer도 그 실패(IO_FAILURE)로 끝난다") + void aProducerThatSwallowsTheSinksWriteFailureStillEndsWithIt() throws IOException { + // given + Path temporary = Files.createFile(directory.resolve("data")); + FileChannel closed = FileChannel.open(temporary, StandardOpenOption.WRITE); + closed.close(); + byte[] content = new byte[16]; + ObjectStorageProviderOperation operation = + ObjectStorageProviderContract.operation(70, content, CancellationView.never()); + AtomicReference swallowed = new AtomicReference<>(); + ObjectContentProducer swallowing = + (context, sink) -> { + try { + sink.write(content, 0, content.length); + } catch (ObjectChunkWriteException failure) { + swallowed.set(failure); + } + }; + + // when + Throwable failed = + catchThrowable( + () -> + new LocalObjectStreamTransfer(LocalObjectStreamTransfer.FaultInjector.none()) + .write(closed, temporary, operation, swallowing)); + + // then + assertThat(failed) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> assertThat(providerFailure.failure()).isEqualTo(Failure.IO_FAILURE)); + } + + @Test + @DisplayName("source의 읽기 실패를 삼키고 돌아온 consumer도 그 실패(IO_FAILURE)로 끝난다") + void aConsumerThatSwallowsTheSourcesReadFailureStillEndsWithIt() throws IOException { + // given + byte[] content = new byte[16]; + Path stored = Files.write(directory.resolve("stored"), content); + FileChannel channel = FileChannel.open(stored, StandardOpenOption.READ); + String key = + ObjectStorageProviderContract.operation(71, content, CancellationView.never()) + .canonicalDataKey(); + ObjectStorageProvider.ReadOperation read = + new ObjectStorageProvider.ReadOperation( + key, + S3ObjectEvidenceMapper.versionForKey(key), + new ObjectContentIdentity(content.length, ObjectDigest.sha256(content)), + ObjectReadRange.of(0, content.length), + CallBudget.fromNow(Duration.ofMinutes(1)), + CancellationView.never(), + 64, + content.length); + AtomicReference swallowed = new AtomicReference<>(); + ObjectContentConsumer swallowing = + (context, source) -> { + try { + channel.close(); + } catch (IOException closeFailure) { + throw new UncheckedIOException(closeFailure); + } + try { + var unused = source.read(new byte[content.length], 0, content.length); + } catch (ObjectChunkReadException failure) { + swallowed.set(failure); + } + }; + + // when + Throwable failed = + catchThrowable( + () -> + new LocalObjectStreamTransfer(LocalObjectStreamTransfer.FaultInjector.none()) + .read(channel, read, swallowing)); + + // then + assertThat(failed) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> assertThat(providerFailure.failure()).isEqualTo(Failure.IO_FAILURE)); + } +} diff --git a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ManagedPublicationOutcomeTest.java b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ManagedPublicationOutcomeTest.java index 58b5d854..ce1d7a66 100644 --- a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ManagedPublicationOutcomeTest.java +++ b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/ManagedPublicationOutcomeTest.java @@ -1,6 +1,7 @@ package dev.caskeleton.adapter.outbound.objectstorage.kernel; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.objectstorage.config.CompiledObjectStorageProvider; import dev.caskeleton.adapter.outbound.objectstorage.config.ObjectStorageBindingCompiler; @@ -10,6 +11,7 @@ import dev.caskeleton.adapter.outbound.objectstorage.config.ObjectStorageDestina import dev.caskeleton.adapter.outbound.objectstorage.config.ObjectStorageProviderContribution; import dev.caskeleton.adapter.outbound.objectstorage.config.ObjectStorageProviderSettings; import dev.caskeleton.adapter.outbound.objectstorage.config.SelectedObjectStorageProviderFactory; +import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlConflictException; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlMutation; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlRecord; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlStore; @@ -30,9 +32,11 @@ import dev.caskeleton.application.objectstorage.model.ObjectDigest; import dev.caskeleton.application.objectstorage.model.ObjectEncryptionRequirement; import dev.caskeleton.application.objectstorage.model.ObjectMediaType; import dev.caskeleton.application.objectstorage.model.ObjectMutationOutcome; +import dev.caskeleton.application.objectstorage.model.ObjectOperationError; import dev.caskeleton.application.objectstorage.model.ObjectPublicationRequirement; import dev.caskeleton.application.objectstorage.model.ObjectPublishReceipt; import dev.caskeleton.application.objectstorage.model.ObjectRetentionRequirement; +import dev.caskeleton.application.objectstorage.model.ObjectStorageFailureException; import dev.caskeleton.application.objectstorage.request.ObjectPublishRequest; import java.nio.file.Path; import java.time.Clock; @@ -100,6 +104,44 @@ class ManagedPublicationOutcomeTest { assertThat(overtaken.reference()).isEqualTo(overtaking.get().reference()); } + @Test + @DisplayName("PUBLISHED로 바꾸는 CAS에서 진 호출은 충돌이 아니라 REPLAYED다") + void aCallThatLosesThePublishedTransitionReplaysTheWinner() { + // given + byte[] content = "outraced-poster-bytes".getBytes(java.nio.charset.StandardCharsets.UTF_8); + ObjectPublishRequest request = request(content); + AtomicReference winner = new AtomicReference<>(); + control.onceBeforePublished(() -> winner.set(publication.publish(request, producing(content)))); + + // when + ObjectPublishReceipt loser = publication.publish(request, producing(content)); + + // then + assertThat(List.of(loser.outcome(), winner.get().outcome())) + .as("the winner published; the loser lost the CAS to it and replays it") + .containsExactly(ObjectMutationOutcome.REPLAYED, ObjectMutationOutcome.APPLIED); + assertThat(loser.reference()).isEqualTo(winner.get().reference()); + } + + @Test + @DisplayName("PUBLISHED 쓰기를 잃었는데 아무도 게시하지 않았으면 포트는 CONFLICT로 알린다") + void aLostPublicationWriteWithNoPublisherIsAPortConflict() { + // given + byte[] content = "contended-poster-bytes".getBytes(java.nio.charset.StandardCharsets.UTF_8); + control.onceLosePublished(); + + // when + Throwable failed = + catchThrowable(() -> publication.publish(request(content), producing(content))); + + // then + assertThat(failed) + .isInstanceOfSatisfying( + ObjectStorageFailureException.class, + portFailure -> + assertThat(portFailure.error()).isEqualTo(ObjectOperationError.CONFLICT)); + } + private static ObjectPublishRequest request(byte[] content) { return new ObjectPublishRequest( new ObjectOperationKey( @@ -183,18 +225,30 @@ class ManagedPublicationOutcomeTest { } /** - * Control store that, once, right after the operation is recorded DATA_UPLOADED, runs another - * call to completion before the first one continues — the overlap of two calls under one key. + * Control store that, once, runs another call to completion in the middle of the first one — the + * overlap of two calls under one key: right after the operation is recorded DATA_UPLOADED, or + * right before the first call's own PUBLISHED write, which then loses its CAS. */ private static final class OvertakingControlStore implements ObjectControlStore { private ObjectControlStore delegate; private Runnable overtake; + private Runnable outrace; + private boolean losePublished; void onceAfterDataUploaded(Runnable call) { overtake = call; } + void onceBeforePublished(Runnable call) { + outrace = call; + } + + /** The next PUBLISHED write loses its CAS although nobody else wrote. */ + void onceLosePublished() { + losePublished = true; + } + @Override public Optional read(String key) { return delegate.read(key); @@ -207,6 +261,19 @@ class ManagedPublicationOutcomeTest { @Override public StoredControlRecord compareAndSet(String key, ObjectControlMutation mutation) { + if (losePublished + && mutation.replacement() instanceof ObjectOperationRecord lost + && lost.stablePhase() == PublicationOperationState.PUBLISHED) { + losePublished = false; + throw new ObjectControlConflictException("injected lost CAS"); + } + if (outrace != null + && mutation.replacement() instanceof ObjectOperationRecord operation + && operation.stablePhase() == PublicationOperationState.PUBLISHED) { + Runnable call = outrace; + outrace = null; + call.run(); + } StoredControlRecord written = delegate.compareAndSet(key, mutation); if (overtake != null && mutation.replacement() instanceof ObjectOperationRecord operation diff --git a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/StagedObjectPublicationLifecycleTest.java b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/StagedObjectPublicationLifecycleTest.java index 2cbc577b..bd63f9a1 100644 --- a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/StagedObjectPublicationLifecycleTest.java +++ b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/kernel/StagedObjectPublicationLifecycleTest.java @@ -12,6 +12,7 @@ import dev.caskeleton.adapter.outbound.objectstorage.config.ObjectStorageDestina import dev.caskeleton.adapter.outbound.objectstorage.config.ObjectStorageProviderContribution; import dev.caskeleton.adapter.outbound.objectstorage.config.ObjectStorageProviderSettings; import dev.caskeleton.adapter.outbound.objectstorage.config.SelectedObjectStorageProviderFactory; +import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlConflictException; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlMutation; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlRecord; import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlStore; @@ -47,6 +48,7 @@ import dev.caskeleton.application.objectstorage.model.ObjectReadRange; import dev.caskeleton.application.objectstorage.model.ObjectRetentionRequirement; import dev.caskeleton.application.objectstorage.model.ObjectScanState; import dev.caskeleton.application.objectstorage.model.ObjectStageReceipt; +import dev.caskeleton.application.objectstorage.model.ObjectStorageFailureException; import dev.caskeleton.application.objectstorage.request.ObjectAbortAuthorization; import dev.caskeleton.application.objectstorage.request.ObjectAbortRequest; import dev.caskeleton.application.objectstorage.request.ObjectFinalizeRequest; @@ -69,10 +71,12 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; import org.assertj.core.api.ThrowableAssert.ThrowingCallable; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -192,6 +196,45 @@ class StagedObjectPublicationLifecycleTest { assertThat(retried.outcome()).isEqualTo(ObjectMutationOutcome.APPLIED); } + @Test + @DisplayName("PUBLISHED로 바꾸는 CAS에서 진 finalize는 같은 claimant가 먼저 끝낸 게시를 REPLAYED로 답한다") + void aFinalizeThatLosesThePublishedWriteReplaysTheWinner() { + // given + ObjectStageReceipt stage = stagedAndVerified(POSTER); + ObjectHandoffReceipt claim = claim(POSTER, stage, START); + AtomicReference winner = new AtomicReference<>(); + control.runOnceBeforeWriting( + StagedObjectPublicationLifecycleTest::publishedOperation, + () -> winner.set(finalizePublication(POSTER, stage, claim, START))); + + // when + ObjectPublishReceipt loser = finalizePublication(POSTER, stage, claim, START); + + // then + assertThat(List.of(loser.outcome(), winner.get().outcome())) + .as("the winner published; the loser lost the CAS to it and replays it") + .containsExactly(ObjectMutationOutcome.REPLAYED, ObjectMutationOutcome.APPLIED); + } + + @Test + @DisplayName("PUBLISHED 쓰기를 잃었는데 아무도 게시하지 않은 finalize는 포트가 CONFLICT로 알린다") + void aFinalizeThatLosesThePublishedWriteWithNoPublisherIsAPortConflict() { + // given + ObjectStageReceipt stage = stagedAndVerified(POSTER); + ObjectHandoffReceipt claim = claim(POSTER, stage, START); + control.loseOnce(StagedObjectPublicationLifecycleTest::publishedOperation); + + // when + Throwable failed = catchThrowable(() -> finalizePublication(POSTER, stage, claim, START)); + + // then + assertThat(failed) + .isInstanceOfSatisfying( + ObjectStorageFailureException.class, + portFailure -> + assertThat(portFailure.error()).isEqualTo(ObjectOperationError.CONFLICT)); + } + @Test void finalizeSucceedsWhenTheCleanVerdictWasRetriedAfterACrash() { // given @@ -375,6 +418,11 @@ class StagedObjectPublicationLifecycleTest { } /** The injected crash hit the write the scenario armed it for. */ + private static boolean publishedOperation(ObjectControlRecord record) { + return record instanceof ObjectOperationRecord operation + && operation.stablePhase() == PublicationOperationState.PUBLISHED; + } + private static void crashed(ThrowingCallable call) { assertThat(catchThrowable(call)).hasMessage("injected crash before a control write"); } @@ -629,16 +677,32 @@ class StagedObjectPublicationLifecycleTest { } } - /** Control store that fails the next matching write once, as a crash before it would. */ + /** + * Control store that, once, before the next matching write: fails it as a crash would, loses its + * CAS although nobody else wrote, or runs another call to completion first (whose own write then + * makes this one lose). + */ private static final class CrashingControlStore implements ObjectControlStore { private ObjectControlStore delegate; private Predicate crashBefore = record -> false; + private Predicate loseBefore = record -> false; + private Predicate outraceBefore = record -> false; + private Runnable outrace = () -> {}; void crashOnceBeforeWriting(Predicate write) { crashBefore = write; } + void loseOnce(Predicate write) { + loseBefore = write; + } + + void runOnceBeforeWriting(Predicate write, Runnable call) { + outraceBefore = write; + outrace = call; + } + @Override public Optional read(String key) { return delegate.read(key); @@ -653,6 +717,16 @@ class StagedObjectPublicationLifecycleTest { @Override public StoredControlRecord compareAndSet(String key, ObjectControlMutation mutation) { crashIfArmed(mutation.replacement()); + if (loseBefore.test(mutation.replacement())) { + loseBefore = ignored -> false; + throw new ObjectControlConflictException("injected lost CAS"); + } + if (outraceBefore.test(mutation.replacement())) { + outraceBefore = ignored -> false; + Runnable call = outrace; + outrace = () -> {}; + call.run(); + } return delegate.compareAndSet(key, mutation); } diff --git a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/routing/ObjectStoragePortBoundaryTest.java b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/routing/ObjectStoragePortBoundaryTest.java new file mode 100644 index 00000000..7574d632 --- /dev/null +++ b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/routing/ObjectStoragePortBoundaryTest.java @@ -0,0 +1,136 @@ +package dev.caskeleton.adapter.outbound.objectstorage.routing; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlConflictException; +import dev.caskeleton.adapter.outbound.objectstorage.control.ObjectControlCorruptionException; +import dev.caskeleton.adapter.outbound.objectstorage.control.UnsupportedObjectControlSchemaException; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException.Failure; +import dev.caskeleton.application.objectstorage.content.ObjectContentProductionException; +import dev.caskeleton.application.objectstorage.model.ObjectOperationError; +import dev.caskeleton.application.objectstorage.model.ObjectStorageFailureException; +import java.util.stream.Stream; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; + +class ObjectStoragePortBoundaryTest { + + @ParameterizedTest(name = "{0} → {1}") + @CsvSource({ + "CONFLICT, CONFLICT", + "PRECONDITION_FAILED, CONFLICT", + "NOT_FOUND, NOT_FOUND", + "CONTENT_MISMATCH, CONTENT_MISMATCH", + "CORRUPT_DATA, CORRUPT_EVIDENCE", + "CANCELLED, CANCELLED", + "PRODUCER_FAILED, CONTENT_PRODUCTION_FAILED", + "CONSUMER_FAILED, CONTENT_CONSUMPTION_FAILED", + "UNSUPPORTED, UNSUPPORTED_CAPABILITY", + "PERMISSION_DENIED, UNSUPPORTED_CAPABILITY", + "OWNER_MISMATCH, UNSUPPORTED_CAPABILITY", + "REGION_MISMATCH, UNSUPPORTED_CAPABILITY", + "RETENTION_BLOCKED, UNSUPPORTED_CAPABILITY", + "DEADLINE_EXCEEDED, PROVIDER_UNAVAILABLE", + "THROTTLED, PROVIDER_UNAVAILABLE", + "INDETERMINATE, PROVIDER_UNAVAILABLE", + "UNKNOWN, PROVIDER_UNAVAILABLE", + "IO_FAILURE, PROVIDER_UNAVAILABLE" + }) + @DisplayName("provider 실패는 포트 실패 하나로 바뀌고 분류는 표대로다") + void aProviderFailureLeavesThePortAsItsClassification( + Failure failure, ObjectOperationError expected) { + // when + Throwable translated = + catchThrowable( + () -> + ObjectStoragePortBoundary.call( + () -> { + throw new ObjectStorageProviderException(failure, "provider failed"); + })); + + // then + assertThat(translated) + .isInstanceOfSatisfying( + ObjectStorageFailureException.class, + portFailure -> assertThat(portFailure.error()).isEqualTo(expected)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("controlFailures") + @DisplayName("제어 레코드 실패도 포트 실패 하나로 바뀐다") + void aControlRecordFailureLeavesThePortAsItsClassification( + RuntimeException failure, ObjectOperationError expected) { + // when + Throwable translated = + catchThrowable( + () -> + ObjectStoragePortBoundary.call( + () -> { + throw failure; + })); + + // then + assertThat(translated) + .isInstanceOfSatisfying( + ObjectStorageFailureException.class, + portFailure -> assertThat(portFailure.error()).isEqualTo(expected)); + } + + static Stream controlFailures() { + return Stream.of( + Arguments.of( + new ObjectControlConflictException("version conflict"), ObjectOperationError.CONFLICT), + Arguments.of( + new ObjectControlCorruptionException("unreadable"), + ObjectOperationError.CORRUPT_EVIDENCE), + Arguments.of( + new UnsupportedObjectControlSchemaException("schema 9"), + ObjectOperationError.CORRUPT_EVIDENCE)); + } + + @Test + @DisplayName("producer 자신의 실패는 그 예외가 원인이다") + void aProducersOwnFailureIsTheCause() { + // given + ObjectContentProductionException producerFailure = + new ObjectContentProductionException("upload stream broke"); + + // when + Throwable translated = + catchThrowable( + () -> + ObjectStoragePortBoundary.call( + () -> { + throw new ObjectStorageProviderException( + Failure.PRODUCER_FAILED, "content producer failed", producerFailure); + })); + + // then + assertThat(translated).hasCause(producerFailure); + } + + @Test + @DisplayName("요청 거절은 바꾸지 않고 그대로 나간다") + void aRefusalPassesThroughUnchanged() { + // given + IllegalStateException refusal = new IllegalStateException("not finalizable"); + + // when + Throwable passed = + catchThrowable( + () -> + ObjectStoragePortBoundary.call( + () -> { + throw refusal; + })); + + // then + assertThat(passed).isSameAs(refusal); + } +} diff --git a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncRequestBodyBridgeTest.java b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncRequestBodyBridgeTest.java index f854e2cb..157174c7 100644 --- a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncRequestBodyBridgeTest.java +++ b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncRequestBodyBridgeTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.objectstorage.s3; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.objectstorage.contract.ObjectStorageProviderContract; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; @@ -125,6 +126,109 @@ class S3AsyncRequestBodyBridgeTest { } } + @Test + @DisplayName("sink의 실패를 삼키고 돌아온 producer도 그 실패(IO_FAILURE)로 끝난다") + void aProducerThatSwallowsTheSinksFailureStillEndsWithIt() throws Exception { + // given + byte[] content = {1, 2, 3}; + ObjectStorageProviderOperation operation = + ObjectStorageProviderContract.operation( + 503, + content, + dev.caskeleton.application.objectstorage.content.CancellationView.never()); + AtomicReference swallowed = new AtomicReference<>(); + var executor = Executors.newSingleThreadExecutor(); + try { + S3AsyncRequestBodyBridge bridge = + new S3AsyncRequestBodyBridge( + operation, + (context, sink) -> { + try { + sink.write(content, 0, content.length); + } catch ( + dev.caskeleton.application.objectstorage.content.ObjectChunkWriteException + failure) { + swallowed.set(failure); + } + }, + executor); + + // when + bridge.subscribe(new RejectingSubscriber()); + + // then + Throwable failed = catchThrowable(() -> bridge.observedIdentity().get(1, TimeUnit.MINUTES)); + assertThat(failed) + .cause() + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()) + .isEqualTo(ObjectStorageProviderException.Failure.IO_FAILURE)); + } finally { + executor.shutdownNow(); + } + } + + @Test + @DisplayName("다 쓰고 스스로 실패한 producer는 SDK가 길이를 다 받고 cancel해도 PRODUCER_FAILED다") + void aProducerThatFailsAfterTheExactLengthIsAProducerFailureNotACancellation() throws Exception { + // given + byte[] content = new byte[64]; + ObjectStorageProviderOperation operation = + ObjectStorageProviderContract.operation( + 504, + content, + dev.caskeleton.application.objectstorage.content.CancellationView.never()); + var executor = Executors.newSingleThreadExecutor(); + try { + S3AsyncRequestBodyBridge bridge = + new S3AsyncRequestBodyBridge( + operation, + (context, sink) -> { + ObjectStorageProviderContract.producer(content).produce(context, sink); + throw new dev.caskeleton.application.objectstorage.content + .ObjectContentProductionException("producer failed late"); + }, + executor); + + // when + bridge.subscribe(new CancelAtLengthSubscriber(content.length)); + + // then + Throwable failed = catchThrowable(() -> bridge.observedIdentity().get(1, TimeUnit.MINUTES)); + assertThat(failed) + .cause() + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()) + .isEqualTo(ObjectStorageProviderException.Failure.PRODUCER_FAILED)); + } finally { + executor.shutdownNow(); + } + } + + /** A subscriber that asks for content and rejects the first chunk it is given. */ + private static final class RejectingSubscriber implements Subscriber { + + @Override + public void onSubscribe(Subscription subscription) { + subscription.request(1); + } + + @Override + public void onNext(ByteBuffer item) { + throw new IllegalStateException("subscriber rejects the chunk"); + } + + @Override + public void onError(Throwable throwable) {} + + @Override + public void onComplete() {} + } + /** * The SDK stops its subscription once it holds Content-Length bytes, before any onComplete: a * cancel after the exact length arrived is the upload finishing, not the caller giving up. The diff --git a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncResponseBodyBridgeTest.java b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncResponseBodyBridgeTest.java index ace4c3db..7ef50dbc 100644 --- a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncResponseBodyBridgeTest.java +++ b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3AsyncResponseBodyBridgeTest.java @@ -18,6 +18,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -110,7 +111,49 @@ class S3AsyncResponseBodyBridgeTest { Throwable thrown = catchThrowable(() -> result.get(HANG_GUARD_MINUTES, TimeUnit.MINUTES)); assertThat(thrown) .isInstanceOf(ExecutionException.class) - .hasCauseInstanceOf(RuntimeException.class); + .cause() + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()) + .isEqualTo(ObjectStorageProviderException.Failure.IO_FAILURE)); + } + + @Test + @DisplayName("source의 실패를 삼키고 돌아온 consumer도 그 실패(IO_FAILURE)로 끝난다") + void aConsumerThatSwallowsTheSourcesFailureStillEndsWithIt() throws Exception { + // given + AtomicReference swallowed = new AtomicReference<>(); + S3AsyncResponseBodyBridge truncated = + new S3AsyncResponseBodyBridge<>( + read(514, new byte[65]), + (context, source) -> { + byte[] buffer = new byte[64]; + try { + while (source.read(buffer, 0, buffer.length) != -1) {} + } catch ( + dev.caskeleton.application.objectstorage.content.ObjectChunkReadException + failure) { + swallowed.set(failure); + } + }, + worker, + ignored -> {}); + var result = truncated.prepare(); + truncated.onResponse(GetObjectResponse.builder().contentLength(65L).build()); + + // when + truncated.onStream(SdkPublisher.fromIterable(List.of(ByteBuffer.wrap(new byte[64])))); + + // then + Throwable thrown = catchThrowable(() -> result.get(HANG_GUARD_MINUTES, TimeUnit.MINUTES)); + assertThat(thrown) + .cause() + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()) + .isEqualTo(ObjectStorageProviderException.Failure.IO_FAILURE)); } /** @@ -181,10 +224,79 @@ class S3AsyncResponseBodyBridgeTest { assertThat(thrown) .isInstanceOf(ExecutionException.class) .cause() - .isInstanceOf(ObjectStorageProviderException.class) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()) + .isEqualTo(ObjectStorageProviderException.Failure.CORRUPT_DATA)) .hasMessage("S3 response chunk exceeds the bounded bridge size"); } + @Test + @DisplayName("전체 범위를 읽었는데 body가 기록한 digest와 다르면 CORRUPT_DATA다") + void aFullBodyWhoseDigestDiffersFromTheRecordIsCorruptData() throws Exception { + // given + byte[] recorded = new byte[64]; + byte[] delivered = new byte[64]; + delivered[0] = 1; + S3AsyncResponseBodyBridge corrupt = + new S3AsyncResponseBodyBridge<>( + read(515, recorded), + (context, source) -> { + byte[] buffer = new byte[64]; + while (source.read(buffer, 0, buffer.length) != -1) {} + }, + worker, + ignored -> {}); + var result = corrupt.prepare(); + corrupt.onResponse(GetObjectResponse.builder().contentLength(64L).build()); + + // when + corrupt.onStream(SdkPublisher.fromIterable(List.of(ByteBuffer.wrap(delivered)))); + + // then + Throwable thrown = catchThrowable(() -> result.get(HANG_GUARD_MINUTES, TimeUnit.MINUTES)); + assertThat(thrown) + .cause() + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()) + .isEqualTo(ObjectStorageProviderException.Failure.CORRUPT_DATA)); + } + + @Test + @DisplayName("요청 범위보다 많이 보내는 응답은 CORRUPT_DATA다") + void aBodyLongerThanTheRequestedRangeIsCorruptData() throws Exception { + // given + S3AsyncResponseBodyBridge longer = + new S3AsyncResponseBodyBridge<>( + read(516, new byte[64]), + (context, source) -> { + byte[] buffer = new byte[64]; + while (source.read(buffer, 0, buffer.length) != -1) {} + }, + worker, + ignored -> {}); + var result = longer.prepare(); + longer.onResponse(GetObjectResponse.builder().contentLength(64L).build()); + + // when + longer.onStream( + SdkPublisher.fromIterable( + List.of(ByteBuffer.wrap(new byte[64]), ByteBuffer.wrap(new byte[1])))); + + // then + Throwable thrown = catchThrowable(() -> result.get(HANG_GUARD_MINUTES, TimeUnit.MINUTES)); + assertThat(thrown) + .cause() + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()) + .isEqualTo(ObjectStorageProviderException.Failure.CORRUPT_DATA)); + } + /** A fresh read with its own budget, so no bridge inherits time another bridge already used. */ /** A read of {@code content} with a 64-byte consumer chunk and room for the whole body. */ private static ObjectStorageProvider.ReadOperation readUpTo(int seed, byte[] content) { diff --git a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ChecksumPolicyTest.java b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ChecksumPolicyTest.java new file mode 100644 index 00000000..4c95dee3 --- /dev/null +++ b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ChecksumPolicyTest.java @@ -0,0 +1,39 @@ +package dev.caskeleton.adapter.outbound.objectstorage.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; +import dev.caskeleton.application.objectstorage.model.ObjectContentIdentity; +import dev.caskeleton.application.objectstorage.model.ObjectDigest; +import java.util.Map; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +class S3ChecksumPolicyTest { + + @Test + @DisplayName("저장된 logical digest가 기록과 다르면 CORRUPT_DATA다") + void storedEvidenceThatDiffersFromTheRecordIsCorruptData() { + // given + byte[] recorded = {1, 2, 3}; + ObjectContentIdentity expected = + new ObjectContentIdentity(recorded.length, ObjectDigest.sha256(recorded)); + String other = ObjectDigest.sha256(new byte[] {9}).base64Value(); + + // when + Throwable failed = + catchThrowable( + () -> + S3ChecksumPolicy.requireMatchingEvidence( + expected, null, Map.of(S3ChecksumPolicy.LOGICAL_SHA_256_METADATA, other))); + + // then + assertThat(failed) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()) + .isEqualTo(ObjectStorageProviderException.Failure.CORRUPT_DATA)); + } +} diff --git a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ManagedObjectProviderTest.java b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ManagedObjectProviderTest.java index fb9095ae..b5d1561c 100644 --- a/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ManagedObjectProviderTest.java +++ b/src/adapter/outbound/objectstorage/src/test/java/dev/caskeleton/adapter/outbound/objectstorage/s3/S3ManagedObjectProviderTest.java @@ -297,6 +297,250 @@ class S3ManagedObjectProviderTest { verify(client, never()).deleteObject(any(DeleteObjectRequest.class)); } + @Test + @org.junit.jupiter.api.DisplayName("HEAD가 보인 객체가 읽기에 실린 기록과 다르면 transfer는 CORRUPT_DATA다") + void aStoredObjectThatDiffersFromItsRecordIsCorruptData() { + // given + byte[] stored = {1, 2, 3}; + byte[] recorded = {4, 5, 6}; + ObjectStorageProviderOperation operation = + ObjectStorageProviderContract.operation(524, stored, CancellationView.never()); + S3AsyncClient client = mock(S3AsyncClient.class); + when(client.headObject(any(HeadObjectRequest.class))) + .thenReturn( + CompletableFuture.completedFuture( + head(operation, "private-version", "\"private-etag\""))); + String key = operation.canonicalDataKey(); + ObjectStorageProvider.ReadOperation read = + new ObjectStorageProvider.ReadOperation( + key, + S3ObjectEvidenceMapper.versionForKey(key), + new dev.caskeleton.application.objectstorage.model.ObjectContentIdentity( + recorded.length, + dev.caskeleton.application.objectstorage.model.ObjectDigest.sha256(recorded)), + ObjectReadRange.of(0, recorded.length), + CallBudget.fromNow(Duration.ofSeconds(5)), + CancellationView.never(), + 64, + 1024); + var executor = Executors.newSingleThreadExecutor(); + try { + S3ManagedObjectProvider provider = + new S3ManagedObjectProvider(client, awsBinding(), executor); + + // when + Throwable failed = catchThrowable(() -> provider.transfer(read, (context, source) -> {})); + + // then + assertThat(failed) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()) + .isEqualTo(ObjectStorageProviderException.Failure.CORRUPT_DATA)); + } finally { + executor.shutdownNow(); + } + } + + @org.junit.jupiter.params.ParameterizedTest(name = "{0}") + @org.junit.jupiter.params.provider.MethodSource("putEvidenceMismatches") + @org.junit.jupiter.api.DisplayName("PUT 응답이 managed profile과 다르면 호출자 바이트 탓이 아니다") + void aPutResponseThatBreaksTheManagedProfileIsTheProviders( + String mismatch, + java.util.function.UnaryOperator response, + ObjectStorageProviderException.Failure expected) { + // given + byte[] content = {1, 2, 3, 4}; + ObjectStorageProviderOperation operation = + ObjectStorageProviderContract.operation(525, content, CancellationView.never()); + S3AsyncClient client = mock(S3AsyncClient.class); + when(client.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class))) + .thenAnswer( + invocation -> { + drain(invocation.getArgument(1)); + return CompletableFuture.completedFuture( + response.apply(PutObjectResponse.builder()).build()); + }); + var executor = Executors.newFixedThreadPool(2); + try { + S3ManagedObjectProvider provider = + new S3ManagedObjectProvider(client, awsBinding(), executor); + + // when + Throwable failed = + catchThrowable( + () -> provider.create(operation, ObjectStorageProviderContract.producer(content))); + + // then + assertThat(failed) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> assertThat(providerFailure.failure()).isEqualTo(expected)); + } finally { + executor.shutdownNow(); + } + } + + static java.util.stream.Stream + putEvidenceMismatches() { + java.util.function.UnaryOperator noEncryption = + builder -> builder.versionId("private-version"); + java.util.function.UnaryOperator otherChecksum = + builder -> + builder + .versionId("private-version") + .serverSideEncryption(ServerSideEncryption.AES256) + .checksumSHA256( + dev.caskeleton.application.objectstorage.model.ObjectDigest.sha256( + new byte[] {9}) + .base64Value()); + return java.util.stream.Stream.of( + org.junit.jupiter.params.provider.Arguments.of( + "암호화 profile 없음", noEncryption, ObjectStorageProviderException.Failure.UNSUPPORTED), + org.junit.jupiter.params.provider.Arguments.of( + "checksum 증거 불일치", otherChecksum, ObjectStorageProviderException.Failure.CORRUPT_DATA)); + } + + @Test + @org.junit.jupiter.api.DisplayName("선언대로 보낸 body를 S3가 BadDigest로 거절하면 다시 해도 되는 IO_FAILURE다") + void aBadDigestForABodyThatMatchedItsDeclarationIsAnIoFailure() { + // given + byte[] content = {1, 2, 3, 4}; + ObjectStorageProviderOperation operation = + ObjectStorageProviderContract.operation(526, content, CancellationView.never()); + S3AsyncClient client = mock(S3AsyncClient.class); + when(client.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class))) + .thenAnswer( + invocation -> { + S3AsyncRequestBodyBridge body = invocation.getArgument(1); + drain(body); + // S3 answers a PUT only after it holds the whole body. + body.observedIdentity().join(); + return CompletableFuture.failedFuture( + S3Exception.builder() + .statusCode(400) + .awsErrorDetails( + software.amazon.awssdk.awscore.exception.AwsErrorDetails.builder() + .errorCode("BadDigest") + .build()) + .build()); + }); + var executor = Executors.newFixedThreadPool(2); + try { + S3ManagedObjectProvider provider = + new S3ManagedObjectProvider(client, awsBinding(), executor); + + // when + Throwable failed = + catchThrowable( + () -> provider.create(operation, ObjectStorageProviderContract.producer(content))); + + // then + assertThat(failed) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()) + .isEqualTo(ObjectStorageProviderException.Failure.IO_FAILURE)); + } finally { + executor.shutdownNow(); + } + } + + @Test + @org.junit.jupiter.api.DisplayName("SDK가 body를 cancel해 PUT이 실패하면 호출자 취소가 아니라 결과 모름이다") + void aPutWhoseBodyTheSdkCancelledIsIndeterminate() { + // given + byte[] content = {1, 2, 3, 4}; + ObjectStorageProviderOperation operation = + ObjectStorageProviderContract.operation(527, content, CancellationView.never()); + S3AsyncClient client = mock(S3AsyncClient.class); + when(client.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class))) + .thenAnswer( + invocation -> { + S3AsyncRequestBodyBridge body = invocation.getArgument(1); + body.subscribe( + new Subscriber<>() { + @Override + public void onSubscribe(Subscription subscription) { + subscription.cancel(); + } + + @Override + public void onNext(ByteBuffer item) {} + + @Override + public void onError(Throwable throwable) {} + + @Override + public void onComplete() {} + }); + // Wait until the body has recorded why it stopped. + body.observedIdentity().handle((identity, failure) -> identity).join(); + return CompletableFuture.failedFuture( + software.amazon.awssdk.core.exception.SdkClientException.create( + "connection reset")); + }); + var executor = Executors.newFixedThreadPool(2); + try { + S3ManagedObjectProvider provider = + new S3ManagedObjectProvider(client, awsBinding(), executor); + + // when + Throwable failed = + catchThrowable( + () -> provider.create(operation, ObjectStorageProviderContract.producer(content))); + + // then + assertThat(failed) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()) + .isEqualTo(ObjectStorageProviderException.Failure.INDETERMINATE)); + } finally { + executor.shutdownNow(); + } + } + + @Test + @org.junit.jupiter.api.DisplayName("HEAD가 권한 거절이면 resolveCreate는 결과 모름이 아니라 그 거절을 던진다") + void aRefusedHeadIsNotAnUnknownCreateOutcome() { + // given + S3AsyncClient client = mock(S3AsyncClient.class); + when(client.headObject(any(HeadObjectRequest.class))) + .thenReturn( + CompletableFuture.failedFuture( + S3Exception.builder() + .statusCode(403) + .awsErrorDetails( + software.amazon.awssdk.awscore.exception.AwsErrorDetails.builder() + .errorCode("AccessDenied") + .build()) + .build())); + ObjectStorageProviderOperation operation = + ObjectStorageProviderContract.operation(528, new byte[] {1}, CancellationView.never()); + var executor = Executors.newSingleThreadExecutor(); + try { + S3ManagedObjectProvider provider = + new S3ManagedObjectProvider(client, awsBinding(), executor); + + // when + Throwable failed = catchThrowable(() -> provider.resolveCreate(operation)); + + // then + assertThat(failed) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()) + .isEqualTo(ObjectStorageProviderException.Failure.PERMISSION_DENIED)); + } finally { + executor.shutdownNow(); + } + } + private static S3ProviderBinding awsBinding() { return S3ProviderBinding.compile( "aws", diff --git a/src/adapter/outbound/objectstorage/src/testFixtures/java/dev/caskeleton/adapter/outbound/objectstorage/contract/ObjectStorageProviderContract.java b/src/adapter/outbound/objectstorage/src/testFixtures/java/dev/caskeleton/adapter/outbound/objectstorage/contract/ObjectStorageProviderContract.java index 38146d51..d97f77c4 100644 --- a/src/adapter/outbound/objectstorage/src/testFixtures/java/dev/caskeleton/adapter/outbound/objectstorage/contract/ObjectStorageProviderContract.java +++ b/src/adapter/outbound/objectstorage/src/testFixtures/java/dev/caskeleton/adapter/outbound/objectstorage/contract/ObjectStorageProviderContract.java @@ -8,6 +8,7 @@ import dev.caskeleton.adapter.outbound.objectstorage.codec.ObjectDataKeyCodec; import dev.caskeleton.adapter.outbound.objectstorage.kernel.ObjectRouteToken; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProvider; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException; +import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderException.Failure; import dev.caskeleton.adapter.outbound.objectstorage.provider.ObjectStorageProviderOperation; import dev.caskeleton.application.objectstorage.content.CallBudget; import dev.caskeleton.application.objectstorage.content.CancellationView; @@ -165,7 +166,11 @@ public abstract class ObjectStorageProviderContract { (context, sink) -> sink.write(content, 0, content.length - 1))); // then - assertThat(failure).isInstanceOf(ObjectStorageProviderException.class); + assertThat(failure) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()).isEqualTo(Failure.CONTENT_MISMATCH)); assertThat(provider().inspect(shortOperation.canonicalDataKey())).isEmpty(); } @@ -188,7 +193,11 @@ public abstract class ObjectStorageProviderContract { })); // then - assertThat(failure).isInstanceOf(ObjectStorageProviderException.class); + assertThat(failure) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()).isEqualTo(Failure.PRODUCER_FAILED)); assertThat(provider().inspect(failing.canonicalDataKey())).isEmpty(); } @@ -203,7 +212,10 @@ public abstract class ObjectStorageProviderContract { Throwable failure = catchThrowable(() -> provider().create(cancelled, producer(content))); // then - assertThat(failure).isInstanceOf(ObjectStorageProviderException.class); + assertThat(failure) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> assertThat(providerFailure.failure()).isEqualTo(Failure.CANCELLED)); assertThat(provider().inspect(cancelled.canonicalDataKey())).isEmpty(); } @@ -228,7 +240,123 @@ public abstract class ObjectStorageProviderContract { Throwable failure = catchThrowable(() -> provider().create(wrongDigest, producer(content))); // then - assertThat(failure).isInstanceOf(ObjectStorageProviderException.class); + assertThat(failure) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()).isEqualTo(Failure.CONTENT_MISMATCH)); + } + + @Test + @DisplayName("consumer가 스스로 실패하면 transfer는 CONSUMER_FAILED이고 원인은 그 예외다") + void aFailingConsumerFailsTheTransferWithItsOwnException() { + // given + byte[] content = content(200); + ObjectStorageProvider.StoredData stored = + provider().create(operation(60, content, CancellationView.never()), producer(content)); + ObjectContentConsumptionException consumerFailure = + new ObjectContentConsumptionException("consumer failed"); + + // when + Throwable failure = + catchThrowable( + () -> + provider() + .transfer( + read(stored, ObjectReadRange.of(0, content.length)), + (context, source) -> { + var unused = source.read(new byte[CHUNK_BYTES], 0, CHUNK_BYTES); + throw consumerFailure; + })); + + // then + assertThat(failure) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure) + .extracting(ObjectStorageProviderException::failure, Throwable::getCause) + .containsExactly(Failure.CONSUMER_FAILED, consumerFailure)); + } + + @Test + @DisplayName("범위를 정확히 읽고 EOF를 읽지 않고 돌아온 consumer의 transfer는 성공한다") + void aConsumerThatReadsTheExactRangeWithoutEofSucceeds() { + // given + byte[] content = content(200); + ObjectStorageProvider.StoredData stored = + provider().create(operation(62, content, CancellationView.never()), producer(content)); + ByteArrayOutputStream exact = new ByteArrayOutputStream(); + + // when + provider() + .transfer( + read(stored, ObjectReadRange.of(0, content.length)), + (context, source) -> { + byte[] buffer = new byte[CHUNK_BYTES]; + while (exact.size() < content.length) { + int count = + source.read(buffer, 0, Math.min(CHUNK_BYTES, content.length - exact.size())); + exact.write(buffer, 0, count); + } + }); + + // then + assertThat(exact.toByteArray()).containsExactly(content); + } + + @Test + @DisplayName("선언한 바이트를 다 쓰고 스스로 실패한 producer의 create는 PRODUCER_FAILED다") + void aProducerThatFailsAfterWritingEverythingIsAProducerFailure() { + // given + byte[] content = content(200); + ObjectStorageProviderOperation complete = operation(63, content, CancellationView.never()); + + // when + Throwable failure = + catchThrowable( + () -> + provider() + .create( + complete, + (context, sink) -> { + producer(content).produce(context, sink); + throw new ObjectContentProductionException("producer failed late"); + })); + + // then + assertThat(failure) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()).isEqualTo(Failure.PRODUCER_FAILED)); + } + + @Test + @DisplayName("범위를 다 읽지 않고 돌아온 consumer의 transfer는 CONSUMER_FAILED다") + void aConsumerThatReturnsBeforeTheEndFailsTheTransfer() { + // given + byte[] content = content(200); + ObjectStorageProvider.StoredData stored = + provider().create(operation(61, content, CancellationView.never()), producer(content)); + + // when + Throwable failure = + catchThrowable( + () -> + provider() + .transfer( + read(stored, ObjectReadRange.of(0, content.length)), + (context, source) -> { + var unused = source.read(new byte[CHUNK_BYTES], 0, CHUNK_BYTES); + })); + + // then + assertThat(failure) + .isInstanceOfSatisfying( + ObjectStorageProviderException.class, + providerFailure -> + assertThat(providerFailure.failure()).isEqualTo(Failure.CONSUMER_FAILED)); } @Test diff --git a/src/application/objectstorage/AGENTS.md b/src/application/objectstorage/AGENTS.md index 6151355f..6c1f194b 100644 --- a/src/application/objectstorage/AGENTS.md +++ b/src/application/objectstorage/AGENTS.md @@ -23,6 +23,14 @@ - adapter testFixtures의 `ObjectStorageProviderContract`는 adapter 안 provider SPI의 계약이다(local-dev와 MinIO lane이 상속). 포트 계약과 섞지 않는다. 그래도 그 시그니처가 이 leaf의 값 타입을 쓰므로, 값 타입을 바꾸면 provider 계약도 같은 변경에서 고친다. - 포트의 결과와 예외는 각 포트 Javadoc이 정한다. `IllegalArgumentException`은 요청 그대로를 거절한 것, `IllegalStateException`은 지금 상태(다른 claimant의 claim, 검증 전, 이미 게시)에서 거절한 것이다. 둘 다 서버 결함이 아니다. 유스케이스는 그대로 던지지 않고 자기 결과나 기능 오류(상태 충돌이면 CONFLICT carrier)로 바꾼다(`docs/manual/06-errors.md`의 「서버 탓 = `IllegalStateException`」과 섞이지 않게). - 예외: 설정되지 않았거나 scan이 필요한 destination으로 보낸 `IllegalArgumentException`은 유스케이스가 고른 destination이 틀린 것, 곧 조립 오류다. 유스케이스는 이것을 `IllegalStateException`(500)으로 바꾸고 클라이언트 오류로 가리지 않는다. +- 저장소의 실패와 호출자의 producer·consumer 실패는 `ObjectStorageFailureException`이다. 유스케이스는 메시지가 아니라 `error()`로 가른다. 분류마다 뜻은 그 클래스 Javadoc이 정한다. 가르는 기준은 셋이다: + - 호출자 쪽(`CONTENT_MISMATCH`, `CONTENT_PRODUCTION_FAILED`, `CONTENT_CONSUMPTION_FAILED`, `CANCELLED`): 올린 내용이나 연결의 문제다. + - 다시 하면 되는 것(`CONFLICT`, `PROVIDER_UNAVAILABLE`): 같은 operation key로 다시 해도 안전하다. + - 서버 결함(`CORRUPT_EVIDENCE`, `UNSUPPORTED_CAPABILITY`, `NOT_FOUND`): 다시 해도 소용없다. `UNSUPPORTED_CAPABILITY`에는 provider가 destination 설정(권한, owner, region, 보존)을 거절한 것도 든다. +- 포트 밖으로 provider·adapter 예외 타입이 나가지 않는다. adapter는 포트 경계 한 곳에서 바꾸고, fake는 같은 분류를 던진다. 새 분류는 `ObjectOperationError`에 더하고 두 쪽과 계약을 같은 변경에서 고친다. +- 그 예외의 메시지와 원인(cause)은 진단용이다. 가르는 것은 `error()`뿐이다. 예외는 하나인데, producer·consumer 자신의 실패는 그 예외가 원인으로 실린다. +- sink·source가 낸 실패는 producer·consumer에게 `ObjectChunkWriteException`·`ObjectChunkReadException`으로 간다. 요청 취소는 `IllegalStateException`으로 간다. 감싸거나 삼켜도 포트는 sink·source의 분류나 `CANCELLED`로 알린다. +- consumer는 요청 범위를 정확히 다 읽으면 돌아와도 된다. EOF를 한 번 더 읽지 않아도 된다. ## 테스트 diff --git a/src/application/objectstorage/README.md b/src/application/objectstorage/README.md index 5aab8af4..df08cca9 100644 --- a/src/application/objectstorage/README.md +++ b/src/application/objectstorage/README.md @@ -11,7 +11,7 @@ ## 공개 표면 -`port`의 포트(`ObjectTransferPort`, `ObjectInspectionPort`, `StagedObjectPublicationPort`, `ManagedObjectPublicationPort`, `ObjectRetirementPort` 등)와 값 타입. 설정 prefix는 없다(`app.object-storage.*`는 adapter의 것). +`port`의 포트(`ObjectTransferPort`, `ObjectInspectionPort`, `StagedObjectPublicationPort`, `ManagedObjectPublicationPort`, `ObjectRetirementPort` 등)와 값 타입. 포트가 던지는 실패는 `model.ObjectStorageFailureException` 하나이고 분류는 `error()`(`ObjectOperationError`)다. 설정 prefix는 없다(`app.object-storage.*`는 adapter의 것). ## 조립 diff --git a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/content/ObjectChunkSink.java b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/content/ObjectChunkSink.java index c9f19765..1f49171a 100644 --- a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/content/ObjectChunkSink.java +++ b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/content/ObjectChunkSink.java @@ -2,7 +2,13 @@ package dev.caskeleton.application.objectstorage.content; import java.util.Objects; -/** Adapter-owned bounded sink; producer arrays are valid only until {@link #write} returns. */ +/** + * Adapter-owned bounded sink; producer arrays are valid only until {@link #write} returns. + * + *

A failure of the sink itself reaches the producer as {@link ObjectChunkWriteException}, and a + * cancelled request as {@link IllegalStateException}. Let either propagate: the port reports the + * sink's own classification, or {@code CANCELLED}, even when the producer wraps or swallows it. + */ @FunctionalInterface public interface ObjectChunkSink { diff --git a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/content/ObjectChunkSource.java b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/content/ObjectChunkSource.java index d33840cc..cf993791 100644 --- a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/content/ObjectChunkSource.java +++ b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/content/ObjectChunkSource.java @@ -2,7 +2,14 @@ package dev.caskeleton.application.objectstorage.content; import java.util.Objects; -/** Adapter-owned bounded source with strict EOF and progress semantics. */ +/** + * Adapter-owned bounded source with strict EOF and progress semantics. + * + *

A failure of the source itself reaches the consumer as {@link ObjectChunkReadException}, and a + * cancelled request as {@link IllegalStateException}. Let either propagate: the port reports the + * source's own classification, or {@code CANCELLED}, even when the consumer wraps or swallows it. A + * consumer that has read the exact requested range may return without reading {@link #EOF}. + */ @FunctionalInterface public interface ObjectChunkSource { diff --git a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/model/ObjectOperationError.java b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/model/ObjectOperationError.java index 1e2c4a52..44399634 100644 --- a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/model/ObjectOperationError.java +++ b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/model/ObjectOperationError.java @@ -1,6 +1,12 @@ package dev.caskeleton.application.objectstorage.model; -/** Bounded provider-neutral operation error classification. */ +/** + * Bounded provider-neutral operation error classification, carried by receipts and by {@link + * ObjectStorageFailureException}. + * + *

A value means what its carrier says. {@code NOT_FOUND} in a resolution receipt is an ordinary + * answer (the key was never seen); in a failure it is stored data that should exist and does not. + */ public enum ObjectOperationError { NONE, INVALID_REQUEST, @@ -11,5 +17,11 @@ public enum ObjectOperationError { CORRUPT_EVIDENCE, OPERATION_EPOCH_NOT_ACTIVE, OPERATION_EXPIRED, - CANCELLED + CANCELLED, + /** The produced bytes do not match the request's declared content identity. */ + CONTENT_MISMATCH, + /** The caller's producer failed; it is the cause of the failure that carries this. */ + CONTENT_PRODUCTION_FAILED, + /** The caller's consumer failed or did not read the exact requested range. */ + CONTENT_CONSUMPTION_FAILED } diff --git a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/model/ObjectStorageFailureException.java b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/model/ObjectStorageFailureException.java new file mode 100644 index 00000000..ad43a820 --- /dev/null +++ b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/model/ObjectStorageFailureException.java @@ -0,0 +1,63 @@ +package dev.caskeleton.application.objectstorage.model; + +import java.util.Objects; + +/** + * A failure of the object store, or of the caller's own producer or consumer, as every + * object-storage port reports it: this one type and a provider-neutral {@link #error() + * classification}. No provider or adapter exception type leaves a port. + * + *

It is not a refusal of the request. Refusals stay {@link IllegalArgumentException} (the + * request as given) and {@link IllegalStateException} (the operation's current state). A port + * reports these classifications here: + * + *

+ * + *

Branch on {@link #error()}. The message and, apart from a callback's own exception, the + * {@linkplain #getCause() cause} are the implementation's, for diagnosis only. + */ +public final class ObjectStorageFailureException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final ObjectOperationError error; + + public ObjectStorageFailureException(ObjectOperationError error, String message) { + this(error, message, null); + } + + public ObjectStorageFailureException( + ObjectOperationError error, String message, Throwable cause) { + super(message, cause); + this.error = Objects.requireNonNull(error, "error must be non-null"); + if (error == ObjectOperationError.NONE) { + throw new IllegalArgumentException("a failure needs an error other than NONE"); + } + } + + /** What failed; the only part of this exception a caller branches on. */ + public ObjectOperationError error() { + return error; + } +} diff --git a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/ManagedObjectPublicationPort.java b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/ManagedObjectPublicationPort.java index 6a4facbb..7ab30a12 100644 --- a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/ManagedObjectPublicationPort.java +++ b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/ManagedObjectPublicationPort.java @@ -2,6 +2,7 @@ package dev.caskeleton.application.objectstorage.port; import dev.caskeleton.application.objectstorage.content.ObjectContentProducer; import dev.caskeleton.application.objectstorage.model.ObjectPublishReceipt; +import dev.caskeleton.application.objectstorage.model.ObjectStorageFailureException; import dev.caskeleton.application.objectstorage.request.ObjectPublishRequest; /** @@ -13,6 +14,12 @@ import dev.caskeleton.application.objectstorage.request.ObjectPublishRequest; * refuses it in the operation's current state (claimed by another, not yet verified, already * published). Both are refusals, not server faults: a use case translates them into its own outcome * or feature error rather than letting them reach a transport as they are. + * + *

Any port method may also throw {@link ObjectStorageFailureException} when the store fails or + * another call changes the same operation meanwhile ({@code PROVIDER_UNAVAILABLE}, {@code + * CONFLICT}, {@code CANCELLED}, {@code CORRUPT_EVIDENCE}, {@code NOT_FOUND}, {@code + * UNSUPPORTED_CAPABILITY}); a method that runs a producer or a consumer names the content + * classifications it adds. No provider or adapter exception type leaves a port. */ public interface ManagedObjectPublicationPort { @@ -26,9 +33,9 @@ public interface ManagedObjectPublicationPort { * @throws IllegalArgumentException the destination is unknown or scan-gated, the request does not * satisfy its policy (including content longer than its object limit), or the key already * names a different intent - * @throws RuntimeException the produced bytes do not match the declared content identity; nothing - * is published and the operation stays in progress for a retry (the exception type is the - * implementation's) + * @throws ObjectStorageFailureException {@code CONTENT_MISMATCH} when the produced bytes are not + * the declared content identity, {@code CONTENT_PRODUCTION_FAILED} when the producer throws; + * either way nothing is published and the operation stays in progress for a retry */ ObjectPublishReceipt publish(ObjectPublishRequest request, ObjectContentProducer producer); } diff --git a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/ObjectScanMaintenancePort.java b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/ObjectScanMaintenancePort.java index 02cb9ced..912557ee 100644 --- a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/ObjectScanMaintenancePort.java +++ b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/ObjectScanMaintenancePort.java @@ -3,6 +3,7 @@ package dev.caskeleton.application.objectstorage.port; import dev.caskeleton.application.objectstorage.content.ObjectContentConsumer; import dev.caskeleton.application.objectstorage.model.ObjectMutationReceipt; import dev.caskeleton.application.objectstorage.model.ObjectScanReadReceipt; +import dev.caskeleton.application.objectstorage.model.ObjectStorageFailureException; import dev.caskeleton.application.objectstorage.request.ObjectScanReadRequest; import dev.caskeleton.application.objectstorage.request.ObjectScanVerdictRequest; @@ -18,9 +19,11 @@ public interface ObjectScanMaintenancePort { * * @return the delivered range; {@code FULL_CONTENT} when the whole stage was read * @throws IllegalArgumentException the stage is unknown or the version is not the stage's - * @throws IllegalStateException the stage is not verified and awaiting a verdict - * @throws RuntimeException the stage was aborted, or the consumer failed or returned before - * reading the whole range (the exception type is the implementation's) + * @throws IllegalStateException the stage is not verified and awaiting a verdict, or it was + * aborted + * @throws ObjectStorageFailureException {@code CONTENT_CONSUMPTION_FAILED} when the consumer + * throws or returns before reading the whole range; {@code CORRUPT_EVIDENCE} when the staged + * bytes are not the stage's recorded content */ ObjectScanReadReceipt transferForScan( ObjectScanReadRequest request, ObjectContentConsumer consumer); diff --git a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/ObjectTransferPort.java b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/ObjectTransferPort.java index 5719ecf9..d693988d 100644 --- a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/ObjectTransferPort.java +++ b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/ObjectTransferPort.java @@ -2,6 +2,7 @@ package dev.caskeleton.application.objectstorage.port; import dev.caskeleton.application.objectstorage.content.ObjectContentConsumer; import dev.caskeleton.application.objectstorage.model.ObjectReadReceipt; +import dev.caskeleton.application.objectstorage.model.ObjectStorageFailureException; import dev.caskeleton.application.objectstorage.request.ObjectReadRequest; /** @@ -17,8 +18,9 @@ public interface ObjectTransferPort { * read and full verification was asked for * @throws IllegalArgumentException the reference is unknown, not published or retired, or the * expected version is not the object's - * @throws RuntimeException the consumer failed or returned before reading the whole range (the - * exception type is the implementation's) + * @throws ObjectStorageFailureException {@code CONTENT_CONSUMPTION_FAILED} when the consumer + * throws or returns before reading the whole range; {@code CORRUPT_EVIDENCE} when the stored + * bytes are not the object's recorded content */ ObjectReadReceipt transfer(ObjectReadRequest request, ObjectContentConsumer consumer); } diff --git a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/StagedObjectPublicationPort.java b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/StagedObjectPublicationPort.java index 06ea9717..d1ec658b 100644 --- a/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/StagedObjectPublicationPort.java +++ b/src/application/objectstorage/src/main/java/dev/caskeleton/application/objectstorage/port/StagedObjectPublicationPort.java @@ -4,6 +4,7 @@ import dev.caskeleton.application.objectstorage.content.ObjectContentProducer; import dev.caskeleton.application.objectstorage.model.ObjectMutationReceipt; import dev.caskeleton.application.objectstorage.model.ObjectPublishReceipt; import dev.caskeleton.application.objectstorage.model.ObjectStageReceipt; +import dev.caskeleton.application.objectstorage.model.ObjectStorageFailureException; import dev.caskeleton.application.objectstorage.model.ObjectVerificationReceipt; import dev.caskeleton.application.objectstorage.request.ObjectAbortRequest; import dev.caskeleton.application.objectstorage.request.ObjectFinalizeRequest; @@ -24,8 +25,9 @@ public interface StagedObjectPublicationPort { * already staged the same key and intent, without calling the producer * @throws IllegalArgumentException the destination is unknown, the request does not satisfy its * policy, or the key already names a different intent - * @throws RuntimeException the produced bytes do not match the declared content identity (the - * exception type is the implementation's) + * @throws ObjectStorageFailureException {@code CONTENT_MISMATCH} when the produced bytes are not + * the declared content identity, {@code CONTENT_PRODUCTION_FAILED} when the producer throws; + * either way nothing is staged and the operation stays in progress for a retry */ ObjectStageReceipt stage(ObjectStageRequest request, ObjectContentProducer producer); diff --git a/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/InMemoryObjectStorage.java b/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/InMemoryObjectStorage.java index cfeb7b3c..43d76450 100644 --- a/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/InMemoryObjectStorage.java +++ b/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/InMemoryObjectStorage.java @@ -3,8 +3,10 @@ package dev.caskeleton.application.objectstorage.testkit; import dev.caskeleton.application.idempotency.RequestFingerprint; import dev.caskeleton.application.objectstorage.content.CallBudget; import dev.caskeleton.application.objectstorage.content.CancellationView; +import dev.caskeleton.application.objectstorage.content.ObjectChunkReadException; import dev.caskeleton.application.objectstorage.content.ObjectChunkSink; import dev.caskeleton.application.objectstorage.content.ObjectChunkSource; +import dev.caskeleton.application.objectstorage.content.ObjectChunkWriteException; import dev.caskeleton.application.objectstorage.content.ObjectContentConsumer; import dev.caskeleton.application.objectstorage.content.ObjectContentConsumptionException; import dev.caskeleton.application.objectstorage.content.ObjectContentProducer; @@ -38,6 +40,7 @@ import dev.caskeleton.application.objectstorage.model.ObjectRetentionRequirement import dev.caskeleton.application.objectstorage.model.ObjectScanReadReceipt; import dev.caskeleton.application.objectstorage.model.ObjectScanState; import dev.caskeleton.application.objectstorage.model.ObjectStageReceipt; +import dev.caskeleton.application.objectstorage.model.ObjectStorageFailureException; import dev.caskeleton.application.objectstorage.model.ObjectVerificationReceipt; import dev.caskeleton.application.objectstorage.port.ManagedObjectPublicationPort; import dev.caskeleton.application.objectstorage.port.ObjectInspectionPort; @@ -600,13 +603,18 @@ public final class InMemoryObjectStorage return new ObjectMutationReceipt(key, outcome, error, now()); } - /** The producer's bytes, refused unless they are exactly the declared content. */ + /** + * The producer's bytes, refused unless they are exactly the declared content. A failure the sink + * raised wins over what the producer made of it; any other producer failure is the producer's. + */ private static byte[] produced( ObjectContentProducer producer, ObjectContentIdentity declared, CallBudget budget, CancellationView cancellation) { + requireBudget(budget); ByteArrayOutputStream produced = new ByteArrayOutputStream(); + ObjectStorageFailureException[] raised = {null}; ObjectContentProductionContext context = ObjectContentProductionContext.open(budget, cancellation, CHUNK_BYTES); try { @@ -615,23 +623,80 @@ public final class InMemoryObjectStorage ObjectChunkSink.scoped( context, (bytes, offset, length) -> { + if (budget.isExpiredAt(System.nanoTime())) { + raised[0] = budgetSpent(); + throw new ObjectChunkWriteException(raised[0].getMessage()); + } if (produced.size() + (long) length > declared.exactLength()) { - throw new IllegalStateException("producer exceeded the declared length"); + raised[0] = + new ObjectStorageFailureException( + ObjectOperationError.CONTENT_MISMATCH, + "producer exceeded the declared length"); + throw new ObjectChunkWriteException(raised[0].getMessage()); } produced.write(bytes, offset, length); })); - } catch (ObjectContentProductionException failure) { - throw new IllegalStateException("content production failed", failure); + } catch (ObjectContentProductionException | RuntimeException failure) { + throw callbackFailure( + raised[0], + cancellation, + failure, + ObjectOperationError.CONTENT_PRODUCTION_FAILED, + "content production failed"); } finally { context.invalidate(); } + if (raised[0] != null) { + throw raised[0]; + } + if (cancellation.isCancelled()) { + throw cancelled(null); + } byte[] content = produced.toByteArray(); if (!identity(content).equals(declared)) { - throw new IllegalStateException("produced content conflicts with declared identity"); + throw new ObjectStorageFailureException( + ObjectOperationError.CONTENT_MISMATCH, + "produced content conflicts with declared identity"); } return content; } + /** The request's own call budget ran out before or during the transfer, as the adapter says. */ + private static void requireBudget(CallBudget budget) { + if (budget.isExpiredAt(System.nanoTime())) { + throw budgetSpent(); + } + } + + private static ObjectStorageFailureException budgetSpent() { + return new ObjectStorageFailureException( + ObjectOperationError.CANCELLED, "content transfer call budget ran out"); + } + + /** + * What a producer's or consumer's failure is reported as: the failure the fake raised inside the + * callback, else cancellation when the request was cancelled, else the callback's own failure. + */ + private static ObjectStorageFailureException callbackFailure( + ObjectStorageFailureException raised, + CancellationView cancellation, + Exception failure, + ObjectOperationError callbackError, + String message) { + if (raised != null) { + return raised; + } + if (cancellation.isCancelled()) { + return cancelled(failure); + } + return new ObjectStorageFailureException(callbackError, message, failure); + } + + private static ObjectStorageFailureException cancelled(Exception cause) { + return new ObjectStorageFailureException( + ObjectOperationError.CANCELLED, "content transfer was cancelled", cause); + } + private static void deliver( Operation operation, ObjectReadRange range, @@ -649,12 +714,18 @@ public final class InMemoryObjectStorage operation.intent.contentIdentity(), operation.version, range); + requireBudget(budget); int end = (int) range.endExclusive(); int[] position = {(int) range.offset()}; + ObjectStorageFailureException[] raised = {null}; ObjectChunkSource source = ObjectChunkSource.scoped( context, (destination, offset, length) -> { + if (budget.isExpiredAt(System.nanoTime())) { + raised[0] = budgetSpent(); + throw new ObjectChunkReadException(raised[0].getMessage()); + } if (position[0] >= end) { return ObjectChunkSource.EOF; } @@ -666,12 +737,25 @@ public final class InMemoryObjectStorage try { consumer.consume(context, source); } catch (ObjectContentConsumptionException | RuntimeException failure) { - throw new IllegalStateException("bounded content consumption failed", failure); + throw callbackFailure( + raised[0], + cancellation, + failure, + ObjectOperationError.CONTENT_CONSUMPTION_FAILED, + "content consumption failed"); } finally { context.invalidate(); } + if (raised[0] != null) { + throw raised[0]; + } + if (cancellation.isCancelled()) { + throw cancelled(null); + } if (position[0] != end) { - throw new IllegalStateException("consumer did not read the exact requested range"); + throw new ObjectStorageFailureException( + ObjectOperationError.CONTENT_CONSUMPTION_FAILED, + "consumer did not read the exact requested range"); } } diff --git a/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/ManagedObjectStoragePortContract.java b/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/ManagedObjectStoragePortContract.java index d41f980d..7a94cea2 100644 --- a/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/ManagedObjectStoragePortContract.java +++ b/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/ManagedObjectStoragePortContract.java @@ -4,6 +4,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.application.objectstorage.content.CancellationView; +import dev.caskeleton.application.objectstorage.content.ObjectContentConsumptionException; +import dev.caskeleton.application.objectstorage.content.ObjectContentProductionException; import dev.caskeleton.application.objectstorage.identity.ObjectOperationKey; import dev.caskeleton.application.objectstorage.identity.ObjectReference; import dev.caskeleton.application.objectstorage.identity.ObjectVersionToken; @@ -18,6 +20,7 @@ import dev.caskeleton.application.objectstorage.model.ObjectPublishReceipt; import dev.caskeleton.application.objectstorage.model.ObjectReadRange; import dev.caskeleton.application.objectstorage.model.ObjectReadReceipt; import dev.caskeleton.application.objectstorage.model.ObjectScanState; +import dev.caskeleton.application.objectstorage.model.ObjectStorageFailureException; import dev.caskeleton.application.objectstorage.port.ManagedObjectPublicationPort; import dev.caskeleton.application.objectstorage.port.ObjectInspectionPort; import dev.caskeleton.application.objectstorage.port.ObjectOperationResolutionPort; @@ -252,11 +255,112 @@ public abstract class ManagedObjectStoragePortContract extends ObjectStoragePort .publish( publishRequest(key("publish-mismatch"), declared), producing(produced))); + // then + assertPortFailure(failed, ObjectOperationError.CONTENT_MISMATCH); + } + + @Test + @DisplayName("선언한 길이보다 많이 쓰는 producer의 게시는 CONTENT_MISMATCH다") + void aProducerThatWritesPastTheDeclaredLengthIsAContentMismatch() { + // given + byte[] declared = content(200, 80); + byte[] longer = content(216, 80); + + // when + Throwable failed = + catchThrowable( + () -> + publication() + .publish(publishRequest(key("publish-too-long"), declared), producing(longer))); + + // then + assertPortFailure(failed, ObjectOperationError.CONTENT_MISMATCH); + } + + @Test + @DisplayName("producer가 스스로 실패하면 CONTENT_PRODUCTION_FAILED이고 원인은 그 예외다") + void aProducerFailureCarriesTheProducersOwnException() { + // given + ObjectContentProductionException producerFailure = + new ObjectContentProductionException("upload stream broke"); + + // when + Throwable failed = + catchThrowable( + () -> + publication() + .publish( + publishRequest(key("publish-producer-failed"), content(200, 81)), + failingWith(producerFailure))); + // then assertThat(failed) - .as("the port refuses; which unchecked exception is the implementation's") - .isInstanceOf(RuntimeException.class) - .isNotInstanceOf(NullPointerException.class); + .isInstanceOfSatisfying( + ObjectStorageFailureException.class, + portFailure -> + assertThat(portFailure) + .extracting(ObjectStorageFailureException::error, Throwable::getCause) + .containsExactly( + ObjectOperationError.CONTENT_PRODUCTION_FAILED, producerFailure)); + } + + @Test + @DisplayName("call budget이 다 된 게시는 CANCELLED다") + void aPublicationWhoseBudgetRanOutIsCancelled() { + // given + byte[] content = content(200, 86); + + // when + Throwable failed = + catchThrowable( + () -> + publication() + .publish( + publishRequest(key("publish-budget-spent"), content, spentBudget()), + producing(content))); + + // then + assertPortFailure(failed, ObjectOperationError.CANCELLED); + } + + @Test + @DisplayName("취소를 삼키고 돌아온 producer의 게시도 CANCELLED다") + void aPublicationWhoseProducerSwallowsTheCancellationIsCancelled() { + // given + byte[] content = content(200, 88); + AtomicBoolean cancelled = new AtomicBoolean(); + + // when + Throwable failed = + catchThrowable( + () -> + publication() + .publish( + publishRequest(key("publish-swallowed-cancel"), content, cancelled::get), + swallowingCancellationAfterFirstChunk(content, cancelled))); + + // then + assertPortFailure(failed, ObjectOperationError.CANCELLED); + } + + @Test + @DisplayName("만드는 도중 취소된 게시는 CANCELLED다") + void aPublicationCancelledWhileProducingIsCancelled() { + // given + byte[] content = content(200, 82); + AtomicBoolean cancelled = new AtomicBoolean(); + + // when + Throwable failed = + catchThrowable( + () -> + publication() + .publish( + publishRequest(key("publish-cancelled"), content, cancelled::get), + cancellingAfterFirstChunk(content, cancelled))); + + // then + assertPortFailure(failed, ObjectOperationError.CANCELLED); } @Test @@ -363,11 +467,74 @@ public abstract class ManagedObjectStoragePortContract extends ObjectStoragePort readingOneChunk())); // then - assertThat(refused) - .as( - "a consumer must read the exact range; which unchecked exception is the implementation's") - .isInstanceOf(RuntimeException.class) - .isNotInstanceOf(NullPointerException.class); + assertPortFailure(refused, ObjectOperationError.CONTENT_CONSUMPTION_FAILED); + } + + @Test + @DisplayName("범위를 정확히 읽고 EOF를 읽지 않고 돌아온 consumer의 전송은 성공한다") + void aConsumerThatReadsTheExactRangeWithoutEofSucceeds() { + // given + byte[] content = content(200, 89); + ObjectPublishReceipt published = publish("transfer-exact-no-eof", content); + ExactReader exact = new ExactReader(content.length); + + // when + transfer().transfer(read(published.reference(), Optional.empty(), 0, content.length), exact); + + // then + assertThat(exact.bytes()).containsExactly(content); + } + + @Test + @DisplayName("call budget이 다 된 전송은 CANCELLED다") + void aTransferWhoseBudgetRanOutIsCancelled() { + // given + byte[] content = content(200, 87); + ObjectPublishReceipt published = publish("transfer-budget-spent", content); + ObjectReadRequest spent = + new ObjectReadRequest( + published.reference(), + Optional.empty(), + ObjectReadRange.of(0, content.length), + ObjectDigestVerification.FULL_CONTENT, + content.length, + spentBudget(), + CancellationView.never()); + + // when + Throwable failed = catchThrowable(() -> transfer().transfer(spent, new Collected())); + + // then + assertPortFailure(failed, ObjectOperationError.CANCELLED); + } + + @Test + @DisplayName("consumer가 스스로 실패하면 CONTENT_CONSUMPTION_FAILED이고 원인은 그 예외다") + void aConsumerFailureCarriesTheConsumersOwnException() { + // given + byte[] content = content(200, 83); + ObjectPublishReceipt published = publish("transfer-consumer-failed", content); + ObjectContentConsumptionException consumerFailure = + new ObjectContentConsumptionException("client went away"); + + // when + Throwable failed = + catchThrowable( + () -> + transfer() + .transfer( + read(published.reference(), Optional.empty(), 0, content.length), + failingAfterOneChunk(consumerFailure))); + + // then + assertThat(failed) + .isInstanceOfSatisfying( + ObjectStorageFailureException.class, + portFailure -> + assertThat(portFailure) + .extracting(ObjectStorageFailureException::error, Throwable::getCause) + .containsExactly( + ObjectOperationError.CONTENT_CONSUMPTION_FAILED, consumerFailure)); } @Test diff --git a/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/ObjectScanMaintenancePortContract.java b/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/ObjectScanMaintenancePortContract.java index 9d765e4c..2d07c3ce 100644 --- a/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/ObjectScanMaintenancePortContract.java +++ b/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/ObjectScanMaintenancePortContract.java @@ -11,6 +11,7 @@ import dev.caskeleton.application.objectstorage.model.ObjectDigestVerification; import dev.caskeleton.application.objectstorage.model.ObjectHandoffReceipt; import dev.caskeleton.application.objectstorage.model.ObjectMutationOutcome; import dev.caskeleton.application.objectstorage.model.ObjectMutationReceipt; +import dev.caskeleton.application.objectstorage.model.ObjectOperationError; import dev.caskeleton.application.objectstorage.model.ObjectOperationResolution; import dev.caskeleton.application.objectstorage.model.ObjectPublicationState; import dev.caskeleton.application.objectstorage.model.ObjectPublishReceipt; @@ -133,11 +134,26 @@ public abstract class ObjectScanMaintenancePortContract extends ObjectStoragePor scan().transferForScan(scanRead(key, stage, 0, content.length), readingOneChunk())); // then - assertThat(refused) - .as( - "a scanner must read the exact range; which unchecked exception is the implementation's") - .isInstanceOf(RuntimeException.class) - .isNotInstanceOf(NullPointerException.class); + assertPortFailure(refused, ObjectOperationError.CONTENT_CONSUMPTION_FAILED); + } + + @Test + @DisplayName("중단한 stage의 scan 전송은 거절한다") + void anAbortedStageIsRefusedForScan() { + // given + ObjectOperationKey key = key("scan-transfer-aborted"); + byte[] content = content(200, 85); + ObjectStageReceipt stage = verifiedStage(key, content); + abortReleased(key, stage); + Collected collected = new Collected(); + + // when + Throwable refused = + catchThrowable( + () -> scan().transferForScan(scanRead(key, stage, 0, content.length), collected)); + + // then + assertThat(refused).isInstanceOf(IllegalStateException.class); } @Test diff --git a/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/ObjectStoragePortContractSupport.java b/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/ObjectStoragePortContractSupport.java index f32fbff5..0d353511 100644 --- a/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/ObjectStoragePortContractSupport.java +++ b/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/ObjectStoragePortContractSupport.java @@ -1,11 +1,16 @@ package dev.caskeleton.application.objectstorage.testkit; +import static org.assertj.core.api.Assertions.assertThat; + import dev.caskeleton.application.objectstorage.content.CallBudget; import dev.caskeleton.application.objectstorage.content.CancellationView; import dev.caskeleton.application.objectstorage.content.ObjectChunkReadException; import dev.caskeleton.application.objectstorage.content.ObjectChunkSource; +import dev.caskeleton.application.objectstorage.content.ObjectChunkWriteException; import dev.caskeleton.application.objectstorage.content.ObjectContentConsumer; +import dev.caskeleton.application.objectstorage.content.ObjectContentConsumptionException; import dev.caskeleton.application.objectstorage.content.ObjectContentProducer; +import dev.caskeleton.application.objectstorage.content.ObjectContentProductionException; import dev.caskeleton.application.objectstorage.content.ObjectContentReadContext; import dev.caskeleton.application.objectstorage.identity.ObjectOperationEpoch; import dev.caskeleton.application.objectstorage.identity.ObjectOperationId; @@ -14,6 +19,8 @@ import dev.caskeleton.application.objectstorage.identity.ObjectReference; import dev.caskeleton.application.objectstorage.model.ObjectContentIdentity; import dev.caskeleton.application.objectstorage.model.ObjectDigest; import dev.caskeleton.application.objectstorage.model.ObjectMediaType; +import dev.caskeleton.application.objectstorage.model.ObjectOperationError; +import dev.caskeleton.application.objectstorage.model.ObjectStorageFailureException; import dev.caskeleton.application.objectstorage.request.ObjectPublishRequest; import dev.caskeleton.application.objectstorage.request.ObjectStageRequest; import java.io.ByteArrayOutputStream; @@ -54,6 +61,23 @@ public abstract class ObjectStoragePortContractSupport { /** A managed publication of {@code content} that satisfies the destination policy. */ protected final ObjectPublishRequest publishRequest(ObjectOperationKey key, byte[] content) { + return publishRequest(key, content, CancellationView.never()); + } + + /** The same publication, cancelled whenever {@code cancellation} says so. */ + protected final ObjectPublishRequest publishRequest( + ObjectOperationKey key, byte[] content, CancellationView cancellation) { + return publishRequest(key, content, budget(), cancellation); + } + + /** The same publication under {@code budget}. */ + protected final ObjectPublishRequest publishRequest( + ObjectOperationKey key, byte[] content, CallBudget budget) { + return publishRequest(key, content, budget, CancellationView.never()); + } + + private ObjectPublishRequest publishRequest( + ObjectOperationKey key, byte[] content, CallBudget budget, CancellationView cancellation) { ObjectStorageDestinationPolicy destination = destination(); return new ObjectPublishRequest( key, @@ -63,8 +87,8 @@ public abstract class ObjectStoragePortContractSupport { destination.retentionRequirement(), destination.encryptionRequirement(), destination.capabilityRequirements(), - budget(), - CancellationView.never()); + budget, + cancellation); } /** A stage of {@code content} that satisfies the destination policy. */ @@ -105,6 +129,43 @@ public abstract class ObjectStoragePortContractSupport { }; } + /** A producer that writes the first chunk, sets {@code cancelled}, then writes the rest. */ + protected static ObjectContentProducer cancellingAfterFirstChunk( + byte[] content, AtomicBoolean cancelled) { + return (context, sink) -> { + sink.write(content, 0, CHUNK); + cancelled.set(true); + for (int offset = CHUNK; offset < content.length; offset += CHUNK) { + sink.write(content, offset, Math.min(CHUNK, content.length - offset)); + } + }; + } + + /** + * A producer that writes the first chunk, sets {@code cancelled}, then swallows the refusal of + * the next write and returns as if it had finished. + */ + protected static ObjectContentProducer swallowingCancellationAfterFirstChunk( + byte[] content, AtomicBoolean cancelled) { + return (context, sink) -> { + sink.write(content, 0, CHUNK); + cancelled.set(true); + try { + sink.write(content, CHUNK, CHUNK); + } catch (ObjectChunkWriteException | IllegalStateException refused) { + // the producer gives up without saying so + cancelled.set(true); + } + }; + } + + /** A producer that writes nothing and fails with {@code failure}, its own exception. */ + protected static ObjectContentProducer failingWith(ObjectContentProductionException failure) { + return (context, sink) -> { + throw failure; + }; + } + /** A producer that records whether it was called and writes nothing. */ protected static ObjectContentProducer recording(AtomicBoolean called) { return (context, sink) -> called.set(true); @@ -122,6 +183,12 @@ public abstract class ObjectStoragePortContractSupport { return CallBudget.fromNow(Duration.ofMinutes(1)); } + /** A call budget that ran out a second ago. */ + protected static CallBudget spentBudget() { + return CallBudget.after( + System.nanoTime() - Duration.ofSeconds(2).toNanos(), Duration.ofSeconds(1)); + } + /** * A canonical reference on the same route as {@code published} that names no object: a different * object id with its own check digits. @@ -138,6 +205,52 @@ public abstract class ObjectStoragePortContractSupport { return (context, source) -> source.read(new byte[CHUNK], 0, CHUNK); } + /** A consumer that reads one chunk and then fails with {@code failure}, its own exception. */ + protected static ObjectContentConsumer failingAfterOneChunk( + ObjectContentConsumptionException failure) { + return (context, source) -> { + var unused = source.read(new byte[CHUNK], 0, CHUNK); + throw failure; + }; + } + + /** Asserts that {@code failure} is the port failure classified as {@code error}. */ + protected static void assertPortFailure(Throwable failure, ObjectOperationError error) { + assertThat(failure) + .isInstanceOfSatisfying( + ObjectStorageFailureException.class, + portFailure -> assertThat(portFailure.error()).isEqualTo(error)); + } + + /** A consumer that reads exactly {@code length} bytes and returns without reading EOF. */ + protected static final class ExactReader implements ObjectContentConsumer { + + private final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + private final int length; + + public ExactReader(int length) { + this.length = length; + } + + @Override + public void consume(ObjectContentReadContext context, ObjectChunkSource source) + throws ObjectChunkReadException { + byte[] buffer = new byte[CHUNK]; + while (bytes.size() < length) { + int count = source.read(buffer, 0, Math.min(CHUNK, length - bytes.size())); + if (count == ObjectChunkSource.EOF) { + return; + } + bytes.write(buffer, 0, count); + } + } + + /** What the transfer delivered. */ + public byte[] bytes() { + return bytes.toByteArray(); + } + } + /** A consumer that keeps every byte it is given. */ protected static final class Collected implements ObjectContentConsumer { diff --git a/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/StagedObjectPublicationPortContract.java b/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/StagedObjectPublicationPortContract.java index fd0e61ff..cdd66e64 100644 --- a/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/StagedObjectPublicationPortContract.java +++ b/src/application/objectstorage/src/testFixtures/java/dev/caskeleton/application/objectstorage/testkit/StagedObjectPublicationPortContract.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.application.objectstorage.content.CancellationView; +import dev.caskeleton.application.objectstorage.content.ObjectContentProductionException; import dev.caskeleton.application.objectstorage.identity.ObjectOperationKey; import dev.caskeleton.application.objectstorage.model.ObjectHandoffReceipt; import dev.caskeleton.application.objectstorage.model.ObjectMutationOutcome; @@ -13,6 +14,7 @@ import dev.caskeleton.application.objectstorage.model.ObjectOperationResolution; import dev.caskeleton.application.objectstorage.model.ObjectPublicationState; import dev.caskeleton.application.objectstorage.model.ObjectPublishReceipt; import dev.caskeleton.application.objectstorage.model.ObjectStageReceipt; +import dev.caskeleton.application.objectstorage.model.ObjectStorageFailureException; import dev.caskeleton.application.objectstorage.model.ObjectVerificationReceipt; import dev.caskeleton.application.objectstorage.port.ObjectInspectionPort; import dev.caskeleton.application.objectstorage.port.ObjectOperationResolutionPort; @@ -123,11 +125,35 @@ public abstract class StagedObjectPublicationPortContract extends ObjectStorageP () -> staged().stage(stageRequest(key("stage-mismatch"), declared), producing(produced))); + // then + assertPortFailure(failed, ObjectOperationError.CONTENT_MISMATCH); + } + + @Test + @DisplayName("producer가 스스로 실패한 stage는 CONTENT_PRODUCTION_FAILED이고 원인은 그 예외다") + void aStageWhoseProducerFailsIsAProductionFailure() { + // given + ObjectContentProductionException producerFailure = + new ObjectContentProductionException("upload stream broke"); + + // when + Throwable failed = + catchThrowable( + () -> + staged() + .stage( + stageRequest(key("stage-producer-failed"), content(200, 84)), + failingWith(producerFailure))); + // then assertThat(failed) - .as("the port refuses; which unchecked exception is the implementation's") - .isInstanceOf(RuntimeException.class) - .isNotInstanceOf(NullPointerException.class); + .isInstanceOfSatisfying( + ObjectStorageFailureException.class, + portFailure -> + assertThat(portFailure) + .extracting(ObjectStorageFailureException::error, Throwable::getCause) + .containsExactly( + ObjectOperationError.CONTENT_PRODUCTION_FAILED, producerFailure)); } @Test -- 2.54.0