refactor: 각 어댑터터별 리펙토링 진행

This commit is contained in:
DongHyeonka
2026-08-24 18:26:40 +09:00
parent 16e33ce986
commit a261df2a2b
100 changed files with 7072 additions and 902 deletions
@@ -157,11 +157,14 @@ public final class AvroMessageCodec implements MessageCodec {
"the Avro codec decodes into GenericRecord, not " + payloadType.getSimpleName());
}
DatumReader<GenericRecord> reader = new GenericDatumReader<>(writerSchema, writerSchema);
DatumReader<GenericRecord> reader = boundedReader(writerSchema, writerSchema);
BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(encoded, null);
try {
return payloadType.cast(reader.read(null, decoder));
} catch (IOException | RuntimeException failure) {
if (failure instanceof MessageTooLargeException tooLarge) {
throw tooLarge;
}
throw new MessageSerializationException(
"AVRO_DECODE_FAILED",
"the payload could not be decoded against the registered schema",
@@ -200,11 +203,14 @@ public final class AvroMessageCodec implements MessageCodec {
Schema writerSchema = schemaFor(type, writerVersion);
Schema readerSchema = schemaFor(type, readerVersion);
DatumReader<GenericRecord> reader = new GenericDatumReader<>(writerSchema, readerSchema);
DatumReader<GenericRecord> reader = boundedReader(writerSchema, readerSchema);
BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(encoded, null);
try {
return reader.read(null, decoder);
} catch (IOException | RuntimeException failure) {
if (failure instanceof MessageTooLargeException tooLarge) {
throw tooLarge;
}
throw new MessageSerializationException(
"AVRO_EVOLUTION_FAILED",
"v%d bytes could not be resolved against the v%d reader schema"
@@ -213,6 +219,38 @@ public final class AvroMessageCodec implements MessageCodec {
}
}
/**
* Builds a reader that will not allocate an array the input could not possibly fill.
*
* <p>Avro writes an array as a declared element count followed by the elements. The count is a
* variable-length integer, so five bytes can claim four hundred million elements, and the generic
* reader allocates the backing array from that claim before reading a single element. Bounding
* the input length does not help: the whole hostile payload is five bytes, well under any limit,
* and the failure is an {@code OutOfMemoryError} rather than an exception the codec could report
* — on a consumer thread that is the process, not the message.
*
* <p>The ceiling is the byte limit itself. Every element costs at least one byte on the wire even
* when it is empty, so a payload of at most {@code maxBytes} bytes cannot honestly contain more
* than {@code maxBytes} elements, and any larger claim is a lie the reader should refuse rather
* than reserve memory for.
*/
private DatumReader<GenericRecord> boundedReader(Schema writerSchema, Schema readerSchema) {
int maxElements = maxBytes;
return new GenericDatumReader<>(writerSchema, readerSchema) {
@Override
protected Object newArray(Object old, int size, Schema schema) {
if (size > maxElements) {
throw new MessageTooLargeException(
"AVRO_COLLECTION_TOO_LARGE",
("the payload declares an array of %d elements, which %d bytes cannot contain; the"
+ " claim was refused before the array was allocated")
.formatted(size, maxElements));
}
return super.newArray(old, size, schema);
}
};
}
private Schema schemaFor(MessageType type, SchemaVersion version) {
Objects.requireNonNull(type, "type must not be null");
Objects.requireNonNull(version, "version must not be null");
@@ -0,0 +1,134 @@
package dev.caskeleton.messaging.schema.avro;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import dev.caskeleton.messaging.api.MessageType;
import dev.caskeleton.messaging.api.SchemaVersion;
import dev.caskeleton.messaging.api.error.MessageSerializationException;
import dev.caskeleton.messaging.api.error.MessageTooLargeException;
import java.io.ByteArrayOutputStream;
import java.util.List;
import java.util.Map;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
/**
* What a hostile Avro payload can make the decoder allocate.
*
* <p>The byte limit is the wrong instrument for this attack and was the only one in place. Avro
* writes a collection as a declared element count followed by the elements, and the count is a
* variable-length integer: five bytes claim four hundred million elements. Those five bytes pass
* every length check the codec applies, and the generic reader then reserves the backing array from
* the claim before reading a single element. The result is an {@code OutOfMemoryError} — not an
* exception the codec can classify, not a message the consumer can dead-letter, and on a broker
* consumer thread it takes the process with it.
*
* <p>Strings, byte arrays and maps were already safe: Avro validates those lengths against the
* bytes actually remaining. Arrays were the one shape that allocated on trust, which is why the
* corpus below is narrow rather than exhaustive — it pins the case that failed, and the two cases
* that must keep working around it.
*/
class AvroHostileInputTest {
private static final MessageType BASKET = new MessageType("basket.filled");
private static final SchemaVersion V1 = new SchemaVersion(1);
private static final SchemaVersion V2 = new SchemaVersion(2);
private static final Schema V1_SCHEMA =
new Schema.Parser()
.parse(
"""
{
"type": "record",
"name": "BasketFilled",
"namespace": "dev.caskeleton.messaging.sample",
"fields": [
{"name": "items", "type": {"type": "array", "items": "string"}}
]
}
""");
private static final Schema V2_SCHEMA =
new Schema.Parser()
.parse(
"""
{
"type": "record",
"name": "BasketFilled",
"namespace": "dev.caskeleton.messaging.sample",
"fields": [
{"name": "items", "type": {"type": "array", "items": "string"}},
{"name": "channel", "type": "string", "default": "WEB"}
]
}
""");
@Test
@DisplayName("an array length no payload of this size could hold is refused before allocation")
void aDeclaredArrayLengthTheInputCannotContainIsRefused() {
AvroMessageCodec codec = new AvroMessageCodec(Map.of(BASKET, Map.of(V1, V1_SCHEMA)));
byte[] hostile = zigzag(400_000_000L);
assertThat(hostile)
.as("the whole attack is five bytes, so no byte limit stands between it and the allocation")
.hasSizeLessThan(16);
assertThatThrownBy(() -> codec.decode(BASKET, V1, hostile, GenericRecord.class))
.isInstanceOf(MessageTooLargeException.class)
.hasMessageContaining("400000000");
}
@Test
@DisplayName("the evolution decode refuses the same claim")
void theEvolutionDecodeAppliesTheSameCollectionBound() {
AvroMessageCodec codec =
new AvroMessageCodec(Map.of(BASKET, Map.of(V1, V1_SCHEMA, V2, V2_SCHEMA)));
assertThatThrownBy(() -> codec.decodeEvolved(BASKET, V1, V2, zigzag(400_000_000L)))
.as("a consumer reading a newer producer takes this path for every message")
.isInstanceOf(MessageTooLargeException.class);
}
@Test
@DisplayName("a count too large for an int fails as a decode error, not an allocation")
void aCountBeyondIntRangeFailsWhileReadingRatherThanWhileReserving() {
AvroMessageCodec codec = new AvroMessageCodec(Map.of(BASKET, Map.of(V1, V1_SCHEMA)));
assertThatThrownBy(() -> codec.decode(BASKET, V1, zigzag(4_294_967_296L), GenericRecord.class))
.as("the count truncates to a harmless int, and the read then runs out of input")
.isInstanceOf(MessageSerializationException.class);
}
@Test
@DisplayName("an array a payload really contains still round trips")
void anHonestArrayIsUnaffectedByTheBound() {
AvroMessageCodec codec = new AvroMessageCodec(Map.of(BASKET, Map.of(V1, V1_SCHEMA)));
GenericRecord basket = new GenericData.Record(V1_SCHEMA);
basket.put("items", List.of("apple", "pear", "plum"));
byte[] encoded = codec.encode(BASKET, V1, basket).bytes();
GenericRecord decoded = codec.decode(BASKET, V1, encoded, GenericRecord.class);
assertThat(decoded.get("items").toString()).isEqualTo("[apple, pear, plum]");
}
/**
* Writes one Avro variable-length integer, the way a producer encodes a collection count.
*
* <p>Hand-written rather than taken from an encoder because the point is to write a count with no
* elements behind it, which no encoder will do.
*/
private static byte[] zigzag(long value) {
long remaining = (value << 1) ^ (value >> 63);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
while ((remaining & ~0x7FL) != 0) {
bytes.write((int) ((remaining & 0x7F) | 0x80));
remaining >>>= 7;
}
bytes.write((int) remaining);
return bytes.toByteArray();
}
}