refactor: 각 어댑터터별 리펙토링 진행
This commit is contained in:
+40
-2
@@ -157,11 +157,14 @@ public final class AvroMessageCodec implements MessageCodec {
|
||||
"the Avro codec decodes into GenericRecord, not " + payloadType.getSimpleName());
|
||||
}
|
||||
|
||||
DatumReader<GenericRecord> reader = new GenericDatumReader<>(writerSchema, writerSchema);
|
||||
DatumReader<GenericRecord> reader = boundedReader(writerSchema, writerSchema);
|
||||
BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(encoded, null);
|
||||
try {
|
||||
return payloadType.cast(reader.read(null, decoder));
|
||||
} catch (IOException | RuntimeException failure) {
|
||||
if (failure instanceof MessageTooLargeException tooLarge) {
|
||||
throw tooLarge;
|
||||
}
|
||||
throw new MessageSerializationException(
|
||||
"AVRO_DECODE_FAILED",
|
||||
"the payload could not be decoded against the registered schema",
|
||||
@@ -200,11 +203,14 @@ public final class AvroMessageCodec implements MessageCodec {
|
||||
Schema writerSchema = schemaFor(type, writerVersion);
|
||||
Schema readerSchema = schemaFor(type, readerVersion);
|
||||
|
||||
DatumReader<GenericRecord> reader = new GenericDatumReader<>(writerSchema, readerSchema);
|
||||
DatumReader<GenericRecord> reader = boundedReader(writerSchema, readerSchema);
|
||||
BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(encoded, null);
|
||||
try {
|
||||
return reader.read(null, decoder);
|
||||
} catch (IOException | RuntimeException failure) {
|
||||
if (failure instanceof MessageTooLargeException tooLarge) {
|
||||
throw tooLarge;
|
||||
}
|
||||
throw new MessageSerializationException(
|
||||
"AVRO_EVOLUTION_FAILED",
|
||||
"v%d bytes could not be resolved against the v%d reader schema"
|
||||
@@ -213,6 +219,38 @@ public final class AvroMessageCodec implements MessageCodec {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a reader that will not allocate an array the input could not possibly fill.
|
||||
*
|
||||
* <p>Avro writes an array as a declared element count followed by the elements. The count is a
|
||||
* variable-length integer, so five bytes can claim four hundred million elements, and the generic
|
||||
* reader allocates the backing array from that claim before reading a single element. Bounding
|
||||
* the input length does not help: the whole hostile payload is five bytes, well under any limit,
|
||||
* and the failure is an {@code OutOfMemoryError} rather than an exception the codec could report
|
||||
* — on a consumer thread that is the process, not the message.
|
||||
*
|
||||
* <p>The ceiling is the byte limit itself. Every element costs at least one byte on the wire even
|
||||
* when it is empty, so a payload of at most {@code maxBytes} bytes cannot honestly contain more
|
||||
* than {@code maxBytes} elements, and any larger claim is a lie the reader should refuse rather
|
||||
* than reserve memory for.
|
||||
*/
|
||||
private DatumReader<GenericRecord> boundedReader(Schema writerSchema, Schema readerSchema) {
|
||||
int maxElements = maxBytes;
|
||||
return new GenericDatumReader<>(writerSchema, readerSchema) {
|
||||
@Override
|
||||
protected Object newArray(Object old, int size, Schema schema) {
|
||||
if (size > maxElements) {
|
||||
throw new MessageTooLargeException(
|
||||
"AVRO_COLLECTION_TOO_LARGE",
|
||||
("the payload declares an array of %d elements, which %d bytes cannot contain; the"
|
||||
+ " claim was refused before the array was allocated")
|
||||
.formatted(size, maxElements));
|
||||
}
|
||||
return super.newArray(old, size, schema);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private Schema schemaFor(MessageType type, SchemaVersion version) {
|
||||
Objects.requireNonNull(type, "type must not be null");
|
||||
Objects.requireNonNull(version, "version must not be null");
|
||||
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
package dev.caskeleton.messaging.schema.avro;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.messaging.api.MessageType;
|
||||
import dev.caskeleton.messaging.api.SchemaVersion;
|
||||
import dev.caskeleton.messaging.api.error.MessageSerializationException;
|
||||
import dev.caskeleton.messaging.api.error.MessageTooLargeException;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.apache.avro.Schema;
|
||||
import org.apache.avro.generic.GenericData;
|
||||
import org.apache.avro.generic.GenericRecord;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* What a hostile Avro payload can make the decoder allocate.
|
||||
*
|
||||
* <p>The byte limit is the wrong instrument for this attack and was the only one in place. Avro
|
||||
* writes a collection as a declared element count followed by the elements, and the count is a
|
||||
* variable-length integer: five bytes claim four hundred million elements. Those five bytes pass
|
||||
* every length check the codec applies, and the generic reader then reserves the backing array from
|
||||
* the claim before reading a single element. The result is an {@code OutOfMemoryError} — not an
|
||||
* exception the codec can classify, not a message the consumer can dead-letter, and on a broker
|
||||
* consumer thread it takes the process with it.
|
||||
*
|
||||
* <p>Strings, byte arrays and maps were already safe: Avro validates those lengths against the
|
||||
* bytes actually remaining. Arrays were the one shape that allocated on trust, which is why the
|
||||
* corpus below is narrow rather than exhaustive — it pins the case that failed, and the two cases
|
||||
* that must keep working around it.
|
||||
*/
|
||||
class AvroHostileInputTest {
|
||||
|
||||
private static final MessageType BASKET = new MessageType("basket.filled");
|
||||
private static final SchemaVersion V1 = new SchemaVersion(1);
|
||||
private static final SchemaVersion V2 = new SchemaVersion(2);
|
||||
|
||||
private static final Schema V1_SCHEMA =
|
||||
new Schema.Parser()
|
||||
.parse(
|
||||
"""
|
||||
{
|
||||
"type": "record",
|
||||
"name": "BasketFilled",
|
||||
"namespace": "dev.caskeleton.messaging.sample",
|
||||
"fields": [
|
||||
{"name": "items", "type": {"type": "array", "items": "string"}}
|
||||
]
|
||||
}
|
||||
""");
|
||||
|
||||
private static final Schema V2_SCHEMA =
|
||||
new Schema.Parser()
|
||||
.parse(
|
||||
"""
|
||||
{
|
||||
"type": "record",
|
||||
"name": "BasketFilled",
|
||||
"namespace": "dev.caskeleton.messaging.sample",
|
||||
"fields": [
|
||||
{"name": "items", "type": {"type": "array", "items": "string"}},
|
||||
{"name": "channel", "type": "string", "default": "WEB"}
|
||||
]
|
||||
}
|
||||
""");
|
||||
|
||||
@Test
|
||||
@DisplayName("an array length no payload of this size could hold is refused before allocation")
|
||||
void aDeclaredArrayLengthTheInputCannotContainIsRefused() {
|
||||
AvroMessageCodec codec = new AvroMessageCodec(Map.of(BASKET, Map.of(V1, V1_SCHEMA)));
|
||||
byte[] hostile = zigzag(400_000_000L);
|
||||
|
||||
assertThat(hostile)
|
||||
.as("the whole attack is five bytes, so no byte limit stands between it and the allocation")
|
||||
.hasSizeLessThan(16);
|
||||
assertThatThrownBy(() -> codec.decode(BASKET, V1, hostile, GenericRecord.class))
|
||||
.isInstanceOf(MessageTooLargeException.class)
|
||||
.hasMessageContaining("400000000");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the evolution decode refuses the same claim")
|
||||
void theEvolutionDecodeAppliesTheSameCollectionBound() {
|
||||
AvroMessageCodec codec =
|
||||
new AvroMessageCodec(Map.of(BASKET, Map.of(V1, V1_SCHEMA, V2, V2_SCHEMA)));
|
||||
|
||||
assertThatThrownBy(() -> codec.decodeEvolved(BASKET, V1, V2, zigzag(400_000_000L)))
|
||||
.as("a consumer reading a newer producer takes this path for every message")
|
||||
.isInstanceOf(MessageTooLargeException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a count too large for an int fails as a decode error, not an allocation")
|
||||
void aCountBeyondIntRangeFailsWhileReadingRatherThanWhileReserving() {
|
||||
AvroMessageCodec codec = new AvroMessageCodec(Map.of(BASKET, Map.of(V1, V1_SCHEMA)));
|
||||
|
||||
assertThatThrownBy(() -> codec.decode(BASKET, V1, zigzag(4_294_967_296L), GenericRecord.class))
|
||||
.as("the count truncates to a harmless int, and the read then runs out of input")
|
||||
.isInstanceOf(MessageSerializationException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an array a payload really contains still round trips")
|
||||
void anHonestArrayIsUnaffectedByTheBound() {
|
||||
AvroMessageCodec codec = new AvroMessageCodec(Map.of(BASKET, Map.of(V1, V1_SCHEMA)));
|
||||
GenericRecord basket = new GenericData.Record(V1_SCHEMA);
|
||||
basket.put("items", List.of("apple", "pear", "plum"));
|
||||
|
||||
byte[] encoded = codec.encode(BASKET, V1, basket).bytes();
|
||||
GenericRecord decoded = codec.decode(BASKET, V1, encoded, GenericRecord.class);
|
||||
|
||||
assertThat(decoded.get("items").toString()).isEqualTo("[apple, pear, plum]");
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes one Avro variable-length integer, the way a producer encodes a collection count.
|
||||
*
|
||||
* <p>Hand-written rather than taken from an encoder because the point is to write a count with no
|
||||
* elements behind it, which no encoder will do.
|
||||
*/
|
||||
private static byte[] zigzag(long value) {
|
||||
long remaining = (value << 1) ^ (value >> 63);
|
||||
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
|
||||
while ((remaining & ~0x7FL) != 0) {
|
||||
bytes.write((int) ((remaining & 0x7F) | 0x80));
|
||||
remaining >>>= 7;
|
||||
}
|
||||
bytes.write((int) remaining);
|
||||
return bytes.toByteArray();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user