Light, fluffy, and always free - The AWS Local Emulator alternative
fix(kinesis): reject records over the 1 MiB size limit (#1896)
soreavis5afb60d0f55e
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.javaindex 0634f84779..7beb2a8edd 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java@@ -426,15 +426,37 @@ public class KinesisJsonHandler {private Response handlePutRecords(JsonNode request, String region) {String streamName = resolveStreamName(request);+ service.describeStream(streamName, region);JsonNode recordsNode = request.path("Records");++ // An oversized record fails the whole request before anything is+ // written — per-record ErrorCode is reserved for throughput/internal+ // failures. Successful decodes are kept for the write loop; malformed+ // Data is left null so it stays a per-record failure there.+ record Entry(JsonNode node, byte[] data) {}+ List<Entry> entries = new ArrayList<>();+ for (JsonNode node : recordsNode) {+ byte[] data;+ try {+ data = Base64.getDecoder().decode(node.path("Data").asText());+ } catch (IllegalArgumentException e) {+ data = null;+ }+ if (data != null) {+ service.validateRecordSize(data, node.path("PartitionKey").asText());+ }+ entries.add(new Entry(node, data));+ }+ObjectNode response = objectMapper.createObjectNode();ArrayNode results = response.putArray("Records");int failed = 0;- for (JsonNode node : recordsNode) {+ for (Entry entry : entries) {try {- byte[] data = Base64.getDecoder().decode(node.path("Data").asText());- String partitionKey = node.path("PartitionKey").asText();+ byte[] data = entry.data() != null ? entry.data()+ : Base64.getDecoder().decode(entry.node().path("Data").asText());+ String partitionKey = entry.node().path("PartitionKey").asText();KinesisService.PutRecordResult result = service.putRecordWithShardId(streamName, data, partitionKey, region);results.addObject().put("SequenceNumber", result.sequenceNumber())diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.javaindex 96746e2a43..ff0eadded8 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java@@ -30,6 +30,10 @@ public class KinesisService {private static final String DEFAULT_STREAM_MODE = "PROVISIONED";private static final int DEFAULT_INSPECTION_RECORD_LIMIT = 100;private static final int MAX_INSPECTION_RECORD_LIMIT = 1000;+ // AWS default per-stream maximum record size (data blob + partition key).+ // Raisable per stream via UpdateMaxRecordSize, which floci doesn't+ // implement, so every stream sits at the default.+ private static final int MAX_RECORD_SIZE_BYTES = 1_048_576;private final StorageBackend<String, KinesisStream> store;private final StorageBackend<String, KinesisConsumer> consumerStore;@@ -368,8 +372,19 @@ public class KinesisService {return putRecordWithShardId(streamName, data, partitionKey, region).sequenceNumber();}+ public void validateRecordSize(byte[] data, String partitionKey) {+ int size = (data != null ? data.length : 0)+ + (partitionKey != null ? partitionKey.getBytes(StandardCharsets.UTF_8).length : 0);+ if (size > MAX_RECORD_SIZE_BYTES) {+ throw new AwsException("InvalidArgumentException",+ "Record size (data + partition key) of " + size + " bytes exceeds the maximum of "+ + MAX_RECORD_SIZE_BYTES + " bytes.", 400);+ }+ }+public PutRecordResult putRecordWithShardId(String streamName, byte[] data, String partitionKey, String region) {KinesisStream stream = resolveStream(streamName, region);+ validateRecordSize(data, partitionKey);KinesisShard shard = selectShard(stream, partitionKey);String sequenceNumber = String.valueOf(sequenceGenerator.incrementAndGet());diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.javaindex 7bc1775a8e..31c97cc483 100644--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java@@ -1030,6 +1030,72 @@ class KinesisIntegrationTest {.body("__type", equalTo("ResourceNotFoundException"));}+ @Test+ @Order(63)+ void putRecordRejectsRecordOverSizeLimit() {+ given()+ .header("X-Amz-Target", "Kinesis_20131202.CreateStream")+ .contentType(KINESIS_CONTENT_TYPE)+ .body("{\"StreamName\": \"size-limit-test\", \"ShardCount\": 1}")+ .when().post("/")+ .then().statusCode(200);++ String oversized = java.util.Base64.getEncoder().encodeToString(new byte[1_048_577]);+ given()+ .header("X-Amz-Target", "Kinesis_20131202.PutRecord")+ .contentType(KINESIS_CONTENT_TYPE)+ .body("{\"StreamName\": \"size-limit-test\","+ + "\"Data\": \"" + oversized + "\","+ + "\"PartitionKey\": \"pk1\"}")+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidArgumentException"));+ }++ @Test+ @Order(64)+ void putRecordsRejectsWholeBatchOverSizeLimit() {+ given()+ .header("X-Amz-Target", "Kinesis_20131202.CreateStream")+ .contentType(KINESIS_CONTENT_TYPE)+ .body("{\"StreamName\": \"batch-size-limit-test\", \"ShardCount\": 1}")+ .when().post("/")+ .then().statusCode(200);++ String oversized = java.util.Base64.getEncoder().encodeToString(new byte[1_048_577]);+ given()+ .header("X-Amz-Target", "Kinesis_20131202.PutRecords")+ .contentType(KINESIS_CONTENT_TYPE)+ .body("{\"StreamName\": \"batch-size-limit-test\","+ + "\"Records\": ["+ + "{\"Data\": \"dGVzdA==\", \"PartitionKey\": \"pk1\"},"+ + "{\"Data\": \"" + oversized + "\", \"PartitionKey\": \"pk2\"}"+ + "]}")+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidArgumentException"));++ // The valid first record must not have landed either+ String iterator = given()+ .header("X-Amz-Target", "Kinesis_20131202.GetShardIterator")+ .contentType(KINESIS_CONTENT_TYPE)+ .body("{\"StreamName\": \"batch-size-limit-test\", \"ShardId\": \"shardId-000000000000\", \"ShardIteratorType\": \"TRIM_HORIZON\"}")+ .when().post("/")+ .then().statusCode(200)+ .extract().jsonPath().getString("ShardIterator");++ given()+ .header("X-Amz-Target", "Kinesis_20131202.GetRecords")+ .contentType(KINESIS_CONTENT_TYPE)+ .body("{\"ShardIterator\": \"" + iterator + "\"}")+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Records.size()", equalTo(0));+ }+private JsonNode decodeFirstEventStreamMessage(byte[] data) throws Exception {ByteBuffer buf = ByteBuffer.wrap(data);// Skip the first message (initial-response)diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.javaindex 8d9c09acef..372aacb93e 100644--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.kinesis;import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;@@ -9,6 +10,8 @@ import jakarta.ws.rs.core.Response;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import java.util.Base64;+import static org.hamcrest.MatcherAssert.assertThat;import static org.hamcrest.Matchers.is;import static org.junit.jupiter.api.Assertions.assertEquals;@@ -342,4 +345,96 @@ class KinesisJsonHandlerTest {() -> handler.handle("UpdateStreamMode", updateReq, REGION));assertEquals("ResourceNotFoundException", ex.getErrorCode());}++ @Test+ void putRecordRejectsRecordOverSizeLimit() {+ createStream("test-stream");++ ObjectNode req = MAPPER.createObjectNode();+ req.put("StreamName", "test-stream");+ // 1_048_574 data bytes + 3-byte partition key = 1_048_577, one over the limit+ req.put("Data", Base64.getEncoder().encodeToString(new byte[1_048_574]));+ req.put("PartitionKey", "pk1");+ AwsException ex = assertThrows(AwsException.class,+ () -> handler.handle("PutRecord", req, REGION));+ assertEquals("InvalidArgumentException", ex.getErrorCode());+ }++ @Test+ void putRecordAcceptsRecordAtSizeLimit() {+ createStream("test-stream");++ ObjectNode req = MAPPER.createObjectNode();+ req.put("StreamName", "test-stream");+ // 1_048_573 data bytes + 3-byte partition key = exactly 1_048_576+ req.put("Data", Base64.getEncoder().encodeToString(new byte[1_048_573]));+ req.put("PartitionKey", "pk1");+ assertThat(handler.handle("PutRecord", req, REGION).getStatus(), is(200));+ }++ @Test+ void putRecordsRejectsWholeBatchOnOversizedRecord() {+ createStream("test-stream");++ ObjectNode req = MAPPER.createObjectNode();+ req.put("StreamName", "test-stream");+ ArrayNode records = req.putArray("Records");+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");+ records.addObject()+ .put("Data", Base64.getEncoder().encodeToString(new byte[1_048_574]))+ .put("PartitionKey", "pk1");+ AwsException ex = assertThrows(AwsException.class,+ () -> handler.handle("PutRecords", req, REGION));+ assertEquals("InvalidArgumentException", ex.getErrorCode());++ // The valid first record must not have landed either+ ObjectNode descReq = MAPPER.createObjectNode();+ descReq.put("StreamName", "test-stream");+ String shardId = responseEntity(handler.handle("DescribeStream", descReq, REGION))+ .get("StreamDescription").get("Shards").get(0).get("ShardId").asText();++ ObjectNode iterReq = MAPPER.createObjectNode();+ iterReq.put("StreamName", "test-stream");+ iterReq.put("ShardId", shardId);+ iterReq.put("ShardIteratorType", "TRIM_HORIZON");+ String iterator = responseEntity(handler.handle("GetShardIterator", iterReq, REGION))+ .get("ShardIterator").asText();++ ObjectNode recReq = MAPPER.createObjectNode();+ recReq.put("ShardIterator", iterator);+ assertEquals(0, responseEntity(handler.handle("GetRecords", recReq, REGION))+ .get("Records").size());+ }++ @Test+ void putRecordsKeepsMalformedDataAsPerRecordFailure() {+ createStream("test-stream");++ ObjectNode req = MAPPER.createObjectNode();+ req.put("StreamName", "test-stream");+ ArrayNode records = req.putArray("Records");+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");+ records.addObject().put("Data", "!!!not-base64!!!").put("PartitionKey", "pk2");+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk3");++ Response resp = handler.handle("PutRecords", req, REGION);+ assertThat(resp.getStatus(), is(200));+ ObjectNode body = responseEntity(resp);+ assertEquals(1, body.get("FailedRecordCount").asInt());++ ArrayNode results = (ArrayNode) body.get("Records");+ assertThat(results.get(0).has("SequenceNumber"), is(true));+ assertEquals("InternalFailure", results.get(1).get("ErrorCode").asText());+ assertThat(results.get(2).has("SequenceNumber"), is(true));+ }++ @Test+ void putRecordsRejectsUnknownStream() {+ ObjectNode req = MAPPER.createObjectNode();+ req.put("StreamName", "missing-stream");+ req.putArray("Records").addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");+ AwsException ex = assertThrows(AwsException.class,+ () -> handler.handle("PutRecords", req, REGION));+ assertEquals("ResourceNotFoundException", ex.getErrorCode());+ }}