Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(kinesis): reject records over the 1 MiB size limit (#1896)

soreavis5afb60d0f55e
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
index 0634f84779..7beb2a8edd 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
@@ -426,15 +426,37 @@ public class KinesisJsonHandler {
private Response handlePutRecords(JsonNode request, String region) {
String streamName = resolveStreamName(request);
+ service.describeStream(streamName, region);
JsonNode recordsNode = request.path("Records");
+
+ // An oversized record fails the whole request before anything is
+ // written — per-record ErrorCode is reserved for throughput/internal
+ // failures. Successful decodes are kept for the write loop; malformed
+ // Data is left null so it stays a per-record failure there.
+ record Entry(JsonNode node, byte[] data) {}
+ List<Entry> entries = new ArrayList<>();
+ for (JsonNode node : recordsNode) {
+ byte[] data;
+ try {
+ data = Base64.getDecoder().decode(node.path("Data").asText());
+ } catch (IllegalArgumentException e) {
+ data = null;
+ }
+ if (data != null) {
+ service.validateRecordSize(data, node.path("PartitionKey").asText());
+ }
+ entries.add(new Entry(node, data));
+ }
+
ObjectNode response = objectMapper.createObjectNode();
ArrayNode results = response.putArray("Records");
int failed = 0;
- for (JsonNode node : recordsNode) {
+ for (Entry entry : entries) {
try {
- byte[] data = Base64.getDecoder().decode(node.path("Data").asText());
- String partitionKey = node.path("PartitionKey").asText();
+ byte[] data = entry.data() != null ? entry.data()
+ : Base64.getDecoder().decode(entry.node().path("Data").asText());
+ String partitionKey = entry.node().path("PartitionKey").asText();
KinesisService.PutRecordResult result = service.putRecordWithShardId(streamName, data, partitionKey, region);
results.addObject()
.put("SequenceNumber", result.sequenceNumber())
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
index 96746e2a43..ff0eadded8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
@@ -30,6 +30,10 @@ public class KinesisService {
private static final String DEFAULT_STREAM_MODE = "PROVISIONED";
private static final int DEFAULT_INSPECTION_RECORD_LIMIT = 100;
private static final int MAX_INSPECTION_RECORD_LIMIT = 1000;
+ // AWS default per-stream maximum record size (data blob + partition key).
+ // Raisable per stream via UpdateMaxRecordSize, which floci doesn't
+ // implement, so every stream sits at the default.
+ private static final int MAX_RECORD_SIZE_BYTES = 1_048_576;
private final StorageBackend<String, KinesisStream> store;
private final StorageBackend<String, KinesisConsumer> consumerStore;
@@ -368,8 +372,19 @@ public class KinesisService {
return putRecordWithShardId(streamName, data, partitionKey, region).sequenceNumber();
}
+ public void validateRecordSize(byte[] data, String partitionKey) {
+ int size = (data != null ? data.length : 0)
+ + (partitionKey != null ? partitionKey.getBytes(StandardCharsets.UTF_8).length : 0);
+ if (size > MAX_RECORD_SIZE_BYTES) {
+ throw new AwsException("InvalidArgumentException",
+ "Record size (data + partition key) of " + size + " bytes exceeds the maximum of "
+ + MAX_RECORD_SIZE_BYTES + " bytes.", 400);
+ }
+ }
+
public PutRecordResult putRecordWithShardId(String streamName, byte[] data, String partitionKey, String region) {
KinesisStream stream = resolveStream(streamName, region);
+ validateRecordSize(data, partitionKey);
KinesisShard shard = selectShard(stream, partitionKey);
String sequenceNumber = String.valueOf(sequenceGenerator.incrementAndGet());
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java
index 7bc1775a8e..31c97cc483 100644
--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java
@@ -1030,6 +1030,72 @@ class KinesisIntegrationTest {
.body("__type", equalTo("ResourceNotFoundException"));
}
+ @Test
+ @Order(63)
+ void putRecordRejectsRecordOverSizeLimit() {
+ given()
+ .header("X-Amz-Target", "Kinesis_20131202.CreateStream")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("{\"StreamName\": \"size-limit-test\", \"ShardCount\": 1}")
+ .when().post("/")
+ .then().statusCode(200);
+
+ String oversized = java.util.Base64.getEncoder().encodeToString(new byte[1_048_577]);
+ given()
+ .header("X-Amz-Target", "Kinesis_20131202.PutRecord")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("{\"StreamName\": \"size-limit-test\","
+ + "\"Data\": \"" + oversized + "\","
+ + "\"PartitionKey\": \"pk1\"}")
+ .when().post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidArgumentException"));
+ }
+
+ @Test
+ @Order(64)
+ void putRecordsRejectsWholeBatchOverSizeLimit() {
+ given()
+ .header("X-Amz-Target", "Kinesis_20131202.CreateStream")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("{\"StreamName\": \"batch-size-limit-test\", \"ShardCount\": 1}")
+ .when().post("/")
+ .then().statusCode(200);
+
+ String oversized = java.util.Base64.getEncoder().encodeToString(new byte[1_048_577]);
+ given()
+ .header("X-Amz-Target", "Kinesis_20131202.PutRecords")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("{\"StreamName\": \"batch-size-limit-test\","
+ + "\"Records\": ["
+ + "{\"Data\": \"dGVzdA==\", \"PartitionKey\": \"pk1\"},"
+ + "{\"Data\": \"" + oversized + "\", \"PartitionKey\": \"pk2\"}"
+ + "]}")
+ .when().post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidArgumentException"));
+
+ // The valid first record must not have landed either
+ String iterator = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetShardIterator")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("{\"StreamName\": \"batch-size-limit-test\", \"ShardId\": \"shardId-000000000000\", \"ShardIteratorType\": \"TRIM_HORIZON\"}")
+ .when().post("/")
+ .then().statusCode(200)
+ .extract().jsonPath().getString("ShardIterator");
+
+ given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetRecords")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("{\"ShardIterator\": \"" + iterator + "\"}")
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("Records.size()", equalTo(0));
+ }
+
private JsonNode decodeFirstEventStreamMessage(byte[] data) throws Exception {
ByteBuffer buf = ByteBuffer.wrap(data);
// Skip the first message (initial-response)
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
index 8d9c09acef..372aacb93e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.kinesis;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
@@ -9,6 +10,8 @@ import jakarta.ws.rs.core.Response;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import java.util.Base64;
+
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -342,4 +345,96 @@ class KinesisJsonHandlerTest {
() -> handler.handle("UpdateStreamMode", updateReq, REGION));
assertEquals("ResourceNotFoundException", ex.getErrorCode());
}
+
+ @Test
+ void putRecordRejectsRecordOverSizeLimit() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ // 1_048_574 data bytes + 3-byte partition key = 1_048_577, one over the limit
+ req.put("Data", Base64.getEncoder().encodeToString(new byte[1_048_574]));
+ req.put("PartitionKey", "pk1");
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecord", req, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode());
+ }
+
+ @Test
+ void putRecordAcceptsRecordAtSizeLimit() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ // 1_048_573 data bytes + 3-byte partition key = exactly 1_048_576
+ req.put("Data", Base64.getEncoder().encodeToString(new byte[1_048_573]));
+ req.put("PartitionKey", "pk1");
+ assertThat(handler.handle("PutRecord", req, REGION).getStatus(), is(200));
+ }
+
+ @Test
+ void putRecordsRejectsWholeBatchOnOversizedRecord() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ ArrayNode records = req.putArray("Records");
+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");
+ records.addObject()
+ .put("Data", Base64.getEncoder().encodeToString(new byte[1_048_574]))
+ .put("PartitionKey", "pk1");
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecords", req, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode());
+
+ // The valid first record must not have landed either
+ ObjectNode descReq = MAPPER.createObjectNode();
+ descReq.put("StreamName", "test-stream");
+ String shardId = responseEntity(handler.handle("DescribeStream", descReq, REGION))
+ .get("StreamDescription").get("Shards").get(0).get("ShardId").asText();
+
+ ObjectNode iterReq = MAPPER.createObjectNode();
+ iterReq.put("StreamName", "test-stream");
+ iterReq.put("ShardId", shardId);
+ iterReq.put("ShardIteratorType", "TRIM_HORIZON");
+ String iterator = responseEntity(handler.handle("GetShardIterator", iterReq, REGION))
+ .get("ShardIterator").asText();
+
+ ObjectNode recReq = MAPPER.createObjectNode();
+ recReq.put("ShardIterator", iterator);
+ assertEquals(0, responseEntity(handler.handle("GetRecords", recReq, REGION))
+ .get("Records").size());
+ }
+
+ @Test
+ void putRecordsKeepsMalformedDataAsPerRecordFailure() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ ArrayNode records = req.putArray("Records");
+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");
+ records.addObject().put("Data", "!!!not-base64!!!").put("PartitionKey", "pk2");
+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk3");
+
+ Response resp = handler.handle("PutRecords", req, REGION);
+ assertThat(resp.getStatus(), is(200));
+ ObjectNode body = responseEntity(resp);
+ assertEquals(1, body.get("FailedRecordCount").asInt());
+
+ ArrayNode results = (ArrayNode) body.get("Records");
+ assertThat(results.get(0).has("SequenceNumber"), is(true));
+ assertEquals("InternalFailure", results.get(1).get("ErrorCode").asText());
+ assertThat(results.get(2).has("SequenceNumber"), is(true));
+ }
+
+ @Test
+ void putRecordsRejectsUnknownStream() {
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "missing-stream");
+ req.putArray("Records").addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecords", req, REGION));
+ assertEquals("ResourceNotFoundException", ex.getErrorCode());
+ }
}