Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(kinesis): add PutRecords request caps and a per-stream max record size (#2438)

soreavis353d9c842fa8
diff --git a/docs/services/kinesis.md b/docs/services/kinesis.md
index 8bca092842..c537aac49c 100644
--- a/docs/services/kinesis.md
+++ b/docs/services/kinesis.md
@@ -35,6 +35,7 @@
| `EnableEnhancedMonitoring` | - |
| `DisableEnhancedMonitoring` | - |
| `UpdateStreamMode` | - |
+| `UpdateMaxRecordSize` | - |
<!-- floci:actions:end -->
## Local Inspection Endpoints
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
index 9db0baa347..a3afa5aace 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
@@ -16,6 +16,7 @@ import jakarta.inject.Inject;
import jakarta.ws.rs.core.Response;
import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Base64;
@@ -66,6 +67,7 @@ public class KinesisJsonHandler {
case "EnableEnhancedMonitoring" -> handleEnableEnhancedMonitoring(request, region);
case "DisableEnhancedMonitoring" -> handleDisableEnhancedMonitoring(request, region);
case "UpdateStreamMode" -> handleUpdateStreamMode(request, region);
+ case "UpdateMaxRecordSize" -> handleUpdateMaxRecordSize(request, region);
default -> Response.status(400)
.entity(new AwsErrorResponse("UnsupportedOperation", "Operation " + action + " is not supported."))
.build();
@@ -112,7 +114,8 @@ public class KinesisJsonHandler {
streamMode = mode;
}
}
- service.createStream(streamName, shardCount, streamMode, region);
+ service.createStream(streamName, shardCount, streamMode,
+ optionalMaxRecordSize(request), region);
return Response.ok(objectMapper.createObjectNode()).build();
}
@@ -135,6 +138,36 @@ public class KinesisJsonHandler {
return Response.ok(objectMapper.createObjectNode()).build();
}
+ /**
+ * MaxRecordSizeInKiB is modelled as an integer, so a float or a value outside int range is a
+ * type error rather than something to truncate into the range check.
+ */
+ private Integer optionalMaxRecordSize(JsonNode request) {
+ JsonNode size = request.path("MaxRecordSizeInKiB");
+ if (size.isMissingNode() || size.isNull()) {
+ return null;
+ }
+ if (!size.isIntegralNumber() || !size.canConvertToInt()) {
+ throw new AwsException("InvalidArgumentException",
+ "MaxRecordSizeInKiB must be an integer", 400);
+ }
+ return size.intValue();
+ }
+
+ private Response handleUpdateMaxRecordSize(JsonNode request, String region) {
+ // UpdateMaxRecordSize identifies the stream by StreamARN only per the AWS API; StreamName is not valid.
+ String streamArn = request.path("StreamARN").asText(null);
+ if (streamArn == null || streamArn.isBlank()) {
+ throw new AwsException("InvalidArgumentException", "StreamARN is required", 400);
+ }
+ Integer size = optionalMaxRecordSize(request);
+ if (size == null) {
+ throw new AwsException("InvalidArgumentException", "MaxRecordSizeInKiB is required", 400);
+ }
+ service.updateMaxRecordSize(extractStreamNameFromArn(streamArn), size, region);
+ return Response.ok(objectMapper.createObjectNode()).build();
+ }
+
private String extractStreamNameFromArn(String streamArn) {
String name = parseStreamNameFromArn(streamArn);
if (name == null) {
@@ -214,6 +247,7 @@ public class KinesisJsonHandler {
summary.put("StreamCreationTimestamp", epochSeconds(stream.getStreamCreationTimestamp()));
summary.put("OpenShardCount", (int) stream.getShards().stream().filter(s -> !s.isClosed()).count());
summary.put("EncryptionType", stream.getEncryptionType());
+ summary.put("MaxRecordSizeInKiB", stream.getMaxRecordSizeInKiB());
if (stream.getKeyId() != null) {
summary.put("KeyId", stream.getKeyId());
}
@@ -432,25 +466,43 @@ public class KinesisJsonHandler {
private Response handlePutRecords(JsonNode request, String region) {
String streamName = resolveStreamName(request);
- service.describeStream(streamName, region);
+ KinesisStream stream = service.describeStream(streamName, region);
JsonNode recordsNode = request.path("Records");
+ service.validateRecordCount(recordsNode.size());
// An oversized record fails the whole request before anything is
// written — per-record ErrorCode is reserved for throughput/internal
// failures. Successful decodes are kept for the write loop; malformed
- // Data is left null so it stays a per-record failure there.
+ // Data is left null so it stays a per-record failure there. The count
+ // cap is checked upfront and the byte cap as the loop goes, so neither
+ // an over-long batch nor an oversized one is fully decoded before it
+ // is rejected.
record Entry(JsonNode node, byte[] data) {}
List<Entry> entries = new ArrayList<>();
+ long totalBytes = 0;
for (JsonNode node : recordsNode) {
- byte[] data;
- try {
- data = Base64.getDecoder().decode(node.path("Data").asText());
- } catch (IllegalArgumentException e) {
- data = null;
+ JsonNode dataNode = node.path("Data");
+ byte[] data = null;
+ if (dataNode.isTextual()) {
+ try {
+ data = Base64.getDecoder().decode(dataNode.textValue());
+ } catch (IllegalArgumentException e) {
+ data = null;
+ }
}
+ String partitionKey = node.path("PartitionKey").asText();
if (data != null) {
- service.validateRecordSize(data, node.path("PartitionKey").asText());
+ service.validateRecordSize(stream, data, partitionKey);
}
+ // Data that did not decode still travelled in the request, so it counts toward the
+ // request cap at the bytes the caller sent rather than as nothing — otherwise a batch
+ // of undecodable records could carry any payload past the limit. A non-string Data is
+ // measured the same way, since it is not a blob AWS would have accepted either.
+ totalBytes += KinesisService.recordSize(
+ data != null ? data.length
+ : dataNode.toString().getBytes(StandardCharsets.UTF_8).length,
+ partitionKey);
+ service.validateRequestSize(totalBytes);
entries.add(new Entry(node, data));
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
index ff0eadded8..78ee4cd864 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
@@ -30,10 +30,12 @@ public class KinesisService {
private static final String DEFAULT_STREAM_MODE = "PROVISIONED";
private static final int DEFAULT_INSPECTION_RECORD_LIMIT = 100;
private static final int MAX_INSPECTION_RECORD_LIMIT = 1000;
- // AWS default per-stream maximum record size (data blob + partition key).
- // Raisable per stream via UpdateMaxRecordSize, which floci doesn't
- // implement, so every stream sits at the default.
- private static final int MAX_RECORD_SIZE_BYTES = 1_048_576;
+ // MaxRecordSizeInKiB bounds, as AWS constrains them on CreateStream and
+ // UpdateMaxRecordSize. The per-stream default lives on KinesisStream.
+ private static final int MIN_RECORD_SIZE_KIB = 1024;
+ private static final int MAX_RECORD_SIZE_KIB = 10240;
+ private static final int MAX_RECORDS_PER_REQUEST = 500;
+ private static final int MAX_REQUEST_SIZE_BYTES = 10 * 1024 * 1024;
private final StorageBackend<String, KinesisStream> store;
private final StorageBackend<String, KinesisConsumer> consumerStore;
@@ -62,11 +64,24 @@ public class KinesisService {
}
public KinesisStream createStream(String streamName, int shardCount, String streamMode, String region) {
+ return createStream(streamName, shardCount, streamMode, null, region);
+ }
+
+ /**
+ * @param maxRecordSizeInKiB CreateStream's optional MaxRecordSizeInKiB; null leaves the stream
+ * at the AWS default. It is a distinct value from 0, which AWS
+ * rejects as below the documented minimum.
+ */
+ public KinesisStream createStream(String streamName, int shardCount, String streamMode,
+ Integer maxRecordSizeInKiB, String region) {
String resolvedMode = streamMode != null ? streamMode : DEFAULT_STREAM_MODE;
if (!VALID_STREAM_MODES.contains(resolvedMode)) {
throw new AwsException("InvalidArgumentException",
"StreamMode must be PROVISIONED or ON_DEMAND, got: " + resolvedMode, 400);
}
+ if (maxRecordSizeInKiB != null) {
+ validateMaxRecordSize(maxRecordSizeInKiB, "InvalidArgumentException");
+ }
String storageKey = regionKey(region, streamName);
if (store.get(storageKey).isPresent()) {
@@ -77,6 +92,9 @@ public class KinesisService {
KinesisStream stream = new KinesisStream(streamName, arn);
stream.setAccountId(regionResolver.getAccountId());
stream.setStreamMode(resolvedMode);
+ if (maxRecordSizeInKiB != null) {
+ stream.setMaxRecordSizeInKiB(maxRecordSizeInKiB);
+ }
for (int i = 0; i < shardCount; i++) {
String shardId = String.format("shardId-%012d", i);
@@ -372,19 +390,82 @@ public class KinesisService {
return putRecordWithShardId(streamName, data, partitionKey, region).sequenceNumber();
}
- public void validateRecordSize(byte[] data, String partitionKey) {
- int size = (data != null ? data.length : 0)
+ /** Record size as AWS measures it: the data blob plus the partition key. */
+ static int recordSize(int dataBytes, String partitionKey) {
+ return dataBytes
+ (partitionKey != null ? partitionKey.getBytes(StandardCharsets.UTF_8).length : 0);
- if (size > MAX_RECORD_SIZE_BYTES) {
+ }
+
+ public void validateRecordSize(KinesisStream stream, byte[] data, String partitionKey) {
+ int size = recordSize(data != null ? data.length : 0, partitionKey);
+ int max = stream.getMaxRecordSizeInKiB() * 1024;
+ if (size > max) {
throw new AwsException("InvalidArgumentException",
"Record size (data + partition key) of " + size + " bytes exceeds the maximum of "
- + MAX_RECORD_SIZE_BYTES + " bytes.", 400);
+ + max + " bytes.", 400);
}
}
+ /**
+ * The request-wide PutRecords caps. Both reject the whole request rather than failing records
+ * individually, because PutRecordsResultEntry.ErrorCode carries only throughput and internal
+ * errors — the same reason the per-record size check aborts the batch.
+ */
+ void validateRecordCount(int recordCount) {
+ if (recordCount < 1) {
+ throw new AwsException("InvalidArgumentException",
+ "A PutRecords request requires at least one record.", 400);
+ }
+ if (recordCount > MAX_RECORDS_PER_REQUEST) {
+ throw new AwsException("InvalidArgumentException",
+ "A PutRecords request supports at most " + MAX_RECORDS_PER_REQUEST
+ + " records, got " + recordCount + ".", 400);
+ }
+ }
+
+ void validateRequestSize(long totalBytes) {
+ if (totalBytes > MAX_REQUEST_SIZE_BYTES) {
+ throw new AwsException("InvalidArgumentException",
+ "A PutRecords request (data + partition keys) is limited to " + MAX_REQUEST_SIZE_BYTES
+ + " bytes, got " + totalBytes + ".", 400);
+ }
+ }
+
+ /**
+ * The bounds are the same on both actions but the code each publishes for a violation is not:
+ * UpdateMaxRecordSize documents ValidationException for an out-of-range size, while CreateStream
+ * reserves ValidationException for the on-demand case and describes InvalidArgumentException as
+ * "a specified parameter exceeds its restrictions".
+ */
+ private static void validateMaxRecordSize(int maxRecordSizeInKiB, String errorCode) {
+ if (maxRecordSizeInKiB < MIN_RECORD_SIZE_KIB || maxRecordSizeInKiB > MAX_RECORD_SIZE_KIB) {
+ throw new AwsException(errorCode,
+ "MaxRecordSizeInKiB must be between " + MIN_RECORD_SIZE_KIB + " and "
+ + MAX_RECORD_SIZE_KIB + ", got " + maxRecordSizeInKiB + ".", 400);
+ }
+ }
+
+ public void updateMaxRecordSize(String streamName, int maxRecordSizeInKiB, String region) {
+ KinesisStream stream = resolveStream(streamName, region);
+ validateMaxRecordSize(maxRecordSizeInKiB, "ValidationException");
+ if ("ON_DEMAND".equals(stream.getStreamMode())) {
+ throw new AwsException("ValidationException",
+ "UpdateMaxRecordSize is only supported for data streams with the provisioned capacity mode.",
+ 400);
+ }
+ if (!"ACTIVE".equals(stream.getStreamStatus())) {
+ throw new AwsException("ResourceInUseException",
+ "Stream " + streamName + " is not ACTIVE (current state: "
+ + stream.getStreamStatus() + ")", 400);
+ }
+ stream.setMaxRecordSizeInKiB(maxRecordSizeInKiB);
+ store.put(regionKey(region, streamName), stream);
+ LOG.infov("Updated max record size for {0} to {1} KiB", streamName, maxRecordSizeInKiB);
+ }
+
public PutRecordResult putRecordWithShardId(String streamName, byte[] data, String partitionKey, String region) {
KinesisStream stream = resolveStream(streamName, region);
- validateRecordSize(data, partitionKey);
+ validateRecordSize(stream, data, partitionKey);
KinesisShard shard = selectShard(stream, partitionKey);
String sequenceNumber = String.valueOf(sequenceGenerator.incrementAndGet());
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisStream.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisStream.java
index e6c12cd533..ca0130da39 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisStream.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisStream.java
@@ -26,6 +26,8 @@ public class KinesisStream {
private String keyId;
private String streamMode = "PROVISIONED";
private Set<String> enhancedMonitoringMetrics = new HashSet<>();
+ // AWS's default; streams persisted before this field existed read as the default too.
+ private int maxRecordSizeInKiB = 1024;
public KinesisStream() {}
@@ -69,6 +71,9 @@ public class KinesisStream {
public String getStreamMode() { return streamMode; }
public void setStreamMode(String streamMode) { this.streamMode = streamMode; }
+ public int getMaxRecordSizeInKiB() { return maxRecordSizeInKiB; }
+ public void setMaxRecordSizeInKiB(int maxRecordSizeInKiB) { this.maxRecordSizeInKiB = maxRecordSizeInKiB; }
+
public Set<String> getEnhancedMonitoringMetrics() { return enhancedMonitoringMetrics; }
public void setEnhancedMonitoringMetrics(Set<String> enhancedMonitoringMetrics) { this.enhancedMonitoringMetrics = enhancedMonitoringMetrics; }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
index 72d7885ac0..656a3318b9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
@@ -495,4 +495,385 @@ class KinesisJsonHandlerTest {
() -> handler.handle("PutRecords", req, REGION));
assertEquals("ResourceNotFoundException", ex.getErrorCode());
}
+
+ private String streamArn(String name) {
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", name);
+ return responseEntity(handler.handle("DescribeStreamSummary", req, REGION))
+ .get("StreamDescriptionSummary").get("StreamARN").asText();
+ }
+
+ private AwsException putRecord(String streamName, int dataBytes) {
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", streamName);
+ req.put("Data", Base64.getEncoder().encodeToString(new byte[dataBytes]));
+ req.put("PartitionKey", "pk1");
+ return assertThrows(AwsException.class, () -> handler.handle("PutRecord", req, REGION));
+ }
+
+ /**
+ * Only PutRecord exposes the ordering: PutRecords resolves the stream in its own preflight, so
+ * a swap inside putRecordWithShardId stays invisible there.
+ */
+ @Test
+ void putRecordResolvesTheStreamBeforeValidatingRecordSize() {
+ assertEquals("ResourceNotFoundException", putRecord("missing-stream", 1_048_577).getErrorCode());
+ }
+
+ /** The partition key counts as UTF-8 bytes: 100 euro signs weigh 300, not 100. */
+ @Test
+ void putRecordMeasuresThePartitionKeyAsUtf8Bytes() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ req.put("Data", Base64.getEncoder().encodeToString(new byte[1_048_400]));
+ req.put("PartitionKey", "€".repeat(100));
+ AwsException ex = assertThrows(AwsException.class, () -> handler.handle("PutRecord", req, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode());
+ }
+
+ @Test
+ void putRecordsRejectsMoreThanFiveHundredRecords() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ ArrayNode records = req.putArray("Records");
+ for (int i = 0; i < 501; i++) {
+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");
+ }
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecords", req, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode());
+ }
+
+ /** The count cap is inclusive: 500 is the largest batch AWS accepts, not the first rejected. */
+ @Test
+ void putRecordsAcceptsExactlyFiveHundredRecords() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ ArrayNode records = req.putArray("Records");
+ for (int i = 0; i < 500; i++) {
+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");
+ }
+ Response resp = handler.handle("PutRecords", req, REGION);
+ assertThat(resp.getStatus(), is(200));
+ assertEquals(0, responseEntity(resp).get("FailedRecordCount").asInt());
+ }
+
+ /** Eleven records that each pass the per-record check still exceed the 10 MiB request cap. */
+ @Test
+ void putRecordsRejectsRequestOverTheTotalSizeLimit() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ ArrayNode records = req.putArray("Records");
+ String data = Base64.getEncoder().encodeToString(new byte[1_048_000]);
+ for (int i = 0; i < 11; i++) {
+ records.addObject().put("Data", data).put("PartitionKey", "pk1");
+ }
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecords", req, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode());
+ }
+
+ @Test
+ void describeStreamSummaryReportsTheDefaultMaxRecordSize() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ assertEquals(1024, responseEntity(handler.handle("DescribeStreamSummary", req, REGION))
+ .get("StreamDescriptionSummary").get("MaxRecordSizeInKiB").asInt());
+ }
+
+ /** An explicit JSON null means "not specified", not a type error — the default still applies. */
+ @Test
+ void createStreamTreatsANullMaxRecordSizeAsAbsent() {
+ ObjectNode create = MAPPER.createObjectNode();
+ create.put("StreamName", "null-size-stream");
+ create.put("ShardCount", 1);
+ create.putNull("MaxRecordSizeInKiB");
+ assertThat(handler.handle("CreateStream", create, REGION).getStatus(), is(200));
+
+ ObjectNode summary = MAPPER.createObjectNode();
+ summary.put("StreamName", "null-size-stream");
+ assertEquals(1024, responseEntity(handler.handle("DescribeStreamSummary", summary, REGION))
+ .get("StreamDescriptionSummary").get("MaxRecordSizeInKiB").asInt());
+ }
+
+ @Test
+ void createStreamHonorsMaxRecordSizeInKiB() {
+ ObjectNode create = MAPPER.createObjectNode();
+ create.put("StreamName", "big-stream");
+ create.put("ShardCount", 1);
+ create.put("MaxRecordSizeInKiB", 2048);
+ assertThat(handler.handle("CreateStream", create, REGION).getStatus(), is(200));
+
+ ObjectNode summary = MAPPER.createObjectNode();
+ summary.put("StreamName", "big-stream");
+ assertEquals(2048, responseEntity(handler.handle("DescribeStreamSummary", summary, REGION))
+ .get("StreamDescriptionSummary").get("MaxRecordSizeInKiB").asInt());
+
+ ObjectNode put = MAPPER.createObjectNode();
+ put.put("StreamName", "big-stream");
+ put.put("Data", Base64.getEncoder().encodeToString(new byte[1_500_000]));
+ put.put("PartitionKey", "pk1");
+ assertThat(handler.handle("PutRecord", put, REGION).getStatus(), is(200));
+ }
+
+ private AwsException createStreamWithMaxRecordSize(String name, int maxRecordSizeInKiB) {
+ ObjectNode create = MAPPER.createObjectNode();
+ create.put("StreamName", name);
+ create.put("ShardCount", 1);
+ create.put("MaxRecordSizeInKiB", maxRecordSizeInKiB);
+ return assertThrows(AwsException.class, () -> handler.handle("CreateStream", create, REGION));
+ }
+
+ /**
+ * Both bounds, and 0 — a real out-of-range value rather than "not specified". CreateStream
+ * reserves ValidationException for the on-demand case, so a range violation is InvalidArgument.
+ */
+ @Test
+ void createStreamRejectsMaxRecordSizeOutOfRange() {
+ assertEquals("InvalidArgumentException", createStreamWithMaxRecordSize("low-stream", 1023).getErrorCode());
+ assertEquals("InvalidArgumentException", createStreamWithMaxRecordSize("high-stream", 10241).getErrorCode());
+ assertEquals("InvalidArgumentException", createStreamWithMaxRecordSize("zero-stream", 0).getErrorCode());
+ }
+
+ /** A long or a float would otherwise truncate into range: 4294969344 narrows to 2048. */
+ @Test
+ void maxRecordSizeMustBeAnIntegerRatherThanTruncatedIntoRange() {
+ for (Object bad : new Object[] {4294969344L, 10240.9d, "2048"}) {
+ ObjectNode create = MAPPER.createObjectNode();
+ create.put("StreamName", "typed-stream");
+ create.put("ShardCount", 1);
+ if (bad instanceof Long l) {
+ create.put("MaxRecordSizeInKiB", l);
+ } else if (bad instanceof Double d) {
+ create.put("MaxRecordSizeInKiB", d);
+ } else {
+ create.put("MaxRecordSizeInKiB", (String) bad);
+ }
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("CreateStream", create, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode(), "for " + bad);
+ }
+ }
+
+ @Test
+ void createStreamAcceptsBothEndsOfTheAllowedRange() {
+ for (int size : new int[] {1024, 10240}) {
+ ObjectNode create = MAPPER.createObjectNode();
+ create.put("StreamName", "edge-" + size);
+ create.put("ShardCount", 1);
+ create.put("MaxRecordSizeInKiB", size);
+ assertThat(handler.handle("CreateStream", create, REGION).getStatus(), is(200));
+
+ ObjectNode summary = MAPPER.createObjectNode();
+ summary.put("StreamName", "edge-" + size);
+ assertEquals(size, responseEntity(handler.handle("DescribeStreamSummary", summary, REGION))
+ .get("StreamDescriptionSummary").get("MaxRecordSizeInKiB").asInt());
+ }
+ }
+
+ /** A Data that is not a base64 string cannot weigh nothing, or it smuggles the payload through. */
+ @Test
+ void putRecordsCountsNonStringDataTowardTheTotalSizeLimit() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ ArrayNode records = req.putArray("Records");
+ for (int i = 0; i < 500; i++) {
+ records.addObject().putObject("Data").put("pad", "X".repeat(21_000));
+ }
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecords", req, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode());
+ }
+
+ /** Multi-byte characters must be counted as UTF-8 bytes, not UTF-16 units. */
+ @Test
+ void putRecordsCountsUndecodableDataAsUtf8Bytes() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ req.putArray("Records").addObject()
+ .put("Data", "€".repeat(4_000_000))
+ .put("PartitionKey", "pk1");
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecords", req, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode());
+ }
+
+ private AwsException updateMaxRecordSize(String streamName, int maxRecordSizeInKiB) {
+ ObjectNode update = MAPPER.createObjectNode();
+ update.put("StreamARN", streamArn(streamName));
+ update.put("MaxRecordSizeInKiB", maxRecordSizeInKiB);
+ return assertThrows(AwsException.class,
+ () -> handler.handle("UpdateMaxRecordSize", update, REGION));
+ }
+
+ @Test
+ void updateMaxRecordSizeRejectsMissingStreamArn() {
+ ObjectNode update = MAPPER.createObjectNode();
+ update.put("MaxRecordSizeInKiB", 2048);
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("UpdateMaxRecordSize", update, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode());
+ }
+
+ @Test
+ void updateMaxRecordSizeRejectsAMissingSize() {
+ createStream("test-stream");
+
+ ObjectNode update = MAPPER.createObjectNode();
+ update.put("StreamARN", streamArn("test-stream"));
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("UpdateMaxRecordSize", update, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode());
+ assertEquals("MaxRecordSizeInKiB is required", ex.getMessage());
+ }
+
+ @Test
+ void putRecordsRejectsAnEmptyRecordArray() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ req.putArray("Records");
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecords", req, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode());
+ }
+
+ /** The preflight runs first, so an empty batch on an unknown stream is still a 404. */
+ @Test
+ void putRecordsResolvesTheStreamBeforeValidatingTheRecordCount() {
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "missing-stream");
+ req.putArray("Records");
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecords", req, REGION));
+ assertEquals("ResourceNotFoundException", ex.getErrorCode());
+ }
+
+ /** Pins the cap's value: ten near-limit records sit just under 10 MiB and must be accepted. */
+ @Test
+ void putRecordsAcceptsARequestJustUnderTheTotalSizeLimit() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ ArrayNode records = req.putArray("Records");
+ String data = Base64.getEncoder().encodeToString(new byte[1_048_000]);
+ for (int i = 0; i < 10; i++) {
+ records.addObject().put("Data", data).put("PartitionKey", "pk1");
+ }
+ assertThat(handler.handle("PutRecords", req, REGION).getStatus(), is(200));
+ }
+
+ /** And the cap itself is inclusive: ten records of 1_048_573 + "pk1" total exactly 10 MiB. */
+ @Test
+ void putRecordsAcceptsARequestExactlyAtTheTotalSizeLimit() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ ArrayNode records = req.putArray("Records");
+ String data = Base64.getEncoder().encodeToString(new byte[1_048_573]);
+ for (int i = 0; i < 10; i++) {
+ records.addObject().put("Data", data).put("PartitionKey", "pk1");
+ }
+ Response resp = handler.handle("PutRecords", req, REGION);
+ assertThat(resp.getStatus(), is(200));
+ assertEquals(0, responseEntity(resp).get("FailedRecordCount").asInt());
+ }
+
+ /** Data that fails to decode still occupies the request, so it cannot be free of the cap. */
+ @Test
+ void putRecordsCountsUndecodableDataTowardTheTotalSizeLimit() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ ArrayNode records = req.putArray("Records");
+ String malformed = "!".repeat(1_400_000);
+ for (int i = 0; i < 12; i++) {
+ records.addObject().put("Data", malformed).put("PartitionKey", "pk1");
+ }
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecords", req, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode());
+ }
+
+ @Test
+ void updateMaxRecordSizeRaisesTheStreamLimit() {
+ createStream("test-stream");
+ assertEquals("InvalidArgumentException", putRecord("test-stream", 1_500_000).getErrorCode());
+
+ ObjectNode update = MAPPER.createObjectNode();
+ update.put("StreamARN", streamArn("test-stream"));
+ update.put("MaxRecordSizeInKiB", 2048);
+ assertThat(handler.handle("UpdateMaxRecordSize", update, REGION).getStatus(), is(200));
+
+ ObjectNode put = MAPPER.createObjectNode();
+ put.put("StreamName", "test-stream");
+ put.put("Data", Base64.getEncoder().encodeToString(new byte[1_500_000]));
+ put.put("PartitionKey", "pk1");
+ assertThat(handler.handle("PutRecord", put, REGION).getStatus(), is(200));
+ }
+
+ @Test
+ void updateMaxRecordSizeRejectsOnDemandStreams() {
+ ObjectNode create = MAPPER.createObjectNode();
+ create.put("StreamName", "on-demand-stream");
+ create.put("ShardCount", 1);
+ create.putObject("StreamModeDetails").put("StreamMode", "ON_DEMAND");
+ assertThat(handler.handle("CreateStream", create, REGION).getStatus(), is(200));
+
+ ObjectNode update = MAPPER.createObjectNode();
+ update.put("StreamARN", streamArn("on-demand-stream"));
+ update.put("MaxRecordSizeInKiB", 2048);
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("UpdateMaxRecordSize", update, REGION));
+ assertEquals("ValidationException", ex.getErrorCode());
+ }
+
+ @Test
+ void updateMaxRecordSizeRejectsAStreamThatIsNotActive() {
+ createStream("test-stream");
+ service.describeStream("test-stream", REGION).setStreamStatus("UPDATING");
+
+ ObjectNode update = MAPPER.createObjectNode();
+ update.put("StreamARN", streamArn("test-stream"));
+ update.put("MaxRecordSizeInKiB", 2048);
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("UpdateMaxRecordSize", update, REGION));
+ assertEquals("ResourceInUseException", ex.getErrorCode());
+ }
+
+ @Test
+ void updateMaxRecordSizeRejectsSizeOutOfRange() {
+ createStream("test-stream");
+ assertEquals("ValidationException", updateMaxRecordSize("test-stream", 10241).getErrorCode());
+ assertEquals("ValidationException", updateMaxRecordSize("test-stream", 1023).getErrorCode());
+ }
+
+ /** Resolve before validate, the same ordering PutRecord follows. */
+ @Test
+ void updateMaxRecordSizeResolvesTheStreamBeforeValidatingTheSize() {
+ ObjectNode update = MAPPER.createObjectNode();
+ update.put("StreamARN", "arn:aws:kinesis:us-east-1:123456789012:stream/missing-stream");
+ update.put("MaxRecordSizeInKiB", 10241);
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("UpdateMaxRecordSize", update, REGION));
+ assertEquals("ResourceNotFoundException", ex.getErrorCode());
+ }
}