Light, fluffy, and always free - The AWS Local Emulator alternative
fix(kinesis): reject non-blob Data instead of coercing it with asText (#2551)
soreavisdf5581e5bd7f
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.javaindex 191990ef1d..23a54b175d 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java@@ -465,9 +465,15 @@ public class KinesisJsonHandler {private Response handlePutRecord(JsonNode request, String region) {String streamName = resolveStreamName(request);+ JsonNode dataNode = request.path("Data");+ // The CBOR transports (the AWS SDK for Java's default for Kinesis) decode Data+ // as a binary node rather than base64 text, so both blob shapes are accepted.+ if (!dataNode.isTextual() && !dataNode.isBinary()) {+ throw new AwsException("SerializationException", "Data must be a base64-encoded string.", 400);+ }byte[] data;try {- data = Base64.getDecoder().decode(request.path("Data").asText());+ data = Base64.getDecoder().decode(dataNode.asText());} catch (IllegalArgumentException e) {throw new AwsException("SerializationException", "Data is not valid base64.", 400);}@@ -489,32 +495,32 @@ public class KinesisJsonHandler {// An oversized record fails the whole request before anything is// written — per-record ErrorCode is reserved for throughput/internal- // failures. Successful decodes are kept for the write loop; malformed- // Data is left null so it stays a per-record failure there. The count- // cap is checked upfront and the byte cap as the loop goes, so neither- // an over-long batch nor an oversized one is fully decoded before it- // is rejected.+ // failures. Successful decodes are kept for the write loop; a Data+ // that is not a decodable blob is left null and fails per-record+ // there, with its partition key still counted by the record-size+ // check. The count cap is checked upfront and the byte cap as the+ // loop goes, so neither an over-long batch nor an oversized one is+ // fully decoded before it is rejected.record Entry(JsonNode node, byte[] data) {}List<Entry> entries = new ArrayList<>();long totalBytes = 0;for (JsonNode node : recordsNode) {JsonNode dataNode = node.path("Data");byte[] data = null;- if (dataNode.isTextual()) {+ if (dataNode.isTextual() || dataNode.isBinary()) {try {- data = Base64.getDecoder().decode(dataNode.textValue());+ data = Base64.getDecoder().decode(dataNode.asText());} catch (IllegalArgumentException e) {data = null;}}String partitionKey = node.path("PartitionKey").asText();- if (data != null) {- service.validateRecordSize(stream, data, partitionKey);- }+ service.validateRecordSize(stream, data, partitionKey);// Data that did not decode still travelled in the request, so it counts toward the// request cap at the bytes the caller sent rather than as nothing — otherwise a batch- // of undecodable records could carry any payload past the limit. A non-string Data is- // measured the same way, since it is not a blob AWS would have accepted either.+ // of undecodable records could carry any payload past the limit. A Data that is+ // neither text nor binary is measured the same way, since it is not a blob AWS+ // would have accepted either.totalBytes += KinesisService.recordSize(data != null ? data.length: dataNode.toString().getBytes(StandardCharsets.UTF_8).length,@@ -529,8 +535,14 @@ public class KinesisJsonHandler {for (Entry entry : entries) {try {- byte[] data = entry.data() != null ? entry.data()- : Base64.getDecoder().decode(entry.node().path("Data").asText());+ byte[] data = entry.data();+ if (data == null) {+ JsonNode dataNode = entry.node().path("Data");+ if (!dataNode.isTextual() && !dataNode.isBinary()) {+ throw new IllegalArgumentException("Data must be a base64-encoded string.");+ }+ data = Base64.getDecoder().decode(dataNode.asText());+ }String partitionKey = entry.node().path("PartitionKey").asText();KinesisService.PutRecordResult result = service.putRecordWithShardId(streamName, data, partitionKey, region);results.addObject()diff --git a/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.javaindex 5c43755fdf..c2b94a75f6 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.java@@ -83,6 +83,50 @@ class AwsJsonCborKinesisAtTimestampIntegrationTest {"AT_TIMESTAMP over CBOR must find the record written after the cutoff");}+ /**+ * A real CBOR-default SDK client encodes {@code Data} as a CBOR byte string (major+ * type 2), which Jackson decodes to a binary node — not the base64 text string+ * Floci's own encoder writes. The request is hand-built for the same reason as the+ * timestamp one: posting Floci-encoded bytes would only prove the emulator agrees+ * with itself. This pins that the floci-io/floci#2516 gate accepts the byte-string+ * shape a real client sends rather than rejecting it as non-blob {@code Data}.+ */+ @Test+ void putRecordOverCborCarriesDataAsAByteString() throws Exception {+ String streamName = "cbor-binary-data-" + System.nanoTime();++ cborCall("Kinesis_20131202.CreateStream",+ AwsJsonCborController.nodeToSmithyCbor(+ JSON.createObjectNode().put("StreamName", streamName).put("ShardCount", 1)));++ byte[] payload = "hello-cbor".getBytes();+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ CBORFactory factory = (CBORFactory) CBOR.getFactory();+ try (CBORGenerator gen = factory.createGenerator(out)) {+ gen.writeStartObject();+ gen.writeStringField("StreamName", streamName);+ gen.writeBinaryField("Data", payload);+ gen.writeStringField("PartitionKey", "pk");+ gen.writeEndObject();+ }+ JsonNode putResponse = cborCall("Kinesis_20131202.PutRecord", out.toByteArray());+ assertFalse(putResponse.path("SequenceNumber").asText().isEmpty(),+ "PutRecord with a CBOR byte-string Data must succeed");++ JsonNode iteratorResponse = cborCall("Kinesis_20131202.GetShardIterator",+ AwsJsonCborController.nodeToSmithyCbor(JSON.createObjectNode()+ .put("StreamName", streamName)+ .put("ShardId", "shardId-000000000000")+ .put("ShardIteratorType", "TRIM_HORIZON")));+ JsonNode records = cborCall("Kinesis_20131202.GetRecords", AwsJsonCborController.nodeToSmithyCbor(+ JSON.createObjectNode().put("ShardIterator", iteratorResponse.path("ShardIterator").asText())));++ assertEquals(1, records.path("Records").size());+ assertEquals(Base64.getEncoder().encodeToString(payload),+ records.path("Records").get(0).path("Data").asText(),+ "the byte-string payload must land byte-exact");+ }+/*** Hand-builds a {@code GetShardIterator} CBOR request body with {@code Timestamp}* encoded exactly as a real AWS SDK for Java v2 client sends it: CBOR tag(1)diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.javaindex 3cfbc90054..5bd93df205 100644--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.kinesis;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.BinaryNode;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;@@ -13,6 +14,7 @@ import org.junit.jupiter.api.Test;import java.math.BigDecimal;import java.time.Instant;import java.util.Base64;+import java.util.List;import java.util.Map;import static org.hamcrest.MatcherAssert.assertThat;@@ -487,6 +489,179 @@ class KinesisJsonHandlerTest {assertThat(results.get(2).has("SequenceNumber"), is(true));}+ @Test+ void putRecordRejectsNonStringData() {+ createStream("test-stream");++ ObjectNode missing = MAPPER.createObjectNode();+ missing.put("StreamName", "test-stream");+ missing.put("PartitionKey", "pk1");+ ObjectNode nullData = missing.deepCopy();+ nullData.putNull("Data");+ ObjectNode boolData = missing.deepCopy();+ boolData.put("Data", true);+ ObjectNode numberData = missing.deepCopy();+ numberData.put("Data", 1234);+ ObjectNode containerData = missing.deepCopy();+ containerData.putObject("Data");++ for (ObjectNode req : List.of(missing, nullData, boolData, numberData, containerData)) {+ AwsException ex = assertThrows(AwsException.class,+ () -> handler.handle("PutRecord", req, REGION));+ assertEquals("SerializationException", ex.getErrorCode());+ assertEquals("Data must be a base64-encoded string.", ex.getMessage());+ assertEquals(400, ex.getHttpStatus());+ }+ assertEquals(0, readAllRecords("test-stream").size());+ }++ @Test+ void putRecordRejectsUndecodableData() {+ createStream("test-stream");++ ObjectNode req = MAPPER.createObjectNode();+ req.put("StreamName", "test-stream");+ req.put("Data", "!!!not-base64!!!");+ req.put("PartitionKey", "pk1");+ AwsException ex = assertThrows(AwsException.class,+ () -> handler.handle("PutRecord", req, REGION));+ assertEquals("SerializationException", ex.getErrorCode());+ assertEquals("Data is not valid base64.", ex.getMessage());+ assertEquals(400, ex.getHttpStatus());+ }++ /**+ * The CBOR transports decode Data as a binary node, not base64 text — the shape an+ * unmodified AWS SDK for Java Kinesis client sends — so the non-string gate must+ * accept it.+ */+ @Test+ void putRecordAcceptsBinaryData() {+ createStream("test-stream");++ ObjectNode req = MAPPER.createObjectNode();+ req.put("StreamName", "test-stream");+ req.set("Data", BinaryNode.valueOf(new byte[] {1, 2, 3}));+ req.put("PartitionKey", "pk1");+ assertThat(handler.handle("PutRecord", req, REGION).getStatus(), is(200));++ ArrayNode records = readAllRecords("test-stream");+ assertEquals(1, records.size());+ assertEquals("AQID", records.get(0).get("Data").asText());+ }++ @Test+ void putRecordAcceptsEmptyStringData() {+ createStream("test-stream");++ ObjectNode req = MAPPER.createObjectNode();+ req.put("StreamName", "test-stream");+ req.put("Data", "");+ req.put("PartitionKey", "pk1");+ assertThat(handler.handle("PutRecord", req, REGION).getStatus(), is(200));++ ArrayNode records = readAllRecords("test-stream");+ assertEquals(1, records.size());+ assertEquals("", records.get(0).get("Data").asText());+ }++ @Test+ void putRecordsFailsNonStringDataPerRecord() {+ createStream("test-stream");++ ObjectNode req = MAPPER.createObjectNode();+ req.put("StreamName", "test-stream");+ ArrayNode records = req.putArray("Records");+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");+ records.addObject().put("Data", "").put("PartitionKey", "pk2");+ records.addObject().put("PartitionKey", "pk3");+ records.addObject().putNull("Data").put("PartitionKey", "pk4");+ records.addObject().put("Data", true).put("PartitionKey", "pk5");+ records.addObject().put("Data", 1234).put("PartitionKey", "pk6");+ ObjectNode containerRow = records.addObject();+ containerRow.putObject("Data");+ containerRow.put("PartitionKey", "pk7");+ ObjectNode binaryRow = records.addObject();+ binaryRow.set("Data", BinaryNode.valueOf(new byte[] {1, 2, 3}));+ binaryRow.put("PartitionKey", "pk8");++ Response resp = handler.handle("PutRecords", req, REGION);+ assertThat(resp.getStatus(), is(200));+ ObjectNode body = responseEntity(resp);+ assertEquals(5, body.get("FailedRecordCount").asInt());++ ArrayNode results = (ArrayNode) body.get("Records");+ assertTrue(results.get(0).has("SequenceNumber"));+ assertTrue(results.get(1).has("SequenceNumber"));+ for (int i = 2; i <= 6; i++) {+ assertEquals("InternalFailure", results.get(i).get("ErrorCode").asText());+ assertEquals("Data must be a base64-encoded string.", results.get(i).get("ErrorMessage").asText());+ }+ assertTrue(results.get(7).has("SequenceNumber"));++ ArrayNode landed = readAllRecords("test-stream");+ assertEquals(3, landed.size());+ assertEquals("dGVzdA==", landed.get(0).get("Data").asText());+ assertEquals("", landed.get(1).get("Data").asText());+ assertEquals("AQID", landed.get(2).get("Data").asText());+ }++ @Test+ void putRecordsCountsABinaryEntryAtItsDecodedSize() {+ createStream("test-stream");++ ObjectNode req = MAPPER.createObjectNode();+ req.put("StreamName", "test-stream");+ ArrayNode records = req.putArray("Records");+ // 9 x 900,000 decoded bytes = 8.1 MB, under the 10 MiB request cap — while the+ // base64 rendering of the same entries would exceed it.+ for (int i = 0; i < 9; i++) {+ ObjectNode row = records.addObject();+ row.set("Data", BinaryNode.valueOf(new byte[900_000]));+ row.put("PartitionKey", "pk" + i);+ }++ Response resp = handler.handle("PutRecords", req, REGION);+ assertThat(resp.getStatus(), is(200));+ assertEquals(0, responseEntity(resp).get("FailedRecordCount").asInt());+ }++ @Test+ void putRecordsRejectsWholeBatchOnOversizedBinaryRecord() {+ createStream("test-stream");++ ObjectNode req = MAPPER.createObjectNode();+ req.put("StreamName", "test-stream");+ ArrayNode records = req.putArray("Records");+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");+ ObjectNode oversized = records.addObject();+ oversized.set("Data", BinaryNode.valueOf(new byte[1_048_574]));+ oversized.put("PartitionKey", "pk1");++ AwsException ex = assertThrows(AwsException.class,+ () -> handler.handle("PutRecords", req, REGION));+ assertEquals("InvalidArgumentException", ex.getErrorCode());+ assertEquals(0, readAllRecords("test-stream").size());+ }++ @Test+ void putRecordsCountsThePartitionKeyOfANonStringEntry() {+ createStream("test-stream");++ ObjectNode req = MAPPER.createObjectNode();+ req.put("StreamName", "test-stream");+ ArrayNode records = req.putArray("Records");+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");+ ObjectNode badRow = records.addObject();+ badRow.putObject("Data");+ badRow.put("PartitionKey", "x".repeat(1_048_577));++ AwsException ex = assertThrows(AwsException.class,+ () -> handler.handle("PutRecords", req, REGION));+ assertEquals("InvalidArgumentException", ex.getErrorCode());+ assertEquals(0, readAllRecords("test-stream").size());+ }+@Testvoid putRecordsRejectsUnknownStream() {ObjectNode req = MAPPER.createObjectNode();@@ -497,6 +672,24 @@ class KinesisJsonHandlerTest {assertEquals("ResourceNotFoundException", ex.getErrorCode());}+ private ArrayNode readAllRecords(String streamName) {+ ObjectNode descReq = MAPPER.createObjectNode();+ descReq.put("StreamName", streamName);+ String shardId = responseEntity(handler.handle("DescribeStream", descReq, REGION))+ .get("StreamDescription").get("Shards").get(0).get("ShardId").asText();++ ObjectNode iterReq = MAPPER.createObjectNode();+ iterReq.put("StreamName", streamName);+ iterReq.put("ShardId", shardId);+ iterReq.put("ShardIteratorType", "TRIM_HORIZON");+ String iterator = responseEntity(handler.handle("GetShardIterator", iterReq, REGION))+ .get("ShardIterator").asText();++ ObjectNode recReq = MAPPER.createObjectNode();+ recReq.put("ShardIterator", iterator);+ return (ArrayNode) responseEntity(handler.handle("GetRecords", recReq, REGION)).get("Records");+ }+private String streamArn(String name) {ObjectNode req = MAPPER.createObjectNode();req.put("StreamName", name);