Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(kinesis): reject non-blob Data instead of coercing it with asText (#2551)

soreavisdf5581e5bd7f
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
index 191990ef1d..23a54b175d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
@@ -465,9 +465,15 @@ public class KinesisJsonHandler {
private Response handlePutRecord(JsonNode request, String region) {
String streamName = resolveStreamName(request);
+ JsonNode dataNode = request.path("Data");
+ // The CBOR transports (the AWS SDK for Java's default for Kinesis) decode Data
+ // as a binary node rather than base64 text, so both blob shapes are accepted.
+ if (!dataNode.isTextual() && !dataNode.isBinary()) {
+ throw new AwsException("SerializationException", "Data must be a base64-encoded string.", 400);
+ }
byte[] data;
try {
- data = Base64.getDecoder().decode(request.path("Data").asText());
+ data = Base64.getDecoder().decode(dataNode.asText());
} catch (IllegalArgumentException e) {
throw new AwsException("SerializationException", "Data is not valid base64.", 400);
}
@@ -489,32 +495,32 @@ public class KinesisJsonHandler {
// An oversized record fails the whole request before anything is
// written — per-record ErrorCode is reserved for throughput/internal
- // failures. Successful decodes are kept for the write loop; malformed
- // Data is left null so it stays a per-record failure there. The count
- // cap is checked upfront and the byte cap as the loop goes, so neither
- // an over-long batch nor an oversized one is fully decoded before it
- // is rejected.
+ // failures. Successful decodes are kept for the write loop; a Data
+ // that is not a decodable blob is left null and fails per-record
+ // there, with its partition key still counted by the record-size
+ // check. The count cap is checked upfront and the byte cap as the
+ // loop goes, so neither an over-long batch nor an oversized one is
+ // fully decoded before it is rejected.
record Entry(JsonNode node, byte[] data) {}
List<Entry> entries = new ArrayList<>();
long totalBytes = 0;
for (JsonNode node : recordsNode) {
JsonNode dataNode = node.path("Data");
byte[] data = null;
- if (dataNode.isTextual()) {
+ if (dataNode.isTextual() || dataNode.isBinary()) {
try {
- data = Base64.getDecoder().decode(dataNode.textValue());
+ data = Base64.getDecoder().decode(dataNode.asText());
} catch (IllegalArgumentException e) {
data = null;
}
}
String partitionKey = node.path("PartitionKey").asText();
- if (data != null) {
- service.validateRecordSize(stream, data, partitionKey);
- }
+ service.validateRecordSize(stream, data, partitionKey);
// Data that did not decode still travelled in the request, so it counts toward the
// request cap at the bytes the caller sent rather than as nothing — otherwise a batch
- // of undecodable records could carry any payload past the limit. A non-string Data is
- // measured the same way, since it is not a blob AWS would have accepted either.
+ // of undecodable records could carry any payload past the limit. A Data that is
+ // neither text nor binary is measured the same way, since it is not a blob AWS
+ // would have accepted either.
totalBytes += KinesisService.recordSize(
data != null ? data.length
: dataNode.toString().getBytes(StandardCharsets.UTF_8).length,
@@ -529,8 +535,14 @@ public class KinesisJsonHandler {
for (Entry entry : entries) {
try {
- byte[] data = entry.data() != null ? entry.data()
- : Base64.getDecoder().decode(entry.node().path("Data").asText());
+ byte[] data = entry.data();
+ if (data == null) {
+ JsonNode dataNode = entry.node().path("Data");
+ if (!dataNode.isTextual() && !dataNode.isBinary()) {
+ throw new IllegalArgumentException("Data must be a base64-encoded string.");
+ }
+ data = Base64.getDecoder().decode(dataNode.asText());
+ }
String partitionKey = entry.node().path("PartitionKey").asText();
KinesisService.PutRecordResult result = service.putRecordWithShardId(streamName, data, partitionKey, region);
results.addObject()
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.java
index 5c43755fdf..c2b94a75f6 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/AwsJsonCborKinesisAtTimestampIntegrationTest.java
@@ -83,6 +83,50 @@ class AwsJsonCborKinesisAtTimestampIntegrationTest {
"AT_TIMESTAMP over CBOR must find the record written after the cutoff");
}
+ /**
+ * A real CBOR-default SDK client encodes {@code Data} as a CBOR byte string (major
+ * type 2), which Jackson decodes to a binary node — not the base64 text string
+ * Floci's own encoder writes. The request is hand-built for the same reason as the
+ * timestamp one: posting Floci-encoded bytes would only prove the emulator agrees
+ * with itself. This pins that the floci-io/floci#2516 gate accepts the byte-string
+ * shape a real client sends rather than rejecting it as non-blob {@code Data}.
+ */
+ @Test
+ void putRecordOverCborCarriesDataAsAByteString() throws Exception {
+ String streamName = "cbor-binary-data-" + System.nanoTime();
+
+ cborCall("Kinesis_20131202.CreateStream",
+ AwsJsonCborController.nodeToSmithyCbor(
+ JSON.createObjectNode().put("StreamName", streamName).put("ShardCount", 1)));
+
+ byte[] payload = "hello-cbor".getBytes();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ CBORFactory factory = (CBORFactory) CBOR.getFactory();
+ try (CBORGenerator gen = factory.createGenerator(out)) {
+ gen.writeStartObject();
+ gen.writeStringField("StreamName", streamName);
+ gen.writeBinaryField("Data", payload);
+ gen.writeStringField("PartitionKey", "pk");
+ gen.writeEndObject();
+ }
+ JsonNode putResponse = cborCall("Kinesis_20131202.PutRecord", out.toByteArray());
+ assertFalse(putResponse.path("SequenceNumber").asText().isEmpty(),
+ "PutRecord with a CBOR byte-string Data must succeed");
+
+ JsonNode iteratorResponse = cborCall("Kinesis_20131202.GetShardIterator",
+ AwsJsonCborController.nodeToSmithyCbor(JSON.createObjectNode()
+ .put("StreamName", streamName)
+ .put("ShardId", "shardId-000000000000")
+ .put("ShardIteratorType", "TRIM_HORIZON")));
+ JsonNode records = cborCall("Kinesis_20131202.GetRecords", AwsJsonCborController.nodeToSmithyCbor(
+ JSON.createObjectNode().put("ShardIterator", iteratorResponse.path("ShardIterator").asText())));
+
+ assertEquals(1, records.path("Records").size());
+ assertEquals(Base64.getEncoder().encodeToString(payload),
+ records.path("Records").get(0).path("Data").asText(),
+ "the byte-string payload must land byte-exact");
+ }
+
/**
* Hand-builds a {@code GetShardIterator} CBOR request body with {@code Timestamp}
* encoded exactly as a real AWS SDK for Java v2 client sends it: CBOR tag(1)
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
index 3cfbc90054..5bd93df205 100644
--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.kinesis;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.BinaryNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
@@ -13,6 +14,7 @@ import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
import java.time.Instant;
import java.util.Base64;
+import java.util.List;
import java.util.Map;
import static org.hamcrest.MatcherAssert.assertThat;
@@ -487,6 +489,179 @@ class KinesisJsonHandlerTest {
assertThat(results.get(2).has("SequenceNumber"), is(true));
}
+ @Test
+ void putRecordRejectsNonStringData() {
+ createStream("test-stream");
+
+ ObjectNode missing = MAPPER.createObjectNode();
+ missing.put("StreamName", "test-stream");
+ missing.put("PartitionKey", "pk1");
+ ObjectNode nullData = missing.deepCopy();
+ nullData.putNull("Data");
+ ObjectNode boolData = missing.deepCopy();
+ boolData.put("Data", true);
+ ObjectNode numberData = missing.deepCopy();
+ numberData.put("Data", 1234);
+ ObjectNode containerData = missing.deepCopy();
+ containerData.putObject("Data");
+
+ for (ObjectNode req : List.of(missing, nullData, boolData, numberData, containerData)) {
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecord", req, REGION));
+ assertEquals("SerializationException", ex.getErrorCode());
+ assertEquals("Data must be a base64-encoded string.", ex.getMessage());
+ assertEquals(400, ex.getHttpStatus());
+ }
+ assertEquals(0, readAllRecords("test-stream").size());
+ }
+
+ @Test
+ void putRecordRejectsUndecodableData() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ req.put("Data", "!!!not-base64!!!");
+ req.put("PartitionKey", "pk1");
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecord", req, REGION));
+ assertEquals("SerializationException", ex.getErrorCode());
+ assertEquals("Data is not valid base64.", ex.getMessage());
+ assertEquals(400, ex.getHttpStatus());
+ }
+
+ /**
+ * The CBOR transports decode Data as a binary node, not base64 text — the shape an
+ * unmodified AWS SDK for Java Kinesis client sends — so the non-string gate must
+ * accept it.
+ */
+ @Test
+ void putRecordAcceptsBinaryData() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ req.set("Data", BinaryNode.valueOf(new byte[] {1, 2, 3}));
+ req.put("PartitionKey", "pk1");
+ assertThat(handler.handle("PutRecord", req, REGION).getStatus(), is(200));
+
+ ArrayNode records = readAllRecords("test-stream");
+ assertEquals(1, records.size());
+ assertEquals("AQID", records.get(0).get("Data").asText());
+ }
+
+ @Test
+ void putRecordAcceptsEmptyStringData() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ req.put("Data", "");
+ req.put("PartitionKey", "pk1");
+ assertThat(handler.handle("PutRecord", req, REGION).getStatus(), is(200));
+
+ ArrayNode records = readAllRecords("test-stream");
+ assertEquals(1, records.size());
+ assertEquals("", records.get(0).get("Data").asText());
+ }
+
+ @Test
+ void putRecordsFailsNonStringDataPerRecord() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ ArrayNode records = req.putArray("Records");
+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");
+ records.addObject().put("Data", "").put("PartitionKey", "pk2");
+ records.addObject().put("PartitionKey", "pk3");
+ records.addObject().putNull("Data").put("PartitionKey", "pk4");
+ records.addObject().put("Data", true).put("PartitionKey", "pk5");
+ records.addObject().put("Data", 1234).put("PartitionKey", "pk6");
+ ObjectNode containerRow = records.addObject();
+ containerRow.putObject("Data");
+ containerRow.put("PartitionKey", "pk7");
+ ObjectNode binaryRow = records.addObject();
+ binaryRow.set("Data", BinaryNode.valueOf(new byte[] {1, 2, 3}));
+ binaryRow.put("PartitionKey", "pk8");
+
+ Response resp = handler.handle("PutRecords", req, REGION);
+ assertThat(resp.getStatus(), is(200));
+ ObjectNode body = responseEntity(resp);
+ assertEquals(5, body.get("FailedRecordCount").asInt());
+
+ ArrayNode results = (ArrayNode) body.get("Records");
+ assertTrue(results.get(0).has("SequenceNumber"));
+ assertTrue(results.get(1).has("SequenceNumber"));
+ for (int i = 2; i <= 6; i++) {
+ assertEquals("InternalFailure", results.get(i).get("ErrorCode").asText());
+ assertEquals("Data must be a base64-encoded string.", results.get(i).get("ErrorMessage").asText());
+ }
+ assertTrue(results.get(7).has("SequenceNumber"));
+
+ ArrayNode landed = readAllRecords("test-stream");
+ assertEquals(3, landed.size());
+ assertEquals("dGVzdA==", landed.get(0).get("Data").asText());
+ assertEquals("", landed.get(1).get("Data").asText());
+ assertEquals("AQID", landed.get(2).get("Data").asText());
+ }
+
+ @Test
+ void putRecordsCountsABinaryEntryAtItsDecodedSize() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ ArrayNode records = req.putArray("Records");
+ // 9 x 900,000 decoded bytes = 8.1 MB, under the 10 MiB request cap — while the
+ // base64 rendering of the same entries would exceed it.
+ for (int i = 0; i < 9; i++) {
+ ObjectNode row = records.addObject();
+ row.set("Data", BinaryNode.valueOf(new byte[900_000]));
+ row.put("PartitionKey", "pk" + i);
+ }
+
+ Response resp = handler.handle("PutRecords", req, REGION);
+ assertThat(resp.getStatus(), is(200));
+ assertEquals(0, responseEntity(resp).get("FailedRecordCount").asInt());
+ }
+
+ @Test
+ void putRecordsRejectsWholeBatchOnOversizedBinaryRecord() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ ArrayNode records = req.putArray("Records");
+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");
+ ObjectNode oversized = records.addObject();
+ oversized.set("Data", BinaryNode.valueOf(new byte[1_048_574]));
+ oversized.put("PartitionKey", "pk1");
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecords", req, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode());
+ assertEquals(0, readAllRecords("test-stream").size());
+ }
+
+ @Test
+ void putRecordsCountsThePartitionKeyOfANonStringEntry() {
+ createStream("test-stream");
+
+ ObjectNode req = MAPPER.createObjectNode();
+ req.put("StreamName", "test-stream");
+ ArrayNode records = req.putArray("Records");
+ records.addObject().put("Data", "dGVzdA==").put("PartitionKey", "pk1");
+ ObjectNode badRow = records.addObject();
+ badRow.putObject("Data");
+ badRow.put("PartitionKey", "x".repeat(1_048_577));
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("PutRecords", req, REGION));
+ assertEquals("InvalidArgumentException", ex.getErrorCode());
+ assertEquals(0, readAllRecords("test-stream").size());
+ }
+
@Test
void putRecordsRejectsUnknownStream() {
ObjectNode req = MAPPER.createObjectNode();
@@ -497,6 +672,24 @@ class KinesisJsonHandlerTest {
assertEquals("ResourceNotFoundException", ex.getErrorCode());
}
+ private ArrayNode readAllRecords(String streamName) {
+ ObjectNode descReq = MAPPER.createObjectNode();
+ descReq.put("StreamName", streamName);
+ String shardId = responseEntity(handler.handle("DescribeStream", descReq, REGION))
+ .get("StreamDescription").get("Shards").get(0).get("ShardId").asText();
+
+ ObjectNode iterReq = MAPPER.createObjectNode();
+ iterReq.put("StreamName", streamName);
+ iterReq.put("ShardId", shardId);
+ iterReq.put("ShardIteratorType", "TRIM_HORIZON");
+ String iterator = responseEntity(handler.handle("GetShardIterator", iterReq, REGION))
+ .get("ShardIterator").asText();
+
+ ObjectNode recReq = MAPPER.createObjectNode();
+ recReq.put("ShardIterator", iterator);
+ return (ArrayNode) responseEntity(handler.handle("GetRecords", recReq, REGION)).get("Records");
+ }
+
private String streamArn(String name) {
ObjectNode req = MAPPER.createObjectNode();
req.put("StreamName", name);