Light, fluffy, and always free - The AWS Local Emulator alternative
fix: serialize GetRecords ApproximateArrivalTimestamp as a plain decimal (#2359)
sheldon-nadarajah-sft5b4d1aff600d
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.javaindex 247c066bf7..656a55e811 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java@@ -365,12 +365,13 @@ public class KinesisJsonHandler {ObjectNode eventPayload = objectMapper.createObjectNode();ArrayNode recordsNode = eventPayload.putArray("Records");for (KinesisRecord rec : records) {- recordsNode.addObject()+ ObjectNode recordNode = recordsNode.addObject().put("Data", Base64.getEncoder().encodeToString(rec.getData())).put("PartitionKey", rec.getPartitionKey())- .put("SequenceNumber", rec.getSequenceNumber())- .put("ApproximateArrivalTimestamp",- rec.getApproximateArrivalTimestamp().toEpochMilli() / 1000.0);+ .put("SequenceNumber", rec.getSequenceNumber());+ if (rec.getApproximateArrivalTimestamp() != null) {+ recordNode.put("ApproximateArrivalTimestamp", epochSeconds(rec.getApproximateArrivalTimestamp()));+ }}if (continuationSeqNo != null) {eventPayload.put("ContinuationSequenceNumber", continuationSeqNo);@@ -671,7 +672,9 @@ public class KinesisJsonHandler {rNode.put("Data", Base64.getEncoder().encodeToString(rec.getData()));rNode.put("PartitionKey", rec.getPartitionKey());rNode.put("SequenceNumber", rec.getSequenceNumber());- rNode.put("ApproximateArrivalTimestamp", rec.getApproximateArrivalTimestamp().toEpochMilli() / 1000.0);+ if (rec.getApproximateArrivalTimestamp() != null) {+ rNode.put("ApproximateArrivalTimestamp", epochSeconds(rec.getApproximateArrivalTimestamp()));+ }}response.put("NextShardIterator", (String) result.get("NextShardIterator"));response.put("MillisBehindLatest", ((Number) result.get("MillisBehindLatest")).longValue());diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.javaindex df4aa5c7d6..c09e467244 100644--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java@@ -12,6 +12,7 @@ import org.junit.jupiter.api.TestMethodOrder;import java.math.BigDecimal;import java.nio.ByteBuffer;+import java.util.regex.Matcher;import java.util.regex.Pattern;import static io.restassured.RestAssured.given;@@ -934,6 +935,39 @@ class KinesisIntegrationTest {.body("Records.size()", equalTo(3));}+ @Test+ @Order(57)+ void getRecordsReturnsPlainDecimalArrivalTimestamp() {+ String stream = "at-timestamp-plain-decimal";+ String shardId = atTimestampCreateStream(stream);+ atTimestampPutAndGetSequence(stream, "cmVjMA==");++ // Timestamp at epoch 1s (before the record) so GetRecords returns it via the same+ // AT_TIMESTAMP resolution path exercised by the other tests in this group.+ String iterator = atTimestampIterator(stream, shardId, 1.0);++ String responseBody = given()+ .header("X-Amz-Target", "Kinesis_20131202.GetRecords")+ .contentType(KINESIS_CONTENT_TYPE)+ .body("{\"ShardIterator\": \"" + iterator + "\"}")+ .when().post("/")+ .then().statusCode(200)+ .body("Records.size()", equalTo(1))+ .extract().asString();++ // ApproximateArrivalTimestamp must be a plain decimal lexeme - real 2020s-era epoch+ // seconds render as a large double that Java serializes in scientific notation+ // (e.g. "1.786959659083E9"), which AWS SDK for Java v2's timestamp unmarshaller cannot+ // parse despite the response returning HTTP 200 (see floci-io/floci#2099 for the same+ // defect against DescribeStream/DescribeStreamSummary's StreamCreationTimestamp).+ Matcher matcher = Pattern+ .compile("\"ApproximateArrivalTimestamp\"\\s*:\\s*(-?\\d+(?:\\.\\d+)?)(?=\\s*[,}])")+ .matcher(responseBody);+ assertTrue(matcher.find(), "timestamp must be serialized as a plain JSON decimal: " + responseBody);+ assertFalse(matcher.group(1).contains("E"));+ assertFalse(matcher.group(1).contains("e"));+ }+@Test@Order(51)void atTimestampBeforeFirstRecordReturnsAll() {diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.javaindex 0a7f909704..fc754f7dfa 100644--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java@@ -778,6 +778,121 @@ class KinesisJsonHandlerTest {assertEquals("ResourceNotFoundException", ex.getErrorCode());}+ @Test+ void getRecordsSerializesApproximateArrivalTimestampAsPlainDecimal() throws Exception {+ createStream("test-stream");++ ObjectNode putReq = MAPPER.createObjectNode();+ putReq.put("StreamName", "test-stream");+ putReq.put("Data", "dGVzdA==");+ putReq.put("PartitionKey", "pk1");+ handler.handle("PutRecord", putReq, REGION);++ ObjectNode descReq = MAPPER.createObjectNode();+ descReq.put("StreamName", "test-stream");+ String shardId = responseEntity(handler.handle("DescribeStream", descReq, REGION))+ .get("StreamDescription").get("Shards").get(0).get("ShardId").asText();++ // A fixed, realistic (2020s-era) arrival timestamp - Instant.now() would also trigger+ // the bug, but a fixed value makes the expected serialized decimal assertable exactly.+ service.describeStream("test-stream", REGION).getShards().get(0).getRecords().get(0)+ .setApproximateArrivalTimestamp(Instant.ofEpochMilli(1_786_959_659_083L));++ ObjectNode iterReq = MAPPER.createObjectNode();+ iterReq.put("StreamName", "test-stream");+ iterReq.put("ShardId", shardId);+ iterReq.put("ShardIteratorType", "TRIM_HORIZON");+ String iterator = responseEntity(handler.handle("GetShardIterator", iterReq, REGION))+ .get("ShardIterator").asText();++ ObjectNode recReq = MAPPER.createObjectNode();+ recReq.put("ShardIterator", iterator);+ JsonNode timestampNode = responseEntity(handler.handle("GetRecords", recReq, REGION))+ .get("Records").get(0).get("ApproximateArrivalTimestamp");++ assertTrue(timestampNode.isNumber());+ assertEquals(new BigDecimal("1786959659.083"), timestampNode.decimalValue());++ String serialized = MAPPER.writeValueAsString(timestampNode);+ assertEquals("1786959659.083", serialized);+ assertFalse(serialized.contains("E"));+ assertFalse(serialized.contains("e"));+ }++ @Test+ void getRecordsWholeSecondArrivalTimestampRemainsNumeric() throws Exception {+ createStream("test-stream");++ ObjectNode putReq = MAPPER.createObjectNode();+ putReq.put("StreamName", "test-stream");+ putReq.put("Data", "dGVzdA==");+ putReq.put("PartitionKey", "pk1");+ handler.handle("PutRecord", putReq, REGION);++ ObjectNode descReq = MAPPER.createObjectNode();+ descReq.put("StreamName", "test-stream");+ String shardId = responseEntity(handler.handle("DescribeStream", descReq, REGION))+ .get("StreamDescription").get("Shards").get(0).get("ShardId").asText();++ service.describeStream("test-stream", REGION).getShards().get(0).getRecords().get(0)+ .setApproximateArrivalTimestamp(Instant.ofEpochMilli(1_786_959_659_000L));++ ObjectNode iterReq = MAPPER.createObjectNode();+ iterReq.put("StreamName", "test-stream");+ iterReq.put("ShardId", shardId);+ iterReq.put("ShardIteratorType", "TRIM_HORIZON");+ String iterator = responseEntity(handler.handle("GetShardIterator", iterReq, REGION))+ .get("ShardIterator").asText();++ ObjectNode recReq = MAPPER.createObjectNode();+ recReq.put("ShardIterator", iterator);+ JsonNode timestampNode = responseEntity(handler.handle("GetRecords", recReq, REGION))+ .get("Records").get(0).get("ApproximateArrivalTimestamp");++ assertEquals(new BigDecimal("1786959659.000"), timestampNode.decimalValue());+ assertEquals("1786959659.000", MAPPER.writeValueAsString(timestampNode));+ }++ @Test+ void getRecordsOmitsArrivalTimestampRatherThanThrowingWhenNull() throws Exception {+ // A record loaded back from hybrid/persistent JSON-backed storage (KinesisRecord is+ // @RegisterForReflection with a no-arg constructor) can have a null+ // approximateArrivalTimestamp if it was persisted by an older Floci version, or the+ // field was otherwise dropped - GetRecords must not NPE serializing the whole response+ // just because one record's timestamp is missing.+ createStream("test-stream");++ ObjectNode putReq = MAPPER.createObjectNode();+ putReq.put("StreamName", "test-stream");+ putReq.put("Data", "dGVzdA==");+ putReq.put("PartitionKey", "pk1");+ handler.handle("PutRecord", putReq, REGION);++ ObjectNode descReq = MAPPER.createObjectNode();+ descReq.put("StreamName", "test-stream");+ String shardId = responseEntity(handler.handle("DescribeStream", descReq, REGION))+ .get("StreamDescription").get("Shards").get(0).get("ShardId").asText();++ service.describeStream("test-stream", REGION).getShards().get(0).getRecords().get(0)+ .setApproximateArrivalTimestamp(null);++ ObjectNode iterReq = MAPPER.createObjectNode();+ iterReq.put("StreamName", "test-stream");+ iterReq.put("ShardId", shardId);+ iterReq.put("ShardIteratorType", "TRIM_HORIZON");+ String iterator = responseEntity(handler.handle("GetShardIterator", iterReq, REGION))+ .get("ShardIterator").asText();++ ObjectNode recReq = MAPPER.createObjectNode();+ recReq.put("ShardIterator", iterator);+ Response resp = handler.handle("GetRecords", recReq, REGION);+ assertThat(resp.getStatus(), is(200));++ ObjectNode record = (ObjectNode) responseEntity(resp).get("Records").get(0);+ assertFalse(record.has("ApproximateArrivalTimestamp"));+ assertTrue(record.has("SequenceNumber"));+ }+private ArrayNode readAllRecords(String streamName) {ObjectNode descReq = MAPPER.createObjectNode();descReq.put("StreamName", streamName);