Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix: serialize GetRecords ApproximateArrivalTimestamp as a plain decimal (#2359)

sheldon-nadarajah-sft5b4d1aff600d
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
index 247c066bf7..656a55e811 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
@@ -365,12 +365,13 @@ public class KinesisJsonHandler {
ObjectNode eventPayload = objectMapper.createObjectNode();
ArrayNode recordsNode = eventPayload.putArray("Records");
for (KinesisRecord rec : records) {
- recordsNode.addObject()
+ ObjectNode recordNode = recordsNode.addObject()
.put("Data", Base64.getEncoder().encodeToString(rec.getData()))
.put("PartitionKey", rec.getPartitionKey())
- .put("SequenceNumber", rec.getSequenceNumber())
- .put("ApproximateArrivalTimestamp",
- rec.getApproximateArrivalTimestamp().toEpochMilli() / 1000.0);
+ .put("SequenceNumber", rec.getSequenceNumber());
+ if (rec.getApproximateArrivalTimestamp() != null) {
+ recordNode.put("ApproximateArrivalTimestamp", epochSeconds(rec.getApproximateArrivalTimestamp()));
+ }
}
if (continuationSeqNo != null) {
eventPayload.put("ContinuationSequenceNumber", continuationSeqNo);
@@ -671,7 +672,9 @@ public class KinesisJsonHandler {
rNode.put("Data", Base64.getEncoder().encodeToString(rec.getData()));
rNode.put("PartitionKey", rec.getPartitionKey());
rNode.put("SequenceNumber", rec.getSequenceNumber());
- rNode.put("ApproximateArrivalTimestamp", rec.getApproximateArrivalTimestamp().toEpochMilli() / 1000.0);
+ if (rec.getApproximateArrivalTimestamp() != null) {
+ rNode.put("ApproximateArrivalTimestamp", epochSeconds(rec.getApproximateArrivalTimestamp()));
+ }
}
response.put("NextShardIterator", (String) result.get("NextShardIterator"));
response.put("MillisBehindLatest", ((Number) result.get("MillisBehindLatest")).longValue());
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java
index df4aa5c7d6..c09e467244 100644
--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java
@@ -12,6 +12,7 @@ import org.junit.jupiter.api.TestMethodOrder;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static io.restassured.RestAssured.given;
@@ -934,6 +935,39 @@ class KinesisIntegrationTest {
.body("Records.size()", equalTo(3));
}
+ @Test
+ @Order(57)
+ void getRecordsReturnsPlainDecimalArrivalTimestamp() {
+ String stream = "at-timestamp-plain-decimal";
+ String shardId = atTimestampCreateStream(stream);
+ atTimestampPutAndGetSequence(stream, "cmVjMA==");
+
+ // Timestamp at epoch 1s (before the record) so GetRecords returns it via the same
+ // AT_TIMESTAMP resolution path exercised by the other tests in this group.
+ String iterator = atTimestampIterator(stream, shardId, 1.0);
+
+ String responseBody = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetRecords")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("{\"ShardIterator\": \"" + iterator + "\"}")
+ .when().post("/")
+ .then().statusCode(200)
+ .body("Records.size()", equalTo(1))
+ .extract().asString();
+
+ // ApproximateArrivalTimestamp must be a plain decimal lexeme - real 2020s-era epoch
+ // seconds render as a large double that Java serializes in scientific notation
+ // (e.g. "1.786959659083E9"), which AWS SDK for Java v2's timestamp unmarshaller cannot
+ // parse despite the response returning HTTP 200 (see floci-io/floci#2099 for the same
+ // defect against DescribeStream/DescribeStreamSummary's StreamCreationTimestamp).
+ Matcher matcher = Pattern
+ .compile("\"ApproximateArrivalTimestamp\"\\s*:\\s*(-?\\d+(?:\\.\\d+)?)(?=\\s*[,}])")
+ .matcher(responseBody);
+ assertTrue(matcher.find(), "timestamp must be serialized as a plain JSON decimal: " + responseBody);
+ assertFalse(matcher.group(1).contains("E"));
+ assertFalse(matcher.group(1).contains("e"));
+ }
+
@Test
@Order(51)
void atTimestampBeforeFirstRecordReturnsAll() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
index 0a7f909704..fc754f7dfa 100644
--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
@@ -778,6 +778,121 @@ class KinesisJsonHandlerTest {
assertEquals("ResourceNotFoundException", ex.getErrorCode());
}
+ @Test
+ void getRecordsSerializesApproximateArrivalTimestampAsPlainDecimal() throws Exception {
+ createStream("test-stream");
+
+ ObjectNode putReq = MAPPER.createObjectNode();
+ putReq.put("StreamName", "test-stream");
+ putReq.put("Data", "dGVzdA==");
+ putReq.put("PartitionKey", "pk1");
+ handler.handle("PutRecord", putReq, REGION);
+
+ ObjectNode descReq = MAPPER.createObjectNode();
+ descReq.put("StreamName", "test-stream");
+ String shardId = responseEntity(handler.handle("DescribeStream", descReq, REGION))
+ .get("StreamDescription").get("Shards").get(0).get("ShardId").asText();
+
+ // A fixed, realistic (2020s-era) arrival timestamp - Instant.now() would also trigger
+ // the bug, but a fixed value makes the expected serialized decimal assertable exactly.
+ service.describeStream("test-stream", REGION).getShards().get(0).getRecords().get(0)
+ .setApproximateArrivalTimestamp(Instant.ofEpochMilli(1_786_959_659_083L));
+
+ ObjectNode iterReq = MAPPER.createObjectNode();
+ iterReq.put("StreamName", "test-stream");
+ iterReq.put("ShardId", shardId);
+ iterReq.put("ShardIteratorType", "TRIM_HORIZON");
+ String iterator = responseEntity(handler.handle("GetShardIterator", iterReq, REGION))
+ .get("ShardIterator").asText();
+
+ ObjectNode recReq = MAPPER.createObjectNode();
+ recReq.put("ShardIterator", iterator);
+ JsonNode timestampNode = responseEntity(handler.handle("GetRecords", recReq, REGION))
+ .get("Records").get(0).get("ApproximateArrivalTimestamp");
+
+ assertTrue(timestampNode.isNumber());
+ assertEquals(new BigDecimal("1786959659.083"), timestampNode.decimalValue());
+
+ String serialized = MAPPER.writeValueAsString(timestampNode);
+ assertEquals("1786959659.083", serialized);
+ assertFalse(serialized.contains("E"));
+ assertFalse(serialized.contains("e"));
+ }
+
+ @Test
+ void getRecordsWholeSecondArrivalTimestampRemainsNumeric() throws Exception {
+ createStream("test-stream");
+
+ ObjectNode putReq = MAPPER.createObjectNode();
+ putReq.put("StreamName", "test-stream");
+ putReq.put("Data", "dGVzdA==");
+ putReq.put("PartitionKey", "pk1");
+ handler.handle("PutRecord", putReq, REGION);
+
+ ObjectNode descReq = MAPPER.createObjectNode();
+ descReq.put("StreamName", "test-stream");
+ String shardId = responseEntity(handler.handle("DescribeStream", descReq, REGION))
+ .get("StreamDescription").get("Shards").get(0).get("ShardId").asText();
+
+ service.describeStream("test-stream", REGION).getShards().get(0).getRecords().get(0)
+ .setApproximateArrivalTimestamp(Instant.ofEpochMilli(1_786_959_659_000L));
+
+ ObjectNode iterReq = MAPPER.createObjectNode();
+ iterReq.put("StreamName", "test-stream");
+ iterReq.put("ShardId", shardId);
+ iterReq.put("ShardIteratorType", "TRIM_HORIZON");
+ String iterator = responseEntity(handler.handle("GetShardIterator", iterReq, REGION))
+ .get("ShardIterator").asText();
+
+ ObjectNode recReq = MAPPER.createObjectNode();
+ recReq.put("ShardIterator", iterator);
+ JsonNode timestampNode = responseEntity(handler.handle("GetRecords", recReq, REGION))
+ .get("Records").get(0).get("ApproximateArrivalTimestamp");
+
+ assertEquals(new BigDecimal("1786959659.000"), timestampNode.decimalValue());
+ assertEquals("1786959659.000", MAPPER.writeValueAsString(timestampNode));
+ }
+
+ @Test
+ void getRecordsOmitsArrivalTimestampRatherThanThrowingWhenNull() throws Exception {
+ // A record loaded back from hybrid/persistent JSON-backed storage (KinesisRecord is
+ // @RegisterForReflection with a no-arg constructor) can have a null
+ // approximateArrivalTimestamp if it was persisted by an older Floci version, or the
+ // field was otherwise dropped - GetRecords must not NPE serializing the whole response
+ // just because one record's timestamp is missing.
+ createStream("test-stream");
+
+ ObjectNode putReq = MAPPER.createObjectNode();
+ putReq.put("StreamName", "test-stream");
+ putReq.put("Data", "dGVzdA==");
+ putReq.put("PartitionKey", "pk1");
+ handler.handle("PutRecord", putReq, REGION);
+
+ ObjectNode descReq = MAPPER.createObjectNode();
+ descReq.put("StreamName", "test-stream");
+ String shardId = responseEntity(handler.handle("DescribeStream", descReq, REGION))
+ .get("StreamDescription").get("Shards").get(0).get("ShardId").asText();
+
+ service.describeStream("test-stream", REGION).getShards().get(0).getRecords().get(0)
+ .setApproximateArrivalTimestamp(null);
+
+ ObjectNode iterReq = MAPPER.createObjectNode();
+ iterReq.put("StreamName", "test-stream");
+ iterReq.put("ShardId", shardId);
+ iterReq.put("ShardIteratorType", "TRIM_HORIZON");
+ String iterator = responseEntity(handler.handle("GetShardIterator", iterReq, REGION))
+ .get("ShardIterator").asText();
+
+ ObjectNode recReq = MAPPER.createObjectNode();
+ recReq.put("ShardIterator", iterator);
+ Response resp = handler.handle("GetRecords", recReq, REGION);
+ assertThat(resp.getStatus(), is(200));
+
+ ObjectNode record = (ObjectNode) responseEntity(resp).get("Records").get(0);
+ assertFalse(record.has("ApproximateArrivalTimestamp"));
+ assertTrue(record.has("SequenceNumber"));
+ }
+
private ArrayNode readAllRecords(String streamName) {
ObjectNode descReq = MAPPER.createObjectNode();
descReq.put("StreamName", streamName);