Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(kinesis): return time-based MillisBehindLatest (#444)

TiMe62f2a64d7f5
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
index 11b1f23064..41c878d5d2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
@@ -393,10 +393,26 @@ public class KinesisService {
Map<String, Object> response = new HashMap<>();
response.put("Records", result);
response.put("NextShardIterator", nextIterator);
- response.put("MillisBehindLatest", Math.max(0, allRecords.size() - nextIndex));
+ response.put("MillisBehindLatest", computeMillisBehindLatest(allRecords, nextIndex));
return response;
}
+ /**
+ * Time delta in ms between the last record returned and the shard tip.
+ * Zero when caught up, the shard is empty, or no records were returned.
+ */
+ private long computeMillisBehindLatest(List<KinesisRecord> allRecords, int nextIndex) {
+ if (nextIndex <= 0 || nextIndex >= allRecords.size()) {
+ return 0L;
+ }
+ Instant lastReturned = allRecords.get(nextIndex - 1).getApproximateArrivalTimestamp();
+ Instant tip = allRecords.get(allRecords.size() - 1).getApproximateArrivalTimestamp();
+ if (lastReturned == null || tip == null) {
+ return 0L;
+ }
+ return Math.max(0L, tip.toEpochMilli() - lastReturned.toEpochMilli());
+ }
+
private KinesisStream resolveStream(String streamName, String region) {
return store.get(regionKey(region, streamName))
.orElseThrow(() -> new AwsException("ResourceNotFoundException", "Stream " + streamName + " not found", 400));
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java
index b571cf2590..1f6d7ec586 100644
--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java
@@ -4,11 +4,14 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.services.kinesis.model.KinesisConsumer;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;
import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.nio.charset.StandardCharsets;
+import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -104,6 +107,78 @@ class KinesisServiceTest {
@SuppressWarnings("unchecked")
var records = (List<?>) result.get("Records");
assertTrue(records.isEmpty());
+ assertEquals(0L, ((Number) result.get("MillisBehindLatest")).longValue());
+ }
+
+ @Test
+ void millisBehindLatestIsZeroOnEmptyShard() {
+ kinesisService.createStream("empty", 1, REGION);
+ String shardId = kinesisService.describeStream("empty", REGION).getShards().getFirst().getShardId();
+ String iterator = kinesisService.getShardIterator("empty", shardId, "TRIM_HORIZON", null, REGION);
+
+ Map<String, Object> result = kinesisService.getRecords(iterator, 10, REGION);
+
+ assertEquals(0L, ((Number) result.get("MillisBehindLatest")).longValue());
+ }
+
+ @Test
+ void millisBehindLatestIsZeroWhenCaughtUp() {
+ kinesisService.createStream("my-stream", 1, REGION);
+ kinesisService.putRecord("my-stream", "a".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ kinesisService.putRecord("my-stream", "b".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+
+ String shardId = kinesisService.describeStream("my-stream", REGION).getShards().getFirst().getShardId();
+ String iterator = kinesisService.getShardIterator("my-stream", shardId, "TRIM_HORIZON", null, REGION);
+
+ Map<String, Object> result = kinesisService.getRecords(iterator, 10, REGION);
+
+ @SuppressWarnings("unchecked")
+ var records = (List<?>) result.get("Records");
+ assertEquals(2, records.size());
+ assertEquals(0L, ((Number) result.get("MillisBehindLatest")).longValue());
+ }
+
+ @Test
+ void millisBehindLatestIsTimeDeltaWhenBatchLimitHit() {
+ kinesisService.createStream("my-stream", 1, REGION);
+ kinesisService.putRecord("my-stream", "a".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ kinesisService.putRecord("my-stream", "b".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ kinesisService.putRecord("my-stream", "c".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+
+ // Overwrite timestamps so we can assert a deterministic delta.
+ KinesisShard shard = kinesisService.describeStream("my-stream", REGION).getShards().getFirst();
+ List<KinesisRecord> records = shard.getRecords();
+ Instant base = Instant.parse("2026-01-01T00:00:00Z");
+ records.get(0).setApproximateArrivalTimestamp(base);
+ records.get(1).setApproximateArrivalTimestamp(base.plusMillis(1500));
+ records.get(2).setApproximateArrivalTimestamp(base.plusMillis(4000));
+
+ String iterator = kinesisService.getShardIterator("my-stream", shard.getShardId(), "TRIM_HORIZON", null, REGION);
+
+ Map<String, Object> result = kinesisService.getRecords(iterator, 2, REGION);
+
+ @SuppressWarnings("unchecked")
+ var returned = (List<?>) result.get("Records");
+ assertEquals(2, returned.size());
+ // Last returned = records[1] at +1500ms, tip = records[2] at +4000ms, delta = 2500ms
+ assertEquals(2500L, ((Number) result.get("MillisBehindLatest")).longValue());
+ }
+
+ @Test
+ void millisBehindLatestIsZeroWhenTimestampsMissing() {
+ kinesisService.createStream("my-stream", 1, REGION);
+ kinesisService.putRecord("my-stream", "a".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ kinesisService.putRecord("my-stream", "b".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+
+ KinesisShard shard = kinesisService.describeStream("my-stream", REGION).getShards().getFirst();
+ // Simulate a record with no arrival timestamp (e.g. legacy data or a partial put).
+ shard.getRecords().getFirst().setApproximateArrivalTimestamp(null);
+
+ String iterator = kinesisService.getShardIterator("my-stream", shard.getShardId(), "TRIM_HORIZON", null, REGION);
+ Map<String, Object> result = kinesisService.getRecords(iterator, 1, REGION);
+
+ // First record returned, second still ahead; null timestamp must not NPE.
+ assertEquals(0L, ((Number) result.get("MillisBehindLatest")).longValue());
}
@Test