Light, fluffy, and always free - The AWS Local Emulator alternative
fix(kinesis): resolve LATEST shard iterator position at GetShardIterator time (#1901)
Feyikemi Ogunsanya3b760bd8e51b
diff --git a/docs/services/kinesis.md b/docs/services/kinesis.mdindex 7555e0ee01..f1a0030507 100644--- a/docs/services/kinesis.md+++ b/docs/services/kinesis.md@@ -49,14 +49,18 @@ aws kinesis describe-stream --stream-name events --endpoint-url $AWS_ENDPOINT_URaws kinesis describe-stream --stream-arn arn:aws:kinesis:us-east-1:000000000000:stream/events --endpoint-url $AWS_ENDPOINT_URL```+## Shard Iterators++`GetShardIterator` supports all five iterator types: `TRIM_HORIZON`, `LATEST`, `AT_SEQUENCE_NUMBER`, `AFTER_SEQUENCE_NUMBER`, `AT_TIMESTAMP`.++A `LATEST` iterator is positioned at the shard tip at the moment the iterator is created, matching AWS: records written after the iterator was obtained are returned, records written before are not. This supports the standard tailing pattern — obtain a `LATEST` iterator, trigger the action that produces the record, then poll `GetRecords` following `NextShardIterator`.+## Configuration| Variable | Default | Description ||---|---|---|| `FLOCI_SERVICES_KINESIS_ENABLED` | `true` | Enable or disable the service |-## Examples-## Enhanced Fan-Out (EFO)`SubscribeToShard` uses a snapshot-and-close model: the server returns one batch of records as a binary EventStream response and closes the connection. The SDK resubscribes automatically using the `ContinuationSequenceNumber` from the last delivered record. All five `StartingPosition` types are supported: `TRIM_HORIZON`, `LATEST`, `AT_SEQUENCE_NUMBER`, `AFTER_SEQUENCE_NUMBER`, `AT_TIMESTAMP`.diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.javaindex cea73014ff..68097da484 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java@@ -378,14 +378,16 @@ public class KinesisService {public String getShardIterator(String streamName, String shardId, String type, String sequenceNumber,Long timestampMillis, String region) {- resolveStream(streamName, region); // validate exists+ KinesisStream stream = resolveStream(streamName, region);// Format: streamName|shardId|type|sequenceNumber|index|timestampMillis// The 6th slot was added for AT_TIMESTAMP; empty for other iterator types.// Old 5-part iterators still decode via split(-1) compatibility in getRecords.+ // For LATEST the index slot carries the shard tip at iterator creation time,+ // so records written afterwards are visible to getRecords.String raw = String.format("%s|%s|%s|%s|%d|%s",streamName, shardId, type,sequenceNumber != null ? sequenceNumber : "",- 0,+ iteratorStartIndex(stream, shardId, type),timestampMillis != null ? timestampMillis.toString() : "");return Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8));}@@ -407,6 +409,16 @@ public class KinesisService {return parts;}+ private int iteratorStartIndex(KinesisStream stream, String shardId, String type) {+ // AWS validates the shard at GetShardIterator time for every iterator type.+ KinesisShard shard = stream.getShards().stream()+ .filter(s -> s.getShardId().equals(shardId))+ .findFirst()+ .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));+ // LATEST resumes from the tip snapshot taken now; other types resolve in getRecords.+ return "LATEST".equals(type) ? shard.getRecords().size() : 0;+ }+private int parseIteratorIndex(String value) {try {return Integer.parseInt(value);@@ -441,11 +453,11 @@ public class KinesisService {List<KinesisRecord> allRecords = shard.getRecords();int startIndex = 0;- // Simple implementation of iterator types- if ("TRIM_HORIZON".equals(type)) {+ // Simple implementation of iterator types.+ // LATEST resumes from the shard tip snapshot encoded at GetShardIterator time,+ // so records appended after the iterator was obtained are returned.+ if ("TRIM_HORIZON".equals(type) || "LATEST".equals(type)) {startIndex = lastIndex;- } else if ("LATEST".equals(type)) {- startIndex = allRecords.size();} else if ("AT_SEQUENCE_NUMBER".equals(type)) {for (int i = 0; i < allRecords.size(); i++) {if (allRecords.get(i).getSequenceNumber().equals(startSeq)) {@@ -531,10 +543,11 @@ public class KinesisService {public String getShardIteratorForAccount(String accountId, String streamName, String shardId,String type, String sequenceNumber, String region) {- resolveStreamForAccount(accountId, streamName, region);+ KinesisStream stream = resolveStreamForAccount(accountId, streamName, region);String raw = String.format("%s|%s|%s|%s|%d|",streamName, shardId, type,- sequenceNumber != null ? sequenceNumber : "", 0);+ sequenceNumber != null ? sequenceNumber : "",+ iteratorStartIndex(stream, shardId, type));return Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8));}@@ -555,10 +568,9 @@ public class KinesisService {List<KinesisRecord> allRecords = shard.getRecords();int startIndex = 0;- if ("TRIM_HORIZON".equals(type)) {+ // LATEST resumes from the shard tip snapshot encoded at GetShardIterator time.+ if ("TRIM_HORIZON".equals(type) || "LATEST".equals(type)) {startIndex = lastIndex;- } else if ("LATEST".equals(type)) {- startIndex = allRecords.size();} else if ("AFTER_SEQUENCE_NUMBER".equals(type)) {for (int i = 0; i < allRecords.size(); i++) {if (allRecords.get(i).getSequenceNumber().equals(startSeq)) {diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.javaindex f837a8d0dd..42ee0afb1b 100644--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java@@ -108,6 +108,50 @@ class KinesisServiceTest {assertEquals(0L, ((Number) result.get("MillisBehindLatest")).longValue());}+ @Test+ void latestIteratorReturnsRecordsWrittenAfterItWasObtained() {+ kinesisService.createStream("my-stream", 1, REGION);+ kinesisService.putRecord("my-stream", "before".getBytes(StandardCharsets.UTF_8), "pk", REGION);++ String shardId = kinesisService.describeStream("my-stream", REGION).getShards().getFirst().getShardId();+ String iterator = kinesisService.getShardIterator("my-stream", shardId, "LATEST", null, REGION);++ kinesisService.putRecord("my-stream", "after".getBytes(StandardCharsets.UTF_8), "pk", REGION);++ Map<String, Object> result = kinesisService.getRecords(iterator, 10, REGION);+ var records = (List<?>) result.get("Records");+ assertEquals(1, records.size());+ assertEquals("after", new String(((KinesisRecord) records.getFirst()).getData(), StandardCharsets.UTF_8));+ }++ @Test+ void latestIteratorContinuationPicksUpSubsequentWrites() {+ kinesisService.createStream("my-stream", 1, REGION);+ String shardId = kinesisService.describeStream("my-stream", REGION).getShards().getFirst().getShardId();+ String iterator = kinesisService.getShardIterator("my-stream", shardId, "LATEST", null, REGION);++ // First poll before any write: empty, but the continuation iterator must not skip ahead.+ Map<String, Object> first = kinesisService.getRecords(iterator, 10, REGION);+ assertTrue(((List<?>) first.get("Records")).isEmpty());++ kinesisService.putRecord("my-stream", "tailed".getBytes(StandardCharsets.UTF_8), "pk", REGION);++ Map<String, Object> second = kinesisService.getRecords((String) first.get("NextShardIterator"), 10, REGION);+ var records = (List<?>) second.get("Records");+ assertEquals(1, records.size());+ assertEquals("tailed", new String(((KinesisRecord) records.getFirst()).getData(), StandardCharsets.UTF_8));+ }++ @Test+ void getShardIteratorUnknownShardThrowsForAllIteratorTypes() {+ kinesisService.createStream("my-stream", 1, REGION);+ for (String type : List.of("LATEST", "TRIM_HORIZON", "AT_SEQUENCE_NUMBER", "AFTER_SEQUENCE_NUMBER", "AT_TIMESTAMP")) {+ AwsException ex = assertThrows(AwsException.class, () ->+ kinesisService.getShardIterator("my-stream", "shardId-999999999999", type, "1", 0L, REGION));+ assertEquals("ResourceNotFoundException", ex.getErrorCode(), type);+ }+ }+@Testvoid millisBehindLatestIsZeroOnEmptyShard() {kinesisService.createStream("empty", 1, REGION);