Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(kinesis): resolve LATEST shard iterator position at GetShardIterator time (#1901)

Feyikemi Ogunsanya3b760bd8e51b
diff --git a/docs/services/kinesis.md b/docs/services/kinesis.md
index 7555e0ee01..f1a0030507 100644
--- a/docs/services/kinesis.md
+++ b/docs/services/kinesis.md
@@ -49,14 +49,18 @@ aws kinesis describe-stream --stream-name events --endpoint-url $AWS_ENDPOINT_UR
aws kinesis describe-stream --stream-arn arn:aws:kinesis:us-east-1:000000000000:stream/events --endpoint-url $AWS_ENDPOINT_URL
```
+## Shard Iterators
+
+`GetShardIterator` supports all five iterator types: `TRIM_HORIZON`, `LATEST`, `AT_SEQUENCE_NUMBER`, `AFTER_SEQUENCE_NUMBER`, `AT_TIMESTAMP`.
+
+A `LATEST` iterator is positioned at the shard tip at the moment the iterator is created, matching AWS: records written after the iterator was obtained are returned, records written before are not. This supports the standard tailing pattern — obtain a `LATEST` iterator, trigger the action that produces the record, then poll `GetRecords` following `NextShardIterator`.
+
## Configuration
| Variable | Default | Description |
|---|---|---|
| `FLOCI_SERVICES_KINESIS_ENABLED` | `true` | Enable or disable the service |
-## Examples
-
## Enhanced Fan-Out (EFO)
`SubscribeToShard` uses a snapshot-and-close model: the server returns one batch of records as a binary EventStream response and closes the connection. The SDK resubscribes automatically using the `ContinuationSequenceNumber` from the last delivered record. All five `StartingPosition` types are supported: `TRIM_HORIZON`, `LATEST`, `AT_SEQUENCE_NUMBER`, `AFTER_SEQUENCE_NUMBER`, `AT_TIMESTAMP`.
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
index cea73014ff..68097da484 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
@@ -378,14 +378,16 @@ public class KinesisService {
public String getShardIterator(String streamName, String shardId, String type, String sequenceNumber,
Long timestampMillis, String region) {
- resolveStream(streamName, region); // validate exists
+ KinesisStream stream = resolveStream(streamName, region);
// Format: streamName|shardId|type|sequenceNumber|index|timestampMillis
// The 6th slot was added for AT_TIMESTAMP; empty for other iterator types.
// Old 5-part iterators still decode via split(-1) compatibility in getRecords.
+ // For LATEST the index slot carries the shard tip at iterator creation time,
+ // so records written afterwards are visible to getRecords.
String raw = String.format("%s|%s|%s|%s|%d|%s",
streamName, shardId, type,
sequenceNumber != null ? sequenceNumber : "",
- 0,
+ iteratorStartIndex(stream, shardId, type),
timestampMillis != null ? timestampMillis.toString() : "");
return Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8));
}
@@ -407,6 +409,16 @@ public class KinesisService {
return parts;
}
+ private int iteratorStartIndex(KinesisStream stream, String shardId, String type) {
+ // AWS validates the shard at GetShardIterator time for every iterator type.
+ KinesisShard shard = stream.getShards().stream()
+ .filter(s -> s.getShardId().equals(shardId))
+ .findFirst()
+ .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));
+ // LATEST resumes from the tip snapshot taken now; other types resolve in getRecords.
+ return "LATEST".equals(type) ? shard.getRecords().size() : 0;
+ }
+
private int parseIteratorIndex(String value) {
try {
return Integer.parseInt(value);
@@ -441,11 +453,11 @@ public class KinesisService {
List<KinesisRecord> allRecords = shard.getRecords();
int startIndex = 0;
- // Simple implementation of iterator types
- if ("TRIM_HORIZON".equals(type)) {
+ // Simple implementation of iterator types.
+ // LATEST resumes from the shard tip snapshot encoded at GetShardIterator time,
+ // so records appended after the iterator was obtained are returned.
+ if ("TRIM_HORIZON".equals(type) || "LATEST".equals(type)) {
startIndex = lastIndex;
- } else if ("LATEST".equals(type)) {
- startIndex = allRecords.size();
} else if ("AT_SEQUENCE_NUMBER".equals(type)) {
for (int i = 0; i < allRecords.size(); i++) {
if (allRecords.get(i).getSequenceNumber().equals(startSeq)) {
@@ -531,10 +543,11 @@ public class KinesisService {
public String getShardIteratorForAccount(String accountId, String streamName, String shardId,
String type, String sequenceNumber, String region) {
- resolveStreamForAccount(accountId, streamName, region);
+ KinesisStream stream = resolveStreamForAccount(accountId, streamName, region);
String raw = String.format("%s|%s|%s|%s|%d|",
streamName, shardId, type,
- sequenceNumber != null ? sequenceNumber : "", 0);
+ sequenceNumber != null ? sequenceNumber : "",
+ iteratorStartIndex(stream, shardId, type));
return Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8));
}
@@ -555,10 +568,9 @@ public class KinesisService {
List<KinesisRecord> allRecords = shard.getRecords();
int startIndex = 0;
- if ("TRIM_HORIZON".equals(type)) {
+ // LATEST resumes from the shard tip snapshot encoded at GetShardIterator time.
+ if ("TRIM_HORIZON".equals(type) || "LATEST".equals(type)) {
startIndex = lastIndex;
- } else if ("LATEST".equals(type)) {
- startIndex = allRecords.size();
} else if ("AFTER_SEQUENCE_NUMBER".equals(type)) {
for (int i = 0; i < allRecords.size(); i++) {
if (allRecords.get(i).getSequenceNumber().equals(startSeq)) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java
index f837a8d0dd..42ee0afb1b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java
@@ -108,6 +108,50 @@ class KinesisServiceTest {
assertEquals(0L, ((Number) result.get("MillisBehindLatest")).longValue());
}
+ @Test
+ void latestIteratorReturnsRecordsWrittenAfterItWasObtained() {
+ kinesisService.createStream("my-stream", 1, REGION);
+ kinesisService.putRecord("my-stream", "before".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+
+ String shardId = kinesisService.describeStream("my-stream", REGION).getShards().getFirst().getShardId();
+ String iterator = kinesisService.getShardIterator("my-stream", shardId, "LATEST", null, REGION);
+
+ kinesisService.putRecord("my-stream", "after".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+
+ Map<String, Object> result = kinesisService.getRecords(iterator, 10, REGION);
+ var records = (List<?>) result.get("Records");
+ assertEquals(1, records.size());
+ assertEquals("after", new String(((KinesisRecord) records.getFirst()).getData(), StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void latestIteratorContinuationPicksUpSubsequentWrites() {
+ kinesisService.createStream("my-stream", 1, REGION);
+ String shardId = kinesisService.describeStream("my-stream", REGION).getShards().getFirst().getShardId();
+ String iterator = kinesisService.getShardIterator("my-stream", shardId, "LATEST", null, REGION);
+
+ // First poll before any write: empty, but the continuation iterator must not skip ahead.
+ Map<String, Object> first = kinesisService.getRecords(iterator, 10, REGION);
+ assertTrue(((List<?>) first.get("Records")).isEmpty());
+
+ kinesisService.putRecord("my-stream", "tailed".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+
+ Map<String, Object> second = kinesisService.getRecords((String) first.get("NextShardIterator"), 10, REGION);
+ var records = (List<?>) second.get("Records");
+ assertEquals(1, records.size());
+ assertEquals("tailed", new String(((KinesisRecord) records.getFirst()).getData(), StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void getShardIteratorUnknownShardThrowsForAllIteratorTypes() {
+ kinesisService.createStream("my-stream", 1, REGION);
+ for (String type : List.of("LATEST", "TRIM_HORIZON", "AT_SEQUENCE_NUMBER", "AFTER_SEQUENCE_NUMBER", "AT_TIMESTAMP")) {
+ AwsException ex = assertThrows(AwsException.class, () ->
+ kinesisService.getShardIterator("my-stream", "shardId-999999999999", type, "1", 0L, REGION));
+ assertEquals("ResourceNotFoundException", ex.getErrorCode(), type);
+ }
+ }
+
@Test
void millisBehindLatestIsZeroOnEmptyShard() {
kinesisService.createStream("empty", 1, REGION);