Light, fluffy, and always free - The AWS Local Emulator alternative
fix(kinesis): signal closed shard exhaustion and return child shards (#6716)
TiM77f247764140
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.javaindex 821c3c3ebf..0f4a4ab7d3 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.java@@ -13,14 +13,17 @@ import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;import software.amazon.awssdk.http.SdkHttpRequest;import software.amazon.awssdk.regions.Region;import software.amazon.awssdk.services.kinesis.KinesisClient;+import software.amazon.awssdk.services.kinesis.model.ChildShard;import software.amazon.awssdk.services.kinesis.model.CreateStreamRequest;import software.amazon.awssdk.services.kinesis.model.DeleteStreamRequest;import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryRequest;import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryResponse;import software.amazon.awssdk.services.kinesis.model.GetRecordsRequest;+import software.amazon.awssdk.services.kinesis.model.GetRecordsResponse;import software.amazon.awssdk.services.kinesis.model.GetShardIteratorRequest;import software.amazon.awssdk.services.kinesis.model.ListShardsRequest;import software.amazon.awssdk.services.kinesis.model.ListShardsResponse;+import software.amazon.awssdk.services.kinesis.model.MergeShardsRequest;import software.amazon.awssdk.services.kinesis.model.PutRecordRequest;import software.amazon.awssdk.services.kinesis.model.PutRecordResponse;import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest;@@ -105,6 +108,58 @@ class KinesisTest {}}+ @Test+ @DisplayName("Closed shards terminate with typed split and merge child metadata")+ void closedShardsReturnNullIteratorAndTypedChildren() {+ try (KinesisClient client = TestFixtures.kinesisClient()) {+ String name = TestFixtures.uniqueName("sdk-kinesis-terminal");+ client.createStream(CreateStreamRequest.builder().streamName(name).shardCount(1).build());+ try {+ Shard parent = client.listShards(ListShardsRequest.builder().streamName(name).build()).shards().get(0);+ for (String data : List.of("first", "last")) {+ client.putRecord(PutRecordRequest.builder().streamName(name).partitionKey("pk")+ .data(SdkBytes.fromUtf8String(data)).build());+ }+ split(client, name, parent);+ String cursor = client.getShardIterator(GetShardIteratorRequest.builder().streamName(name)+ .shardId(parent.shardId()).shardIteratorType(ShardIteratorType.TRIM_HORIZON).build()).shardIterator();+ GetRecordsResponse first = client.getRecords(GetRecordsRequest.builder().shardIterator(cursor).limit(1).build());+ assertThat(first.records()).hasSize(1);+ assertThat(first.nextShardIterator()).isNotNull();+ assertThat(first.hasChildShards()).isFalse();+ GetRecordsResponse last = client.getRecords(GetRecordsRequest.builder()+ .shardIterator(first.nextShardIterator()).limit(1).build());+ assertThat(last.records()).singleElement().satisfies(record ->+ assertThat(record.data().asUtf8String()).isEqualTo("last"));+ assertThat(last.nextShardIterator()).isNull();+ List<Shard> children = client.listShards(ListShardsRequest.builder().streamName(name).build())+ .shards().stream().filter(shard -> parent.shardId().equals(shard.parentShardId()))+ .collect(Collectors.toList());+ assertThat(last.childShards()).hasSize(2);+ for (int i = 0; i < children.size(); i++) {+ ChildShard child = last.childShards().get(i);+ assertThat(child.shardId()).isEqualTo(children.get(i).shardId());+ assertThat(child.parentShards()).containsExactly(parent.shardId());+ assertThat(child.hashKeyRange()).isEqualTo(children.get(i).hashKeyRange());+ }+ client.mergeShards(MergeShardsRequest.builder().streamName(name)+ .shardToMerge(children.get(0).shardId()).adjacentShardToMerge(children.get(1).shardId()).build());+ for (Shard child : children) {+ String mergedCursor = client.getShardIterator(GetShardIteratorRequest.builder().streamName(name)+ .shardId(child.shardId()).shardIteratorType(ShardIteratorType.TRIM_HORIZON).build()).shardIterator();+ GetRecordsResponse merged = client.getRecords(GetRecordsRequest.builder().shardIterator(mergedCursor).build());+ assertThat(merged.nextShardIterator()).isNull();+ assertThat(merged.childShards()).singleElement().satisfies(descendant -> {+ assertThat(descendant.parentShards()).containsExactly(children.get(0).shardId(), children.get(1).shardId());+ assertThat(descendant.hashKeyRange()).isEqualTo(parent.hashKeyRange());+ });+ }+ } finally {+ client.deleteStream(DeleteStreamRequest.builder().streamName(name).build());+ }+ }+ }+private static void split(KinesisClient client, String streamName, Shard shard) {BigInteger start = new BigInteger(shard.hashKeyRange().startingHashKey());BigInteger end = new BigInteger(shard.hashKeyRange().endingHashKey());diff --git a/docs/services/kinesis.md b/docs/services/kinesis.mdindex 51d150256a..3a41f4337a 100644--- a/docs/services/kinesis.md+++ b/docs/services/kinesis.md@@ -80,6 +80,18 @@ aws kinesis describe-stream --stream-arn arn:aws:kinesis:us-east-1:000000000000:A `LATEST` iterator is positioned at the shard tip at the moment the iterator is created, matching AWS: records written after the iterator was obtained are returned, records written before are not. This supports the standard tailing pattern: obtain a `LATEST` iterator, trigger the action that produces the record, then poll `GetRecords` following `NextShardIterator`.+A closed shard returns a JSON null `NextShardIterator` on its final page, including+when that page contains records. Earlier pages keep their continuation iterator.+Only the terminal page includes `ChildShards`: each direct child's ID, all of its+parent IDs and its hash-key range. A caught-up open shard keeps a non-null iterator.+Consumers should process the final records before stopping at the null iterator.++The internal Pipes and Firehose readers retain a sequence cursor after the final+record to avoid replaying a terminal batch. Firehose commits that cursor only after+delivery succeeds; its unflushed buffer remains in memory. Pipes keeps its existing+advance-before-delivery policy and first-shard polling limitation; child scheduling+and full lineage orchestration are not implemented.+## Record RetentionA stream's retention period (default 24 hours, `IncreaseStreamRetentionPeriod`/`DecreaseStreamRetentionPeriod` up to 8760 hours) is enforced: a record older than the current retention period is not returned by `GetRecords` or the `GET /_aws/kinesis/records` inspection endpoint, and is actually removed from memory rather than merely hidden. Expiry is checked lazily on each stream's next `PutRecord`/`PutRecords` or read after a record ages out; there is no scheduled sweep. After `DecreaseStreamRetentionPeriod`, records outside the new window are inaccessible from the next put or read, in line with AWS making them inaccessible almost immediately. Shard iterators (including `NextShardIterator` continuation tokens and `LATEST`) resolve by sequence number, so they keep working correctly across pruning even if the exact record they were positioned at has since expired.@@ -125,7 +137,14 @@ action, so that limit is intentionally unenforced here.## Enhanced Fan-Out (EFO)-`SubscribeToShard` uses a snapshot-and-close model: the server returns one batch of records as a binary EventStream response and closes the connection. The SDK resubscribes automatically using the `ContinuationSequenceNumber` from the last delivered record. All five `StartingPosition` types are supported: `TRIM_HORIZON`, `LATEST`, `AT_SEQUENCE_NUMBER`, `AFTER_SEQUENCE_NUMBER`, `AT_TIMESTAMP`.+`SubscribeToShard` uses a snapshot-and-close model: the server returns one batch+of records as a binary EventStream response and closes the connection. For a+nonterminal batch, use `ContinuationSequenceNumber` from the last delivered record+when resubscribing. At a closed shard's end, the event includes the same direct+`ChildShards` metadata and omits `ContinuationSequenceNumber`, even when it includes+final records. This remains a snapshot response rather than a long-lived+subscription. All five `StartingPosition` types are supported: `TRIM_HORIZON`,+`LATEST`, `AT_SEQUENCE_NUMBER`, `AFTER_SEQUENCE_NUMBER`, `AT_TIMESTAMP`.```bashexport AWS_ENDPOINT_URL=http://localhost:4566diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex d9e319047e..979c815c81 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -987,12 +987,18 @@ public class FirehoseService implements ResourceProvider {@SuppressWarnings("unchecked")List<KinesisRecord> records = (List<KinesisRecord>) page.get("Records");String next = (String) page.get("NextShardIterator");- String advanced = next != null ? next : iterator;+ String advanced = next != null || records == null || records.isEmpty() ? next :+ kinesisService.getShardIteratorForAccount(sourceAccountId, sourceName, shard.getShardId(),+ "AFTER_SEQUENCE_NUMBER", records.getLast().getSequenceNumber(), region);+ if (advanced == null) {+ advanced = iterator;+ }+ String checkpoint = advanced;String shardId = shard.getShardId();if (records == null || records.isEmpty()) {// Nothing was buffered, so this advance owes S3 nothing and is safe to// park as pending on its own.- advanceSourceIterator(streamKey, shardId, advanced);+ advanceSourceIterator(streamKey, shardId, checkpoint);continue;}// Through putRecordBatch so source records buffer, and trigger a flush, on@@ -1002,7 +1008,7 @@ public class FirehoseService implements ResourceProvider {putRecordBatch(streamKey, deliveryStreamName, records.stream().map(record -> new Record(record.getData())).toList(),- () -> advanceSourceIterator(streamKey, shardId, advanced));+ () -> advanceSourceIterator(streamKey, shardId, checkpoint));}}diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.javaindex dce8899831..d1ff0ed470 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java@@ -417,7 +417,7 @@ public class KinesisJsonHandler {List<KinesisRecord> records = (List<KinesisRecord>) result.get("Records");long millisBehind = ((Number) result.get("MillisBehindLatest")).longValue();- String continuationSeqNo = records.isEmpty() ? null+ String continuationSeqNo = result.get("NextShardIterator") == null || records.isEmpty() ? null: records.get(records.size() - 1).getSequenceNumber();ObjectNode eventPayload = objectMapper.createObjectNode();@@ -435,7 +435,7 @@ public class KinesisJsonHandler {eventPayload.put("ContinuationSequenceNumber", continuationSeqNo);}eventPayload.put("MillisBehindLatest", millisBehind);- eventPayload.putArray("ChildShards");+ writeChildShards(eventPayload, result);try {// The Go SDK (and other SDKs) expect an initial-response message before@@ -735,10 +735,29 @@ public class KinesisJsonHandler {}}response.put("NextShardIterator", (String) result.get("NextShardIterator"));+ writeChildShards(response, result);response.put("MillisBehindLatest", ((Number) result.get("MillisBehindLatest")).longValue());return Response.ok(response).build();}+ @SuppressWarnings("unchecked")+ private void writeChildShards(ObjectNode response, Map<String, Object> result) {+ List<KinesisService.ChildShard> children = (List<KinesisService.ChildShard>) result.get("ChildShards");+ if (children == null) {+ return;+ }+ ArrayNode childNodes = response.putArray("ChildShards");+ for (KinesisService.ChildShard child : children) {+ ObjectNode node = childNodes.addObject();+ node.put("ShardId", child.shardId());+ ArrayNode parents = node.putArray("ParentShards");+ child.parentShards().forEach(parents::add);+ node.putObject("HashKeyRange")+ .put("StartingHashKey", child.hashKeyRange().startingHashKey())+ .put("EndingHashKey", child.hashKeyRange().endingHashKey());+ }+ }+private Response handleIncreaseStreamRetentionPeriod(JsonNode request, String region) {String streamName = resolveStreamName(request);int retentionPeriodHours = request.path("RetentionPeriodHours").asInt();diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.javaindex 3e0e51df60..0c2137271f 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java@@ -893,14 +893,8 @@ public class KinesisService implements ResourceProvider {}}- KinesisStream stream = resolveStream(streamName, region);- KinesisShard shard = stream.getShards().stream()- .filter(s -> s.getShardId().equals(shardId))- .findFirst()- .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));- pruneExpiredRecords(stream, shard);-- KinesisShard.RecordsSnapshot snapshot = shard.snapshotRecords();+ ShardReadSnapshot read = snapshotShard(null, streamName, shardId, region);+ KinesisShard.RecordsSnapshot snapshot = read.records();List<KinesisRecord> allRecords = snapshot.records();int startIndex = 0;@@ -946,6 +940,45 @@ public class KinesisService implements ResourceProvider {}}+ return recordsPage(streamName, shardId, read, startIndex, limit);+ }++ public record ChildShard(String shardId, List<String> parentShards, KinesisShard.HashKeyRange hashKeyRange) {}++ private record ShardReadSnapshot(KinesisShard.RecordsSnapshot records, boolean closed,+ List<ChildShard> children) {}++ private ShardReadSnapshot snapshotShard(String accountId, String streamName, String shardId, String region) {+ // Share the append/reshard monitor so a terminal snapshot includes the final append and topology.+ synchronized (lockFor(regionKey(region, streamName))) {+ KinesisStream stream = resolveStreamForAccount(accountId, streamName, region);+ KinesisShard shard = stream.getShards().stream()+ .filter(candidate -> candidate.getShardId().equals(shardId))+ .findFirst()+ .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));+ pruneExpiredRecords(stream, shard);+ List<ChildShard> children = new ArrayList<>();+ if (shard.isClosed()) {+ for (KinesisShard child : stream.getShards()) {+ if (shardId.equals(child.getParentShardId()) || shardId.equals(child.getAdjacentParentShardId())) {+ List<String> parents = new ArrayList<>();+ if (child.getParentShardId() != null) {+ parents.add(child.getParentShardId());+ }+ if (child.getAdjacentParentShardId() != null) {+ parents.add(child.getAdjacentParentShardId());+ }+ children.add(new ChildShard(child.getShardId(), List.copyOf(parents), child.getHashKeyRange()));+ }+ }+ }+ return new ShardReadSnapshot(shard.snapshotRecords(), shard.isClosed(), List.copyOf(children));+ }+ }++ private Map<String, Object> recordsPage(String streamName, String shardId, ShardReadSnapshot read,+ int startIndex, Integer limit) {+ List<KinesisRecord> allRecords = read.records().records();int max = limit != null ? Math.min(limit, 1000) : 1000;List<KinesisRecord> result = new ArrayList<>();int nextIndex = startIndex;@@ -953,13 +986,15 @@ public class KinesisService implements ResourceProvider {result.add(allRecords.get(i));nextIndex = i + 1;}-- String nextIterator = buildContinuationIterator(streamName, shardId, allRecords, nextIndex);-+ boolean terminal = read.closed() && nextIndex >= allRecords.size();Map<String, Object> response = new HashMap<>();response.put("Records", result);- response.put("NextShardIterator", nextIterator);+ response.put("NextShardIterator", terminal ? null+ : buildContinuationIterator(streamName, shardId, allRecords, nextIndex));response.put("MillisBehindLatest", computeMillisBehindLatest(allRecords, nextIndex));+ if (terminal) {+ response.put("ChildShards", read.children());+ }return response;}@@ -1149,14 +1184,8 @@ public class KinesisService implements ResourceProvider {}}- KinesisStream stream = resolveStreamForAccount(accountId, streamName, region);- KinesisShard shard = stream.getShards().stream()- .filter(s -> s.getShardId().equals(shardId))- .findFirst()- .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));- pruneExpiredRecords(stream, shard);-- KinesisShard.RecordsSnapshot snapshot = shard.snapshotRecords();+ ShardReadSnapshot read = snapshotShard(accountId, streamName, shardId, region);+ KinesisShard.RecordsSnapshot snapshot = read.records();List<KinesisRecord> allRecords = snapshot.records();int startIndex = 0;// LATEST resolves the same way as AFTER_SEQUENCE_NUMBER (see the longer explanation in@@ -1186,20 +1215,7 @@ public class KinesisService implements ResourceProvider {}}- int max = limit != null ? Math.min(limit, 1000) : 1000;- List<KinesisRecord> result = new ArrayList<>();- int nextIndex = startIndex;- for (int i = startIndex; i < allRecords.size() && result.size() < max; i++) {- result.add(allRecords.get(i));- nextIndex = i + 1;- }-- String nextIterator = buildContinuationIterator(streamName, shardId, allRecords, nextIndex);- Map<String, Object> response = new HashMap<>();- response.put("Records", result);- response.put("NextShardIterator", nextIterator);- response.put("MillisBehindLatest", computeMillisBehindLatest(allRecords, nextIndex));- return response;+ return recordsPage(streamName, shardId, read, startIndex, limit);}private KinesisShard selectShard(KinesisStream stream, String partitionKey, String explicitHashKey) {diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.javaindex 95e6e9e8ac..27a9e51088 100644--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java@@ -12,6 +12,7 @@ import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlIdentity;import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader;import io.github.hectorvent.floci.services.kinesis.KinesisService;+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;import io.github.hectorvent.floci.services.pipes.model.DesiredState;import io.github.hectorvent.floci.services.pipes.model.Pipe;import io.github.hectorvent.floci.services.sqs.SqsService;@@ -245,7 +246,7 @@ public class PipesPoller implements Resettable {}}- private void pollKinesis(Pipe pipe, String region) {+ void pollKinesis(Pipe pipe, String region) {String pipeKey = pipeKey(pipe);String streamName = extractResourceName(pipe.getSource());String pipeAccountId = pipe.getAccountId();@@ -262,10 +263,17 @@ public class PipesPoller implements Resettable {? kinesisService.getRecordsForAccount(pipeAccountId, iterator, batchSize, region): kinesisService.getRecords(iterator, batchSize, region);String nextIterator = (String) result.get("NextShardIterator");- if (nextIterator != null) {- kinesisIterators.put(pipeKey, nextIterator);- }List<?> records = (List<?>) result.get("Records");+ if (nextIterator == null && records != null && !records.isEmpty()) {+ KinesisRecord last = (KinesisRecord) records.getLast();+ nextIterator = pipeAccountId != null+ ? kinesisService.getShardIteratorForAccount(pipeAccountId, streamName,+ "shardId-000000000000", "AFTER_SEQUENCE_NUMBER", last.getSequenceNumber(), region)+ : kinesisService.getShardIterator(streamName, "shardId-000000000000",+ "AFTER_SEQUENCE_NUMBER", last.getSequenceNumber(), null, region);+ }+ // Remember even the initial exhausted cursor, before filtering or delivery.+ kinesisIterators.put(pipeKey, nextIterator != null ? nextIterator : iterator);if (records == null || records.isEmpty()) {return;}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.javaindex 2ad60fbe44..80b0e19b74 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java@@ -1,5 +1,7 @@package io.github.hectorvent.floci.services.firehose;+import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;@@ -17,6 +19,7 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.EnumSource;+import org.junit.jupiter.params.provider.ValueSource;import org.mockito.ArgumentCaptor;import org.mockito.Mockito;@@ -778,6 +781,83 @@ class FirehoseServiceTest {java.util.List.of(), "KinesisStreamAsSource", kinesisSource(sourceStream));}+ private void closeSourceParent() {+ kinesisService.splitShard("src-stream", "shardId-000000000000", "100", "us-east-1");+ }++ @ParameterizedTest+ @ValueSource(ints = {1, 501})+ void terminalSourceBatchDoesNotReplayBeforeOrAfterFlushAndRecreation(int count) {+ createSourcedDeliveryStream("sourced-stream", "src-stream");+ for (int i = 0; i < count; i++) {+ kinesisService.putRecord("src-stream", "terminal".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");+ }+ closeSourceParent();+ firehoseService.pollKinesisSources();+ firehoseService.pollKinesisSources();+ firehoseService.flush("sourced-stream");+ firehoseService = newService(0);+ firehoseService.pollKinesisSources();+ firehoseService.flush("sourced-stream");+ assertEquals("terminal\n".repeat(count), delivered("sink").text());+ }++ @Test+ void terminalSourceBatchIsRereadWhenServiceIsRecreatedBeforeFlush() {+ createSourcedDeliveryStream("sourced-stream", "src-stream");+ kinesisService.putRecord("src-stream", "terminal".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");+ closeSourceParent();+ firehoseService.pollKinesisSources();+ verifyNothingDelivered();+ firehoseService = newService(0);+ firehoseService.pollKinesisSources();+ firehoseService.flush("sourced-stream");+ firehoseService.pollKinesisSources();+ firehoseService.flush("sourced-stream");+ assertEquals("terminal\n", delivered("sink").text());+ }++ @Test+ void failedTerminalFlushKeepsPendingBatchWithoutCommittingOrDuplicating() {+ createSourcedDeliveryStream("sourced-stream", "src-stream");+ kinesisService.putRecord("src-stream", "terminal".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");+ closeSourceParent();+ when(s3Service.putObject(anyString(), anyString(), any(byte[].class), anyString(),+ anyMap(), any(PutObjectOptions.class))).thenThrow(new RuntimeException("s3 unavailable"));+ firehoseService.pollKinesisSources();+ firehoseService.flush("sourced-stream");+ assertTrue(backends.get("firehose/source-iterators.json").scan(key -> true).isEmpty());+ Mockito.reset(s3Service);+ firehoseService.pollKinesisSources();+ firehoseService.flush("sourced-stream");+ firehoseService.pollKinesisSources();+ firehoseService.flush("sourced-stream");+ assertEquals("terminal\n", delivered("sink").text());+ }++ @Test+ @SuppressWarnings("unchecked")+ void committedTerminalCursorSurvivesSerializationRoundTrip() throws Exception {+ createSourcedDeliveryStream("sourced-stream", "src-stream");+ kinesisService.putRecord("src-stream", "terminal".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");+ closeSourceParent();+ firehoseService.pollKinesisSources();+ firehoseService.flush("sourced-stream");+ AccountAwareStorageBackend<Map<String, String>> cursors =+ (AccountAwareStorageBackend<Map<String, String>>) backends.get("firehose/source-iterators.json");+ ObjectMapper mapper = new ObjectMapper();+ for (String key : cursors.keys()) {+ Map<String, String> restored = mapper.readValue(mapper.writeValueAsBytes(cursors.get(key).orElseThrow()),+ new TypeReference<Map<String, String>>() {});+ cursors.put(key, restored);+ }+ assertFalse(cursors.keys().isEmpty());+ firehoseService = newService(0);+ firehoseService.pollKinesisSources();+ firehoseService.flush("sourced-stream");+ assertEquals("terminal\n", delivered("sink").text());+ }+@Testvoid kinesisSourceRecordsAreDelivered() {createSourcedDeliveryStream("sourced-stream", "src-stream");diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.javaindex 082a92de10..db0c1dc6b0 100644--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java@@ -17,6 +17,7 @@ import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.ValueSource;import java.math.BigDecimal;+import java.nio.ByteBuffer;import java.time.Clock;import java.time.Instant;import java.time.ZoneOffset;@@ -59,6 +60,53 @@ class KinesisJsonHandlerTest {handler = new KinesisJsonHandler(service, MAPPER, 300_000);}+ @Test+ void terminalGetRecordsIsJsonNullWithExactChildMetadata() throws Exception {+ createStream("test-stream");+ service.putRecord("test-stream", new byte[]{1}, "pk", REGION);+ service.splitShard("test-stream", FIRST_SHARD_ID, "100", REGION);+ String cursor = service.getShardIterator("test-stream", FIRST_SHARD_ID, "TRIM_HORIZON", null, null, REGION);+ ObjectNode request = MAPPER.createObjectNode().put("ShardIterator", cursor);+ JsonNode page = MAPPER.readTree(responseEntity(handler.handle("GetRecords", request, REGION)).toString());+ assertTrue(page.has("NextShardIterator"));+ assertTrue(page.get("NextShardIterator").isNull());+ assertEquals(1, page.path("Records").size());+ assertEquals(MAPPER.readTree("""+ [{"ShardId":"shardId-000000000001","ParentShards":["shardId-000000000000"],+ "HashKeyRange":{"StartingHashKey":"0","EndingHashKey":"99"}},+ {"ShardId":"shardId-000000000002","ParentShards":["shardId-000000000000"],+ "HashKeyRange":{"StartingHashKey":"100","EndingHashKey":"340282366920938463463374607431768211455"}}]+ """), page.path("ChildShards"));+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void subscriptionTerminalMetadataDoesNotContinueEvenWithRecords(boolean nonempty) throws Exception {+ createStream("test-stream");+ if (nonempty) {+ service.putRecord("test-stream", new byte[]{1}, "pk", REGION);+ }+ String arn = service.registerStreamConsumer(STREAM_ARN, "terminal-consumer", REGION).getConsumerArn();+ service.splitShard("test-stream", FIRST_SHARD_ID, "100", REGION);+ ObjectNode request = MAPPER.createObjectNode().put("ConsumerARN", arn).put("ShardId", FIRST_SHARD_ID);+ request.putObject("StartingPosition").put("Type", "TRIM_HORIZON");+ byte[] wire = (byte[]) handler.handle("SubscribeToShard", request, REGION).getEntity();+ ByteBuffer buffer = ByteBuffer.wrap(wire);+ int initialLength = buffer.getInt();+ buffer.position(initialLength);+ int eventLength = buffer.getInt();+ int headerLength = buffer.getInt();+ buffer.position(initialLength + 12 + headerLength);+ byte[] payload = new byte[eventLength - headerLength - 16];+ buffer.get(payload);+ JsonNode event = MAPPER.readTree(payload);+ assertEquals(nonempty ? 1 : 0, event.path("Records").size());+ assertFalse(event.has("ContinuationSequenceNumber"));+ assertEquals(2, event.path("ChildShards").size());+ assertEquals(FIRST_SHARD_ID, event.path("ChildShards").get(0).path("ParentShards").get(0).asText());+ assertEquals("99", event.path("ChildShards").get(0).path("HashKeyRange").path("EndingHashKey").asText());+ }+// Retention is measured against the service clock, so a test that plants a fixed arrival// timestamp pins the clock just after it instead of relying on the wall clock.private void useClockFixedAt(Instant now) {diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisTerminalReadServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisTerminalReadServiceTest.javanew file mode 100644index 0000000000..b811bac3d0--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisTerminalReadServiceTest.java@@ -0,0 +1,202 @@+package io.github.hectorvent.floci.services.kinesis;++import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;+import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.nio.charset.StandardCharsets;+import java.time.Instant;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.spy;++class KinesisTerminalReadServiceTest {+ private static final String REGION = "us-east-1";+ private static final String ACCOUNT = "000000000000";+ private static final String PARENT = "shardId-000000000000";+ private final AccountAwareStorageBackend<KinesisStream> streams = AccountAwareStorageBackend.inMemory(ACCOUNT);+ private final KinesisService service = new KinesisService(streams, AccountAwareStorageBackend.inMemory(ACCOUNT),+ new RegionResolver(REGION, ACCOUNT));++ private String iterator(String type, String sequence, Long timestamp) {+ return service.getShardIterator("s", PARENT, type, sequence, timestamp, REGION);+ }++ private Map<String, Object> read(boolean account, String iterator, int limit) {+ return account ? service.getRecordsForAccount(ACCOUNT, iterator, limit, REGION)+ : service.getRecords(iterator, limit, REGION);+ }++ private String put(String value) {+ return service.putRecord("s", value.getBytes(StandardCharsets.UTF_8), "pk", REGION);+ }++ private void split() {+ service.splitShard("s", PARENT, "170141183460469231731687303715884105728", REGION);+ }++ @SuppressWarnings("unchecked")+ private List<KinesisService.ChildShard> children(Map<String, Object> page) {+ return (List<KinesisService.ChildShard>) page.get("ChildShards");+ }++ @Test+ void atSequenceNumberOnClosedShardIncludesTheRequestedRecord() {+ service.createStream("s", 1, REGION);+ put("first");+ String last = put("last");+ split();+ Map<String, Object> page = service.getRecords(iterator("AT_SEQUENCE_NUMBER", last, null), 10, REGION);+ assertEquals(1, ((List<?>) page.get("Records")).size());+ assertNull(page.get("NextShardIterator"));+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void finalNonemptyPageTerminatesAndOnlyTerminalPagesHaveChildren(boolean account) {+ service.createStream("s", 1, REGION);+ put("first");+ put("last");+ split();+ Map<String, Object> first = read(account, iterator("TRIM_HORIZON", null, null), 1);+ assertEquals(1, ((List<?>) first.get("Records")).size());+ assertNotNull(first.get("NextShardIterator"));+ assertFalse(first.containsKey("ChildShards"));+ Map<String, Object> last = read(account, (String) first.get("NextShardIterator"), 1);+ assertEquals(1, ((List<?>) last.get("Records")).size());+ assertNull(last.get("NextShardIterator"));+ assertEquals(0L, last.get("MillisBehindLatest"));+ assertEquals(List.of("shardId-000000000001", "shardId-000000000002"),+ children(last).stream().map(KinesisService.ChildShard::shardId).toList());+ assertEquals(List.of(PARENT), children(last).getFirst().parentShards());+ assertEquals(new KinesisShard.HashKeyRange("0", "170141183460469231731687303715884105727"),+ children(last).getFirst().hashKeyRange());+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void caughtUpOpenShardRemainsReadableAndClosedPositionsTerminate(boolean account) {+ service.createStream("s", 1, REGION);+ String sequence = put("last");+ String latest = iterator("LATEST", null, null);+ Map<String, Object> open = read(account, latest, 10);+ assertTrue(((List<?>) open.get("Records")).isEmpty());+ assertNotNull(open.get("NextShardIterator"));+ assertFalse(open.containsKey("ChildShards"));+ split();+ for (String cursor : List.of(latest, iterator("AFTER_SEQUENCE_NUMBER", sequence, null),+ iterator("AT_TIMESTAMP", null, Instant.now().plusSeconds(60).toEpochMilli()))) {+ Map<String, Object> closed = read(account, cursor, 10);+ assertTrue(((List<?>) closed.get("Records")).isEmpty());+ assertNull(closed.get("NextShardIterator"));+ assertEquals(2, children(closed).size());+ }+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void emptyOrRetentionPrunedParentsTerminateWithDirectChildren(boolean account) {+ KinesisStream stream = service.createStream("s", 1, REGION);+ String cursor = iterator("TRIM_HORIZON", null, null);+ split();+ assertNull(read(account, cursor, 10).get("NextShardIterator"));+ stream.getShards().getFirst().addRecord(new KinesisRecord(new byte[]{1}, "pk", "1",+ Instant.now().minusSeconds(172800)));+ service.splitShard("s", "shardId-000000000001", "85070591730234615865843651857942052864", REGION);+ Map<String, Object> page = read(account, cursor, 10);+ assertTrue(((List<?>) page.get("Records")).isEmpty());+ assertNull(page.get("NextShardIterator"));+ assertEquals(List.of("shardId-000000000001", "shardId-000000000002"),+ children(page).stream().map(KinesisService.ChildShard::shardId).toList());+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void bothMergeParentsReportBothParentIds(boolean account) {+ service.createStream("s", 2, REGION);+ String adjacent = "shardId-000000000001";+ service.mergeShards("s", PARENT, adjacent, REGION);+ for (String parent : List.of(PARENT, adjacent)) {+ String cursor = service.getShardIterator("s", parent, "TRIM_HORIZON", null, null, REGION);+ Map<String, Object> page = read(account, cursor, 10);+ assertNull(page.get("NextShardIterator"));+ assertEquals(List.of(new KinesisService.ChildShard("shardId-000000000002", List.of(PARENT, adjacent),+ new KinesisShard.HashKeyRange("0", "340282366920938463463374607431768211455"))), children(page));+ }+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void snapshotSerializesWithExplicitAccountAppendsAndResharding(boolean account) throws Exception {+ KinesisStream stream = service.createStream("s", 1, REGION);+ KinesisShard parent = spy(stream.getShards().getFirst());+ stream.getShards().set(0, parent);+ put("first");+ CountDownLatch captured = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ doAnswer(invocation -> {+ KinesisShard.RecordsSnapshot snapshot = (KinesisShard.RecordsSnapshot) invocation.callRealMethod();+ captured.countDown();+ assertTrue(release.await(5, TimeUnit.SECONDS));+ return snapshot;+ }).when(parent).snapshotRecords();+ String cursor = iterator("TRIM_HORIZON", null, null);+ try (ExecutorService pool = Executors.newFixedThreadPool(2)) {+ Future<Map<String, Object>> reading = pool.submit(() -> read(account, cursor, 10));+ try {+ assertTrue(captured.await(5, TimeUnit.SECONDS));+ CountDownLatch writing = new CountDownLatch(1);+ Future<?> mutation = pool.submit(() -> {+ writing.countDown();+ service.putRecordForAccount(ACCOUNT, "s", new byte[]{2}, "pk", REGION);+ split();+ });+ assertTrue(writing.await(5, TimeUnit.SECONDS));+ assertThrows(TimeoutException.class, () -> mutation.get(150, TimeUnit.MILLISECONDS));+ release.countDown();+ Map<String, Object> open = reading.get(5, TimeUnit.SECONDS);+ assertNotNull(open.get("NextShardIterator"));+ assertEquals(1, ((List<?>) open.get("Records")).size());+ mutation.get(5, TimeUnit.SECONDS);+ Map<String, Object> terminal = read(account, (String) open.get("NextShardIterator"), 10);+ assertEquals(1, ((List<?>) terminal.get("Records")).size());+ assertNull(terminal.get("NextShardIterator"));+ assertEquals(2, children(terminal).size());+ } finally {+ release.countDown();+ }+ }+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void accountAndRegionResolveTheirOwnTerminalState(boolean closedOther) {+ service.createStream("s", 1, REGION);+ split();+ KinesisStream other = new KinesisStream("s", "arn:aws:kinesis:us-east-1:111111111111:stream/s");+ KinesisShard shard = new KinesisShard(PARENT, "0", "100", "0");+ shard.setClosed(closedOther);+ other.getShards().add(shard);+ streams.putForAccount("111111111111", REGION + "::s", other);+ String cursor = iterator("TRIM_HORIZON", null, null);+ Map<String, Object> page = service.getRecordsForAccount("111111111111", cursor, 10, REGION);+ assertEquals(closedOther, page.get("NextShardIterator") == null);+ assertNull(service.getRecords(cursor, 10, REGION).get("NextShardIterator"));+ service.createStream("s", 1, "eu-west-1");+ assertNotNull(service.getRecords(cursor, 10, "eu-west-1").get("NextShardIterator"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePollerTest.javaindex 2eeda9f07e..a4c7ddc883 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePollerTest.java@@ -30,6 +30,7 @@ import java.time.Duration;import java.time.Instant;import java.util.ArrayList;import java.util.Base64;+import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Optional;@@ -125,6 +126,25 @@ class KinesisEventSourcePollerTest {clock::get);}+ @Test+ void finalNonemptyPageWithNullIteratorStillInvokesAndCheckpoints() throws Exception {+ stubFunction();+ stubStreamWith(List.of(record("123", "pk", "terminal")));+ Map<String, Object> terminal = new HashMap<>();+ terminal.put("Records", List.of(record("123", "pk", "terminal")));+ terminal.put("NextShardIterator", null);+ terminal.put("ChildShards", List.of());+ when(kinesisService.getRecordsForAccount(eq(ACCOUNT), eq("iter-0"), anyInt(), eq(REGION)))+ .thenReturn(terminal);+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(new InvokeResult());+ EventSourceMapping mapping = esm();+ pollOnce(mapping);+ verify(executorService).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));+ assertEquals("123", mapping.getShardSequenceNumbers().get(SHARD));+ verify(esmStore).saveForAccount(eq(ACCOUNT), eq(mapping));+ }+private EventSourceMapping esm(String... patterns) {EventSourceMapping esm = new EventSourceMapping();esm.setUuid("k-esm");diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKinesisTerminalTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKinesisTerminalTest.javanew file mode 100644index 0000000000..fc3d452d39--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKinesisTerminalTest.java@@ -0,0 +1,98 @@+package io.github.hectorvent.floci.services.pipes;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader;+import io.github.hectorvent.floci.services.kinesis.KinesisService;+import io.github.hectorvent.floci.services.pipes.model.Pipe;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.vertx.core.Vertx;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;++import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.concurrent.atomic.AtomicInteger;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.mockito.ArgumentMatchers.*;+import static org.mockito.Mockito.*;++class PipesKinesisTerminalTest {+ @ParameterizedTest+ @CsvSource({"true,0,success", "false,0,success", "true,1,success", "false,1,success", "true,2,success", "false,2,success",+ "true,1,filtered", "true,2,filtered", "true,1,dlq", "true,2,dlq", "true,1,drop", "true,2,drop"})+ void terminalPagesAdvanceBeforeFilteringOrDelivery(boolean account, int recordCount, String outcome) throws Exception {+ ObjectMapper mapper = new ObjectMapper().findAndRegisterModules();+ StorageFactory factory = mock(StorageFactory.class);+ Map<String, AccountAwareStorageBackend<?>> stores = new HashMap<>();+ when(factory.create(anyString(), anyString(), any())).thenAnswer(inv -> stores.computeIfAbsent(+ inv.getArgument(1), key -> AccountAwareStorageBackend.inMemory("111111111111")));+ KinesisService kinesis = spy(new KinesisService(factory, new RegionResolver("eu-west-1", "111111111111")));+ kinesis.createStream("s", 1, "eu-west-1");+ for (int i = 0; i < recordCount; i++) {+ kinesis.putRecord("s", new byte[]{1}, "pk", "eu-west-1");+ }+ kinesis.splitShard("s", "shardId-000000000000", "100", "eu-west-1");+ AtomicInteger terminalRecordPages = new AtomicInteger();+ doAnswer(invocation -> {+ @SuppressWarnings("unchecked")+ Map<String, Object> page = (Map<String, Object>) invocation.callRealMethod();+ if (page.get("NextShardIterator") == null && !((List<?>) page.get("Records")).isEmpty()) {+ terminalRecordPages.incrementAndGet();+ }+ return page;+ }).when(kinesis).getRecordsForAccount(anyString(), anyString(), any(), anyString());+ doAnswer(invocation -> {+ @SuppressWarnings("unchecked")+ Map<String, Object> page = (Map<String, Object>) invocation.callRealMethod();+ if (page.get("NextShardIterator") == null && !((List<?>) page.get("Records")).isEmpty()) {+ terminalRecordPages.incrementAndGet();+ }+ return page;+ }).when(kinesis).getRecords(anyString(), any(), anyString());+ Pipe pipe = new Pipe();+ pipe.setName("terminal");+ pipe.setArn("arn:aws:pipes:eu-west-1:111111111111:pipe/terminal");+ pipe.setSource("arn:aws:kinesis:eu-west-1:111111111111:stream/s");+ pipe.setTarget("arn:aws:lambda:eu-west-1:111111111111:function:target");+ pipe.setAccountId(account ? "111111111111" : null);+ pipe.setSourceParameters(mapper.readTree("""+ {"KinesisStreamParameters":{"BatchSize":1,+ "DeadLetterConfig":{"Arn":"arn:aws:sqs:eu-west-1:111111111111:dlq"}}}+ """));+ if ("filtered".equals(outcome)) {+ pipe.getSourceParameters().withObject("/FilterCriteria").putArray("Filters").addObject()+ .put("Pattern", "{\"eventSource\":[\"aws:sqs\"]}");+ }+ SqsService sqs = mock(SqsService.class);+ PipesTargetInvoker target = mock(PipesTargetInvoker.class);+ if ("dlq".equals(outcome) || "drop".equals(outcome)) {+ doThrow(new RuntimeException("target down")).when(target).invoke(eq(pipe), anyString(), eq("eu-west-1"));+ }+ if ("drop".equals(outcome)) {+ when(sqs.sendMessage(anyString(), anyString(), anyInt(), anyString()))+ .thenThrow(new RuntimeException("dlq down"));+ }+ EmulatorConfig config = mock(EmulatorConfig.class);+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ PipesPoller poller = new PipesPoller(mock(Vertx.class), sqs, kinesis, mock(DynamoDbStreamReader.class),+ mock(PipesKafkaConsumerManager.class), target, new PipesFilterMatcher(mapper), mapper, config);+ try {+ for (int i = 0; i < recordCount + 3; i++) {+ poller.pollKinesis(pipe, "eu-west-1");+ }+ verify(target, times("filtered".equals(outcome) ? 0 : recordCount))+ .invoke(eq(pipe), anyString(), eq("eu-west-1"));+ verify(sqs, times("dlq".equals(outcome) || "drop".equals(outcome) ? recordCount : 0))+ .sendMessage(anyString(), anyString(), eq(0), eq("eu-west-1"));+ assertEquals(recordCount == 0 ? 0 : 1, terminalRecordPages.get(), "the terminal record page must be consumed once");+ } finally {+ poller.shutdown();+ }+ }+}