Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(kinesis): signal closed shard exhaustion and return child shards (#6716)

TiM77f247764140
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.java
index 821c3c3ebf..0f4a4ab7d3 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.java
@@ -13,14 +13,17 @@ import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
import software.amazon.awssdk.http.SdkHttpRequest;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.kinesis.KinesisClient;
+import software.amazon.awssdk.services.kinesis.model.ChildShard;
import software.amazon.awssdk.services.kinesis.model.CreateStreamRequest;
import software.amazon.awssdk.services.kinesis.model.DeleteStreamRequest;
import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryRequest;
import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryResponse;
import software.amazon.awssdk.services.kinesis.model.GetRecordsRequest;
+import software.amazon.awssdk.services.kinesis.model.GetRecordsResponse;
import software.amazon.awssdk.services.kinesis.model.GetShardIteratorRequest;
import software.amazon.awssdk.services.kinesis.model.ListShardsRequest;
import software.amazon.awssdk.services.kinesis.model.ListShardsResponse;
+import software.amazon.awssdk.services.kinesis.model.MergeShardsRequest;
import software.amazon.awssdk.services.kinesis.model.PutRecordRequest;
import software.amazon.awssdk.services.kinesis.model.PutRecordResponse;
import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest;
@@ -105,6 +108,58 @@ class KinesisTest {
}
}
+ @Test
+ @DisplayName("Closed shards terminate with typed split and merge child metadata")
+ void closedShardsReturnNullIteratorAndTypedChildren() {
+ try (KinesisClient client = TestFixtures.kinesisClient()) {
+ String name = TestFixtures.uniqueName("sdk-kinesis-terminal");
+ client.createStream(CreateStreamRequest.builder().streamName(name).shardCount(1).build());
+ try {
+ Shard parent = client.listShards(ListShardsRequest.builder().streamName(name).build()).shards().get(0);
+ for (String data : List.of("first", "last")) {
+ client.putRecord(PutRecordRequest.builder().streamName(name).partitionKey("pk")
+ .data(SdkBytes.fromUtf8String(data)).build());
+ }
+ split(client, name, parent);
+ String cursor = client.getShardIterator(GetShardIteratorRequest.builder().streamName(name)
+ .shardId(parent.shardId()).shardIteratorType(ShardIteratorType.TRIM_HORIZON).build()).shardIterator();
+ GetRecordsResponse first = client.getRecords(GetRecordsRequest.builder().shardIterator(cursor).limit(1).build());
+ assertThat(first.records()).hasSize(1);
+ assertThat(first.nextShardIterator()).isNotNull();
+ assertThat(first.hasChildShards()).isFalse();
+ GetRecordsResponse last = client.getRecords(GetRecordsRequest.builder()
+ .shardIterator(first.nextShardIterator()).limit(1).build());
+ assertThat(last.records()).singleElement().satisfies(record ->
+ assertThat(record.data().asUtf8String()).isEqualTo("last"));
+ assertThat(last.nextShardIterator()).isNull();
+ List<Shard> children = client.listShards(ListShardsRequest.builder().streamName(name).build())
+ .shards().stream().filter(shard -> parent.shardId().equals(shard.parentShardId()))
+ .collect(Collectors.toList());
+ assertThat(last.childShards()).hasSize(2);
+ for (int i = 0; i < children.size(); i++) {
+ ChildShard child = last.childShards().get(i);
+ assertThat(child.shardId()).isEqualTo(children.get(i).shardId());
+ assertThat(child.parentShards()).containsExactly(parent.shardId());
+ assertThat(child.hashKeyRange()).isEqualTo(children.get(i).hashKeyRange());
+ }
+ client.mergeShards(MergeShardsRequest.builder().streamName(name)
+ .shardToMerge(children.get(0).shardId()).adjacentShardToMerge(children.get(1).shardId()).build());
+ for (Shard child : children) {
+ String mergedCursor = client.getShardIterator(GetShardIteratorRequest.builder().streamName(name)
+ .shardId(child.shardId()).shardIteratorType(ShardIteratorType.TRIM_HORIZON).build()).shardIterator();
+ GetRecordsResponse merged = client.getRecords(GetRecordsRequest.builder().shardIterator(mergedCursor).build());
+ assertThat(merged.nextShardIterator()).isNull();
+ assertThat(merged.childShards()).singleElement().satisfies(descendant -> {
+ assertThat(descendant.parentShards()).containsExactly(children.get(0).shardId(), children.get(1).shardId());
+ assertThat(descendant.hashKeyRange()).isEqualTo(parent.hashKeyRange());
+ });
+ }
+ } finally {
+ client.deleteStream(DeleteStreamRequest.builder().streamName(name).build());
+ }
+ }
+ }
+
private static void split(KinesisClient client, String streamName, Shard shard) {
BigInteger start = new BigInteger(shard.hashKeyRange().startingHashKey());
BigInteger end = new BigInteger(shard.hashKeyRange().endingHashKey());
diff --git a/docs/services/kinesis.md b/docs/services/kinesis.md
index 51d150256a..3a41f4337a 100644
--- a/docs/services/kinesis.md
+++ b/docs/services/kinesis.md
@@ -80,6 +80,18 @@ aws kinesis describe-stream --stream-arn arn:aws:kinesis:us-east-1:000000000000:
A `LATEST` iterator is positioned at the shard tip at the moment the iterator is created, matching AWS: records written after the iterator was obtained are returned, records written before are not. This supports the standard tailing pattern: obtain a `LATEST` iterator, trigger the action that produces the record, then poll `GetRecords` following `NextShardIterator`.
+A closed shard returns a JSON null `NextShardIterator` on its final page, including
+when that page contains records. Earlier pages keep their continuation iterator.
+Only the terminal page includes `ChildShards`: each direct child's ID, all of its
+parent IDs and its hash-key range. A caught-up open shard keeps a non-null iterator.
+Consumers should process the final records before stopping at the null iterator.
+
+The internal Pipes and Firehose readers retain a sequence cursor after the final
+record to avoid replaying a terminal batch. Firehose commits that cursor only after
+delivery succeeds; its unflushed buffer remains in memory. Pipes keeps its existing
+advance-before-delivery policy and first-shard polling limitation; child scheduling
+and full lineage orchestration are not implemented.
+
## Record Retention
A stream's retention period (default 24 hours, `IncreaseStreamRetentionPeriod`/`DecreaseStreamRetentionPeriod` up to 8760 hours) is enforced: a record older than the current retention period is not returned by `GetRecords` or the `GET /_aws/kinesis/records` inspection endpoint, and is actually removed from memory rather than merely hidden. Expiry is checked lazily on each stream's next `PutRecord`/`PutRecords` or read after a record ages out; there is no scheduled sweep. After `DecreaseStreamRetentionPeriod`, records outside the new window are inaccessible from the next put or read, in line with AWS making them inaccessible almost immediately. Shard iterators (including `NextShardIterator` continuation tokens and `LATEST`) resolve by sequence number, so they keep working correctly across pruning even if the exact record they were positioned at has since expired.
@@ -125,7 +137,14 @@ action, so that limit is intentionally unenforced here.
## Enhanced Fan-Out (EFO)
-`SubscribeToShard` uses a snapshot-and-close model: the server returns one batch of records as a binary EventStream response and closes the connection. The SDK resubscribes automatically using the `ContinuationSequenceNumber` from the last delivered record. All five `StartingPosition` types are supported: `TRIM_HORIZON`, `LATEST`, `AT_SEQUENCE_NUMBER`, `AFTER_SEQUENCE_NUMBER`, `AT_TIMESTAMP`.
+`SubscribeToShard` uses a snapshot-and-close model: the server returns one batch
+of records as a binary EventStream response and closes the connection. For a
+nonterminal batch, use `ContinuationSequenceNumber` from the last delivered record
+when resubscribing. At a closed shard's end, the event includes the same direct
+`ChildShards` metadata and omits `ContinuationSequenceNumber`, even when it includes
+final records. This remains a snapshot response rather than a long-lived
+subscription. All five `StartingPosition` types are supported: `TRIM_HORIZON`,
+`LATEST`, `AT_SEQUENCE_NUMBER`, `AFTER_SEQUENCE_NUMBER`, `AT_TIMESTAMP`.
```bash
export AWS_ENDPOINT_URL=http://localhost:4566
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index d9e319047e..979c815c81 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -987,12 +987,18 @@ public class FirehoseService implements ResourceProvider {
@SuppressWarnings("unchecked")
List<KinesisRecord> records = (List<KinesisRecord>) page.get("Records");
String next = (String) page.get("NextShardIterator");
- String advanced = next != null ? next : iterator;
+ String advanced = next != null || records == null || records.isEmpty() ? next :
+ kinesisService.getShardIteratorForAccount(sourceAccountId, sourceName, shard.getShardId(),
+ "AFTER_SEQUENCE_NUMBER", records.getLast().getSequenceNumber(), region);
+ if (advanced == null) {
+ advanced = iterator;
+ }
+ String checkpoint = advanced;
String shardId = shard.getShardId();
if (records == null || records.isEmpty()) {
// Nothing was buffered, so this advance owes S3 nothing and is safe to
// park as pending on its own.
- advanceSourceIterator(streamKey, shardId, advanced);
+ advanceSourceIterator(streamKey, shardId, checkpoint);
continue;
}
// Through putRecordBatch so source records buffer, and trigger a flush, on
@@ -1002,7 +1008,7 @@ public class FirehoseService implements ResourceProvider {
putRecordBatch(streamKey, deliveryStreamName, records.stream()
.map(record -> new Record(record.getData()))
.toList(),
- () -> advanceSourceIterator(streamKey, shardId, advanced));
+ () -> advanceSourceIterator(streamKey, shardId, checkpoint));
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
index dce8899831..d1ff0ed470 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
@@ -417,7 +417,7 @@ public class KinesisJsonHandler {
List<KinesisRecord> records = (List<KinesisRecord>) result.get("Records");
long millisBehind = ((Number) result.get("MillisBehindLatest")).longValue();
- String continuationSeqNo = records.isEmpty() ? null
+ String continuationSeqNo = result.get("NextShardIterator") == null || records.isEmpty() ? null
: records.get(records.size() - 1).getSequenceNumber();
ObjectNode eventPayload = objectMapper.createObjectNode();
@@ -435,7 +435,7 @@ public class KinesisJsonHandler {
eventPayload.put("ContinuationSequenceNumber", continuationSeqNo);
}
eventPayload.put("MillisBehindLatest", millisBehind);
- eventPayload.putArray("ChildShards");
+ writeChildShards(eventPayload, result);
try {
// The Go SDK (and other SDKs) expect an initial-response message before
@@ -735,10 +735,29 @@ public class KinesisJsonHandler {
}
}
response.put("NextShardIterator", (String) result.get("NextShardIterator"));
+ writeChildShards(response, result);
response.put("MillisBehindLatest", ((Number) result.get("MillisBehindLatest")).longValue());
return Response.ok(response).build();
}
+ @SuppressWarnings("unchecked")
+ private void writeChildShards(ObjectNode response, Map<String, Object> result) {
+ List<KinesisService.ChildShard> children = (List<KinesisService.ChildShard>) result.get("ChildShards");
+ if (children == null) {
+ return;
+ }
+ ArrayNode childNodes = response.putArray("ChildShards");
+ for (KinesisService.ChildShard child : children) {
+ ObjectNode node = childNodes.addObject();
+ node.put("ShardId", child.shardId());
+ ArrayNode parents = node.putArray("ParentShards");
+ child.parentShards().forEach(parents::add);
+ node.putObject("HashKeyRange")
+ .put("StartingHashKey", child.hashKeyRange().startingHashKey())
+ .put("EndingHashKey", child.hashKeyRange().endingHashKey());
+ }
+ }
+
private Response handleIncreaseStreamRetentionPeriod(JsonNode request, String region) {
String streamName = resolveStreamName(request);
int retentionPeriodHours = request.path("RetentionPeriodHours").asInt();
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
index 3e0e51df60..0c2137271f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
@@ -893,14 +893,8 @@ public class KinesisService implements ResourceProvider {
}
}
- KinesisStream stream = resolveStream(streamName, region);
- KinesisShard shard = stream.getShards().stream()
- .filter(s -> s.getShardId().equals(shardId))
- .findFirst()
- .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));
- pruneExpiredRecords(stream, shard);
-
- KinesisShard.RecordsSnapshot snapshot = shard.snapshotRecords();
+ ShardReadSnapshot read = snapshotShard(null, streamName, shardId, region);
+ KinesisShard.RecordsSnapshot snapshot = read.records();
List<KinesisRecord> allRecords = snapshot.records();
int startIndex = 0;
@@ -946,6 +940,45 @@ public class KinesisService implements ResourceProvider {
}
}
+ return recordsPage(streamName, shardId, read, startIndex, limit);
+ }
+
+ public record ChildShard(String shardId, List<String> parentShards, KinesisShard.HashKeyRange hashKeyRange) {}
+
+ private record ShardReadSnapshot(KinesisShard.RecordsSnapshot records, boolean closed,
+ List<ChildShard> children) {}
+
+ private ShardReadSnapshot snapshotShard(String accountId, String streamName, String shardId, String region) {
+ // Share the append/reshard monitor so a terminal snapshot includes the final append and topology.
+ synchronized (lockFor(regionKey(region, streamName))) {
+ KinesisStream stream = resolveStreamForAccount(accountId, streamName, region);
+ KinesisShard shard = stream.getShards().stream()
+ .filter(candidate -> candidate.getShardId().equals(shardId))
+ .findFirst()
+ .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));
+ pruneExpiredRecords(stream, shard);
+ List<ChildShard> children = new ArrayList<>();
+ if (shard.isClosed()) {
+ for (KinesisShard child : stream.getShards()) {
+ if (shardId.equals(child.getParentShardId()) || shardId.equals(child.getAdjacentParentShardId())) {
+ List<String> parents = new ArrayList<>();
+ if (child.getParentShardId() != null) {
+ parents.add(child.getParentShardId());
+ }
+ if (child.getAdjacentParentShardId() != null) {
+ parents.add(child.getAdjacentParentShardId());
+ }
+ children.add(new ChildShard(child.getShardId(), List.copyOf(parents), child.getHashKeyRange()));
+ }
+ }
+ }
+ return new ShardReadSnapshot(shard.snapshotRecords(), shard.isClosed(), List.copyOf(children));
+ }
+ }
+
+ private Map<String, Object> recordsPage(String streamName, String shardId, ShardReadSnapshot read,
+ int startIndex, Integer limit) {
+ List<KinesisRecord> allRecords = read.records().records();
int max = limit != null ? Math.min(limit, 1000) : 1000;
List<KinesisRecord> result = new ArrayList<>();
int nextIndex = startIndex;
@@ -953,13 +986,15 @@ public class KinesisService implements ResourceProvider {
result.add(allRecords.get(i));
nextIndex = i + 1;
}
-
- String nextIterator = buildContinuationIterator(streamName, shardId, allRecords, nextIndex);
-
+ boolean terminal = read.closed() && nextIndex >= allRecords.size();
Map<String, Object> response = new HashMap<>();
response.put("Records", result);
- response.put("NextShardIterator", nextIterator);
+ response.put("NextShardIterator", terminal ? null
+ : buildContinuationIterator(streamName, shardId, allRecords, nextIndex));
response.put("MillisBehindLatest", computeMillisBehindLatest(allRecords, nextIndex));
+ if (terminal) {
+ response.put("ChildShards", read.children());
+ }
return response;
}
@@ -1149,14 +1184,8 @@ public class KinesisService implements ResourceProvider {
}
}
- KinesisStream stream = resolveStreamForAccount(accountId, streamName, region);
- KinesisShard shard = stream.getShards().stream()
- .filter(s -> s.getShardId().equals(shardId))
- .findFirst()
- .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));
- pruneExpiredRecords(stream, shard);
-
- KinesisShard.RecordsSnapshot snapshot = shard.snapshotRecords();
+ ShardReadSnapshot read = snapshotShard(accountId, streamName, shardId, region);
+ KinesisShard.RecordsSnapshot snapshot = read.records();
List<KinesisRecord> allRecords = snapshot.records();
int startIndex = 0;
// LATEST resolves the same way as AFTER_SEQUENCE_NUMBER (see the longer explanation in
@@ -1186,20 +1215,7 @@ public class KinesisService implements ResourceProvider {
}
}
- int max = limit != null ? Math.min(limit, 1000) : 1000;
- List<KinesisRecord> result = new ArrayList<>();
- int nextIndex = startIndex;
- for (int i = startIndex; i < allRecords.size() && result.size() < max; i++) {
- result.add(allRecords.get(i));
- nextIndex = i + 1;
- }
-
- String nextIterator = buildContinuationIterator(streamName, shardId, allRecords, nextIndex);
- Map<String, Object> response = new HashMap<>();
- response.put("Records", result);
- response.put("NextShardIterator", nextIterator);
- response.put("MillisBehindLatest", computeMillisBehindLatest(allRecords, nextIndex));
- return response;
+ return recordsPage(streamName, shardId, read, startIndex, limit);
}
private KinesisShard selectShard(KinesisStream stream, String partitionKey, String explicitHashKey) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
index 95e6e9e8ac..27a9e51088 100644
--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
@@ -12,6 +12,7 @@ import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlIdentity;
import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader;
import io.github.hectorvent.floci.services.kinesis.KinesisService;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;
import io.github.hectorvent.floci.services.pipes.model.DesiredState;
import io.github.hectorvent.floci.services.pipes.model.Pipe;
import io.github.hectorvent.floci.services.sqs.SqsService;
@@ -245,7 +246,7 @@ public class PipesPoller implements Resettable {
}
}
- private void pollKinesis(Pipe pipe, String region) {
+ void pollKinesis(Pipe pipe, String region) {
String pipeKey = pipeKey(pipe);
String streamName = extractResourceName(pipe.getSource());
String pipeAccountId = pipe.getAccountId();
@@ -262,10 +263,17 @@ public class PipesPoller implements Resettable {
? kinesisService.getRecordsForAccount(pipeAccountId, iterator, batchSize, region)
: kinesisService.getRecords(iterator, batchSize, region);
String nextIterator = (String) result.get("NextShardIterator");
- if (nextIterator != null) {
- kinesisIterators.put(pipeKey, nextIterator);
- }
List<?> records = (List<?>) result.get("Records");
+ if (nextIterator == null && records != null && !records.isEmpty()) {
+ KinesisRecord last = (KinesisRecord) records.getLast();
+ nextIterator = pipeAccountId != null
+ ? kinesisService.getShardIteratorForAccount(pipeAccountId, streamName,
+ "shardId-000000000000", "AFTER_SEQUENCE_NUMBER", last.getSequenceNumber(), region)
+ : kinesisService.getShardIterator(streamName, "shardId-000000000000",
+ "AFTER_SEQUENCE_NUMBER", last.getSequenceNumber(), null, region);
+ }
+ // Remember even the initial exhausted cursor, before filtering or delivery.
+ kinesisIterators.put(pipeKey, nextIterator != null ? nextIterator : iterator);
if (records == null || records.isEmpty()) {
return;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
index 2ad60fbe44..80b0e19b74 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
@@ -1,5 +1,7 @@
package io.github.hectorvent.floci.services.firehose;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
@@ -17,6 +19,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
@@ -778,6 +781,83 @@ class FirehoseServiceTest {
java.util.List.of(), "KinesisStreamAsSource", kinesisSource(sourceStream));
}
+ private void closeSourceParent() {
+ kinesisService.splitShard("src-stream", "shardId-000000000000", "100", "us-east-1");
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {1, 501})
+ void terminalSourceBatchDoesNotReplayBeforeOrAfterFlushAndRecreation(int count) {
+ createSourcedDeliveryStream("sourced-stream", "src-stream");
+ for (int i = 0; i < count; i++) {
+ kinesisService.putRecord("src-stream", "terminal".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");
+ }
+ closeSourceParent();
+ firehoseService.pollKinesisSources();
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+ firehoseService = newService(0);
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+ assertEquals("terminal\n".repeat(count), delivered("sink").text());
+ }
+
+ @Test
+ void terminalSourceBatchIsRereadWhenServiceIsRecreatedBeforeFlush() {
+ createSourcedDeliveryStream("sourced-stream", "src-stream");
+ kinesisService.putRecord("src-stream", "terminal".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");
+ closeSourceParent();
+ firehoseService.pollKinesisSources();
+ verifyNothingDelivered();
+ firehoseService = newService(0);
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+ assertEquals("terminal\n", delivered("sink").text());
+ }
+
+ @Test
+ void failedTerminalFlushKeepsPendingBatchWithoutCommittingOrDuplicating() {
+ createSourcedDeliveryStream("sourced-stream", "src-stream");
+ kinesisService.putRecord("src-stream", "terminal".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");
+ closeSourceParent();
+ when(s3Service.putObject(anyString(), anyString(), any(byte[].class), anyString(),
+ anyMap(), any(PutObjectOptions.class))).thenThrow(new RuntimeException("s3 unavailable"));
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+ assertTrue(backends.get("firehose/source-iterators.json").scan(key -> true).isEmpty());
+ Mockito.reset(s3Service);
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+ assertEquals("terminal\n", delivered("sink").text());
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ void committedTerminalCursorSurvivesSerializationRoundTrip() throws Exception {
+ createSourcedDeliveryStream("sourced-stream", "src-stream");
+ kinesisService.putRecord("src-stream", "terminal".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");
+ closeSourceParent();
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+ AccountAwareStorageBackend<Map<String, String>> cursors =
+ (AccountAwareStorageBackend<Map<String, String>>) backends.get("firehose/source-iterators.json");
+ ObjectMapper mapper = new ObjectMapper();
+ for (String key : cursors.keys()) {
+ Map<String, String> restored = mapper.readValue(mapper.writeValueAsBytes(cursors.get(key).orElseThrow()),
+ new TypeReference<Map<String, String>>() {});
+ cursors.put(key, restored);
+ }
+ assertFalse(cursors.keys().isEmpty());
+ firehoseService = newService(0);
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+ assertEquals("terminal\n", delivered("sink").text());
+ }
+
@Test
void kinesisSourceRecordsAreDelivered() {
createSourcedDeliveryStream("sourced-stream", "src-stream");
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
index 082a92de10..db0c1dc6b0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
@@ -17,6 +17,7 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.time.Clock;
import java.time.Instant;
import java.time.ZoneOffset;
@@ -59,6 +60,53 @@ class KinesisJsonHandlerTest {
handler = new KinesisJsonHandler(service, MAPPER, 300_000);
}
+ @Test
+ void terminalGetRecordsIsJsonNullWithExactChildMetadata() throws Exception {
+ createStream("test-stream");
+ service.putRecord("test-stream", new byte[]{1}, "pk", REGION);
+ service.splitShard("test-stream", FIRST_SHARD_ID, "100", REGION);
+ String cursor = service.getShardIterator("test-stream", FIRST_SHARD_ID, "TRIM_HORIZON", null, null, REGION);
+ ObjectNode request = MAPPER.createObjectNode().put("ShardIterator", cursor);
+ JsonNode page = MAPPER.readTree(responseEntity(handler.handle("GetRecords", request, REGION)).toString());
+ assertTrue(page.has("NextShardIterator"));
+ assertTrue(page.get("NextShardIterator").isNull());
+ assertEquals(1, page.path("Records").size());
+ assertEquals(MAPPER.readTree("""
+ [{"ShardId":"shardId-000000000001","ParentShards":["shardId-000000000000"],
+ "HashKeyRange":{"StartingHashKey":"0","EndingHashKey":"99"}},
+ {"ShardId":"shardId-000000000002","ParentShards":["shardId-000000000000"],
+ "HashKeyRange":{"StartingHashKey":"100","EndingHashKey":"340282366920938463463374607431768211455"}}]
+ """), page.path("ChildShards"));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void subscriptionTerminalMetadataDoesNotContinueEvenWithRecords(boolean nonempty) throws Exception {
+ createStream("test-stream");
+ if (nonempty) {
+ service.putRecord("test-stream", new byte[]{1}, "pk", REGION);
+ }
+ String arn = service.registerStreamConsumer(STREAM_ARN, "terminal-consumer", REGION).getConsumerArn();
+ service.splitShard("test-stream", FIRST_SHARD_ID, "100", REGION);
+ ObjectNode request = MAPPER.createObjectNode().put("ConsumerARN", arn).put("ShardId", FIRST_SHARD_ID);
+ request.putObject("StartingPosition").put("Type", "TRIM_HORIZON");
+ byte[] wire = (byte[]) handler.handle("SubscribeToShard", request, REGION).getEntity();
+ ByteBuffer buffer = ByteBuffer.wrap(wire);
+ int initialLength = buffer.getInt();
+ buffer.position(initialLength);
+ int eventLength = buffer.getInt();
+ int headerLength = buffer.getInt();
+ buffer.position(initialLength + 12 + headerLength);
+ byte[] payload = new byte[eventLength - headerLength - 16];
+ buffer.get(payload);
+ JsonNode event = MAPPER.readTree(payload);
+ assertEquals(nonempty ? 1 : 0, event.path("Records").size());
+ assertFalse(event.has("ContinuationSequenceNumber"));
+ assertEquals(2, event.path("ChildShards").size());
+ assertEquals(FIRST_SHARD_ID, event.path("ChildShards").get(0).path("ParentShards").get(0).asText());
+ assertEquals("99", event.path("ChildShards").get(0).path("HashKeyRange").path("EndingHashKey").asText());
+ }
+
// Retention is measured against the service clock, so a test that plants a fixed arrival
// timestamp pins the clock just after it instead of relying on the wall clock.
private void useClockFixedAt(Instant now) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisTerminalReadServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisTerminalReadServiceTest.java
new file mode 100644
index 0000000000..b811bac3d0
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisTerminalReadServiceTest.java
@@ -0,0 +1,202 @@
+package io.github.hectorvent.floci.services.kinesis;
+
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.spy;
+
+class KinesisTerminalReadServiceTest {
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT = "000000000000";
+ private static final String PARENT = "shardId-000000000000";
+ private final AccountAwareStorageBackend<KinesisStream> streams = AccountAwareStorageBackend.inMemory(ACCOUNT);
+ private final KinesisService service = new KinesisService(streams, AccountAwareStorageBackend.inMemory(ACCOUNT),
+ new RegionResolver(REGION, ACCOUNT));
+
+ private String iterator(String type, String sequence, Long timestamp) {
+ return service.getShardIterator("s", PARENT, type, sequence, timestamp, REGION);
+ }
+
+ private Map<String, Object> read(boolean account, String iterator, int limit) {
+ return account ? service.getRecordsForAccount(ACCOUNT, iterator, limit, REGION)
+ : service.getRecords(iterator, limit, REGION);
+ }
+
+ private String put(String value) {
+ return service.putRecord("s", value.getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ }
+
+ private void split() {
+ service.splitShard("s", PARENT, "170141183460469231731687303715884105728", REGION);
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<KinesisService.ChildShard> children(Map<String, Object> page) {
+ return (List<KinesisService.ChildShard>) page.get("ChildShards");
+ }
+
+ @Test
+ void atSequenceNumberOnClosedShardIncludesTheRequestedRecord() {
+ service.createStream("s", 1, REGION);
+ put("first");
+ String last = put("last");
+ split();
+ Map<String, Object> page = service.getRecords(iterator("AT_SEQUENCE_NUMBER", last, null), 10, REGION);
+ assertEquals(1, ((List<?>) page.get("Records")).size());
+ assertNull(page.get("NextShardIterator"));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void finalNonemptyPageTerminatesAndOnlyTerminalPagesHaveChildren(boolean account) {
+ service.createStream("s", 1, REGION);
+ put("first");
+ put("last");
+ split();
+ Map<String, Object> first = read(account, iterator("TRIM_HORIZON", null, null), 1);
+ assertEquals(1, ((List<?>) first.get("Records")).size());
+ assertNotNull(first.get("NextShardIterator"));
+ assertFalse(first.containsKey("ChildShards"));
+ Map<String, Object> last = read(account, (String) first.get("NextShardIterator"), 1);
+ assertEquals(1, ((List<?>) last.get("Records")).size());
+ assertNull(last.get("NextShardIterator"));
+ assertEquals(0L, last.get("MillisBehindLatest"));
+ assertEquals(List.of("shardId-000000000001", "shardId-000000000002"),
+ children(last).stream().map(KinesisService.ChildShard::shardId).toList());
+ assertEquals(List.of(PARENT), children(last).getFirst().parentShards());
+ assertEquals(new KinesisShard.HashKeyRange("0", "170141183460469231731687303715884105727"),
+ children(last).getFirst().hashKeyRange());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void caughtUpOpenShardRemainsReadableAndClosedPositionsTerminate(boolean account) {
+ service.createStream("s", 1, REGION);
+ String sequence = put("last");
+ String latest = iterator("LATEST", null, null);
+ Map<String, Object> open = read(account, latest, 10);
+ assertTrue(((List<?>) open.get("Records")).isEmpty());
+ assertNotNull(open.get("NextShardIterator"));
+ assertFalse(open.containsKey("ChildShards"));
+ split();
+ for (String cursor : List.of(latest, iterator("AFTER_SEQUENCE_NUMBER", sequence, null),
+ iterator("AT_TIMESTAMP", null, Instant.now().plusSeconds(60).toEpochMilli()))) {
+ Map<String, Object> closed = read(account, cursor, 10);
+ assertTrue(((List<?>) closed.get("Records")).isEmpty());
+ assertNull(closed.get("NextShardIterator"));
+ assertEquals(2, children(closed).size());
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void emptyOrRetentionPrunedParentsTerminateWithDirectChildren(boolean account) {
+ KinesisStream stream = service.createStream("s", 1, REGION);
+ String cursor = iterator("TRIM_HORIZON", null, null);
+ split();
+ assertNull(read(account, cursor, 10).get("NextShardIterator"));
+ stream.getShards().getFirst().addRecord(new KinesisRecord(new byte[]{1}, "pk", "1",
+ Instant.now().minusSeconds(172800)));
+ service.splitShard("s", "shardId-000000000001", "85070591730234615865843651857942052864", REGION);
+ Map<String, Object> page = read(account, cursor, 10);
+ assertTrue(((List<?>) page.get("Records")).isEmpty());
+ assertNull(page.get("NextShardIterator"));
+ assertEquals(List.of("shardId-000000000001", "shardId-000000000002"),
+ children(page).stream().map(KinesisService.ChildShard::shardId).toList());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void bothMergeParentsReportBothParentIds(boolean account) {
+ service.createStream("s", 2, REGION);
+ String adjacent = "shardId-000000000001";
+ service.mergeShards("s", PARENT, adjacent, REGION);
+ for (String parent : List.of(PARENT, adjacent)) {
+ String cursor = service.getShardIterator("s", parent, "TRIM_HORIZON", null, null, REGION);
+ Map<String, Object> page = read(account, cursor, 10);
+ assertNull(page.get("NextShardIterator"));
+ assertEquals(List.of(new KinesisService.ChildShard("shardId-000000000002", List.of(PARENT, adjacent),
+ new KinesisShard.HashKeyRange("0", "340282366920938463463374607431768211455"))), children(page));
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void snapshotSerializesWithExplicitAccountAppendsAndResharding(boolean account) throws Exception {
+ KinesisStream stream = service.createStream("s", 1, REGION);
+ KinesisShard parent = spy(stream.getShards().getFirst());
+ stream.getShards().set(0, parent);
+ put("first");
+ CountDownLatch captured = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ KinesisShard.RecordsSnapshot snapshot = (KinesisShard.RecordsSnapshot) invocation.callRealMethod();
+ captured.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ return snapshot;
+ }).when(parent).snapshotRecords();
+ String cursor = iterator("TRIM_HORIZON", null, null);
+ try (ExecutorService pool = Executors.newFixedThreadPool(2)) {
+ Future<Map<String, Object>> reading = pool.submit(() -> read(account, cursor, 10));
+ try {
+ assertTrue(captured.await(5, TimeUnit.SECONDS));
+ CountDownLatch writing = new CountDownLatch(1);
+ Future<?> mutation = pool.submit(() -> {
+ writing.countDown();
+ service.putRecordForAccount(ACCOUNT, "s", new byte[]{2}, "pk", REGION);
+ split();
+ });
+ assertTrue(writing.await(5, TimeUnit.SECONDS));
+ assertThrows(TimeoutException.class, () -> mutation.get(150, TimeUnit.MILLISECONDS));
+ release.countDown();
+ Map<String, Object> open = reading.get(5, TimeUnit.SECONDS);
+ assertNotNull(open.get("NextShardIterator"));
+ assertEquals(1, ((List<?>) open.get("Records")).size());
+ mutation.get(5, TimeUnit.SECONDS);
+ Map<String, Object> terminal = read(account, (String) open.get("NextShardIterator"), 10);
+ assertEquals(1, ((List<?>) terminal.get("Records")).size());
+ assertNull(terminal.get("NextShardIterator"));
+ assertEquals(2, children(terminal).size());
+ } finally {
+ release.countDown();
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void accountAndRegionResolveTheirOwnTerminalState(boolean closedOther) {
+ service.createStream("s", 1, REGION);
+ split();
+ KinesisStream other = new KinesisStream("s", "arn:aws:kinesis:us-east-1:111111111111:stream/s");
+ KinesisShard shard = new KinesisShard(PARENT, "0", "100", "0");
+ shard.setClosed(closedOther);
+ other.getShards().add(shard);
+ streams.putForAccount("111111111111", REGION + "::s", other);
+ String cursor = iterator("TRIM_HORIZON", null, null);
+ Map<String, Object> page = service.getRecordsForAccount("111111111111", cursor, 10, REGION);
+ assertEquals(closedOther, page.get("NextShardIterator") == null);
+ assertNull(service.getRecords(cursor, 10, REGION).get("NextShardIterator"));
+ service.createStream("s", 1, "eu-west-1");
+ assertNotNull(service.getRecords(cursor, 10, "eu-west-1").get("NextShardIterator"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePollerTest.java
index 2eeda9f07e..a4c7ddc883 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePollerTest.java
@@ -30,6 +30,7 @@ import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Base64;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -125,6 +126,25 @@ class KinesisEventSourcePollerTest {
clock::get);
}
+ @Test
+ void finalNonemptyPageWithNullIteratorStillInvokesAndCheckpoints() throws Exception {
+ stubFunction();
+ stubStreamWith(List.of(record("123", "pk", "terminal")));
+ Map<String, Object> terminal = new HashMap<>();
+ terminal.put("Records", List.of(record("123", "pk", "terminal")));
+ terminal.put("NextShardIterator", null);
+ terminal.put("ChildShards", List.of());
+ when(kinesisService.getRecordsForAccount(eq(ACCOUNT), eq("iter-0"), anyInt(), eq(REGION)))
+ .thenReturn(terminal);
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult());
+ EventSourceMapping mapping = esm();
+ pollOnce(mapping);
+ verify(executorService).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));
+ assertEquals("123", mapping.getShardSequenceNumbers().get(SHARD));
+ verify(esmStore).saveForAccount(eq(ACCOUNT), eq(mapping));
+ }
+
private EventSourceMapping esm(String... patterns) {
EventSourceMapping esm = new EventSourceMapping();
esm.setUuid("k-esm");
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKinesisTerminalTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKinesisTerminalTest.java
new file mode 100644
index 0000000000..fc3d452d39
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKinesisTerminalTest.java
@@ -0,0 +1,98 @@
+package io.github.hectorvent.floci.services.pipes;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
+import io.github.hectorvent.floci.services.pipes.model.Pipe;
+import io.github.hectorvent.floci.services.sqs.SqsService;
+import io.vertx.core.Vertx;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+
+class PipesKinesisTerminalTest {
+ @ParameterizedTest
+ @CsvSource({"true,0,success", "false,0,success", "true,1,success", "false,1,success", "true,2,success", "false,2,success",
+ "true,1,filtered", "true,2,filtered", "true,1,dlq", "true,2,dlq", "true,1,drop", "true,2,drop"})
+ void terminalPagesAdvanceBeforeFilteringOrDelivery(boolean account, int recordCount, String outcome) throws Exception {
+ ObjectMapper mapper = new ObjectMapper().findAndRegisterModules();
+ StorageFactory factory = mock(StorageFactory.class);
+ Map<String, AccountAwareStorageBackend<?>> stores = new HashMap<>();
+ when(factory.create(anyString(), anyString(), any())).thenAnswer(inv -> stores.computeIfAbsent(
+ inv.getArgument(1), key -> AccountAwareStorageBackend.inMemory("111111111111")));
+ KinesisService kinesis = spy(new KinesisService(factory, new RegionResolver("eu-west-1", "111111111111")));
+ kinesis.createStream("s", 1, "eu-west-1");
+ for (int i = 0; i < recordCount; i++) {
+ kinesis.putRecord("s", new byte[]{1}, "pk", "eu-west-1");
+ }
+ kinesis.splitShard("s", "shardId-000000000000", "100", "eu-west-1");
+ AtomicInteger terminalRecordPages = new AtomicInteger();
+ doAnswer(invocation -> {
+ @SuppressWarnings("unchecked")
+ Map<String, Object> page = (Map<String, Object>) invocation.callRealMethod();
+ if (page.get("NextShardIterator") == null && !((List<?>) page.get("Records")).isEmpty()) {
+ terminalRecordPages.incrementAndGet();
+ }
+ return page;
+ }).when(kinesis).getRecordsForAccount(anyString(), anyString(), any(), anyString());
+ doAnswer(invocation -> {
+ @SuppressWarnings("unchecked")
+ Map<String, Object> page = (Map<String, Object>) invocation.callRealMethod();
+ if (page.get("NextShardIterator") == null && !((List<?>) page.get("Records")).isEmpty()) {
+ terminalRecordPages.incrementAndGet();
+ }
+ return page;
+ }).when(kinesis).getRecords(anyString(), any(), anyString());
+ Pipe pipe = new Pipe();
+ pipe.setName("terminal");
+ pipe.setArn("arn:aws:pipes:eu-west-1:111111111111:pipe/terminal");
+ pipe.setSource("arn:aws:kinesis:eu-west-1:111111111111:stream/s");
+ pipe.setTarget("arn:aws:lambda:eu-west-1:111111111111:function:target");
+ pipe.setAccountId(account ? "111111111111" : null);
+ pipe.setSourceParameters(mapper.readTree("""
+ {"KinesisStreamParameters":{"BatchSize":1,
+ "DeadLetterConfig":{"Arn":"arn:aws:sqs:eu-west-1:111111111111:dlq"}}}
+ """));
+ if ("filtered".equals(outcome)) {
+ pipe.getSourceParameters().withObject("/FilterCriteria").putArray("Filters").addObject()
+ .put("Pattern", "{\"eventSource\":[\"aws:sqs\"]}");
+ }
+ SqsService sqs = mock(SqsService.class);
+ PipesTargetInvoker target = mock(PipesTargetInvoker.class);
+ if ("dlq".equals(outcome) || "drop".equals(outcome)) {
+ doThrow(new RuntimeException("target down")).when(target).invoke(eq(pipe), anyString(), eq("eu-west-1"));
+ }
+ if ("drop".equals(outcome)) {
+ when(sqs.sendMessage(anyString(), anyString(), anyInt(), anyString()))
+ .thenThrow(new RuntimeException("dlq down"));
+ }
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ PipesPoller poller = new PipesPoller(mock(Vertx.class), sqs, kinesis, mock(DynamoDbStreamReader.class),
+ mock(PipesKafkaConsumerManager.class), target, new PipesFilterMatcher(mapper), mapper, config);
+ try {
+ for (int i = 0; i < recordCount + 3; i++) {
+ poller.pollKinesis(pipe, "eu-west-1");
+ }
+ verify(target, times("filtered".equals(outcome) ? 0 : recordCount))
+ .invoke(eq(pipe), anyString(), eq("eu-west-1"));
+ verify(sqs, times("dlq".equals(outcome) || "drop".equals(outcome) ? recordCount : 0))
+ .sendMessage(anyString(), anyString(), eq(0), eq("eu-west-1"));
+ assertEquals(recordCount == 0 ? 0 : 1, terminalRecordPages.get(), "the terminal record page must be consumed once");
+ } finally {
+ poller.shutdown();
+ }
+ }
+}