Light, fluffy, and always free - The AWS Local Emulator alternative
fix(kinesis): stop routing records to closed shards (#6414)
TiM39f37c21ffd3
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.javaindex 0a05647a53..821c3c3ebf 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.java@@ -5,6 +5,7 @@ import org.junit.jupiter.api.DisplayName;import org.junit.jupiter.api.Test;import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;+import software.amazon.awssdk.core.SdkBytes;import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;import software.amazon.awssdk.core.interceptor.Context;import software.amazon.awssdk.core.interceptor.ExecutionAttributes;@@ -16,13 +17,26 @@ import software.amazon.awssdk.services.kinesis.model.CreateStreamRequest;import software.amazon.awssdk.services.kinesis.model.DeleteStreamRequest;import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryRequest;import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryResponse;+import software.amazon.awssdk.services.kinesis.model.GetRecordsRequest;+import software.amazon.awssdk.services.kinesis.model.GetShardIteratorRequest;import software.amazon.awssdk.services.kinesis.model.ListShardsRequest;import software.amazon.awssdk.services.kinesis.model.ListShardsResponse;-+import software.amazon.awssdk.services.kinesis.model.PutRecordRequest;+import software.amazon.awssdk.services.kinesis.model.PutRecordResponse;+import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest;+import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry;+import software.amazon.awssdk.services.kinesis.model.PutRecordsResponse;+import software.amazon.awssdk.services.kinesis.model.Shard;+import software.amazon.awssdk.services.kinesis.model.ShardIteratorType;+import software.amazon.awssdk.services.kinesis.model.SplitShardRequest;++import java.math.BigInteger;import java.util.ArrayList;import java.util.HashSet;import java.util.List;+import java.util.Set;import java.util.concurrent.atomic.AtomicReference;+import java.util.stream.Collectors;import static org.assertj.core.api.Assertions.assertThat;import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;@@ -42,6 +56,62 @@ class KinesisTest {}}+ @Test+ @DisplayName("PutRecord and PutRecords route to open descendants after repeated splits")+ void writesUseOpenDescendantsAfterRepeatedSplits() {+ try (KinesisClient client = TestFixtures.kinesisClient()) {+ String streamName = TestFixtures.uniqueName("sdk-kinesis-reshard-routing");+ client.createStream(CreateStreamRequest.builder().streamName(streamName).shardCount(1).build());+ try {+ Shard parent = client.listShards(ListShardsRequest.builder().streamName(streamName).build())+ .shards().get(0);+ client.putRecord(PutRecordRequest.builder().streamName(streamName).partitionKey("b")+ .data(SdkBytes.fromUtf8String("before split")).build());+ split(client, streamName, parent);+ List<Shard> children = client.listShards(ListShardsRequest.builder().streamName(streamName).build())+ .shards().stream().filter(shard -> shard.sequenceNumberRange().endingSequenceNumber() == null)+ .collect(Collectors.toList());+ for (Shard child : children) {+ split(client, streamName, child);+ }+ Set<String> openIds = client.listShards(ListShardsRequest.builder().streamName(streamName).build())+ .shards().stream().filter(shard -> shard.sequenceNumberRange().endingSequenceNumber() == null)+ .map(Shard::shardId).collect(Collectors.toSet());+ assertThat(openIds).hasSize(4);++ PutRecordResponse single = client.putRecord(PutRecordRequest.builder().streamName(streamName)+ .partitionKey("b").data(SdkBytes.fromUtf8String("single")).build());+ assertThat(openIds).contains(single.shardId());+ PutRecordsResponse batch = client.putRecords(PutRecordsRequest.builder().streamName(streamName)+ .records(PutRecordsRequestEntry.builder().partitionKey("b")+ .data(SdkBytes.fromUtf8String("batch ordinary")).build(),+ PutRecordsRequestEntry.builder().partitionKey("b")+ .explicitHashKey(parent.hashKeyRange().endingHashKey())+ .data(SdkBytes.fromUtf8String("batch explicit")).build()).build());+ assertThat(batch.failedRecordCount()).isZero();+ assertThat(batch.records()).allSatisfy(record -> {+ assertThat(openIds).contains(record.shardId());+ assertThat(record.sequenceNumber()).isNotBlank();+ });+ String iterator = client.getShardIterator(GetShardIteratorRequest.builder().streamName(streamName)+ .shardId(parent.shardId()).shardIteratorType(ShardIteratorType.TRIM_HORIZON).build())+ .shardIterator();+ assertThat(client.getRecords(GetRecordsRequest.builder().shardIterator(iterator).build()).records())+ .singleElement().satisfies(record ->+ assertThat(record.data().asUtf8String()).isEqualTo("before split"));+ } finally {+ client.deleteStream(DeleteStreamRequest.builder().streamName(streamName).build());+ }+ }+ }++ private static void split(KinesisClient client, String streamName, Shard shard) {+ BigInteger start = new BigInteger(shard.hashKeyRange().startingHashKey());+ BigInteger end = new BigInteger(shard.hashKeyRange().endingHashKey());+ client.splitShard(SplitShardRequest.builder().streamName(streamName).shardToSplit(shard.shardId())+ .newStartingHashKey(start.add(end).add(BigInteger.ONE).divide(BigInteger.TWO).toString()).build());+ }+@Testvoid awsSdkV2UsesRootCborRoute() {AtomicReference<SdkHttpRequest> requestRef = new AtomicReference<>();diff --git a/docs/services/kinesis.md b/docs/services/kinesis.mdindex 375bb37f4d..51d150256a 100644--- a/docs/services/kinesis.md+++ b/docs/services/kinesis.md@@ -86,7 +86,17 @@ A stream's retention period (default 24 hours, `IncreaseStreamRetentionPeriod`/`## Record Routing-`PutRecord` and `PutRecords` honor `ExplicitHashKey` when it is provided. The value must be a decimal integer in the Kinesis hash-key space, and records are written to the open shard whose `HashKeyRange` contains that value. Without `ExplicitHashKey`, Floci keeps using the partition key to choose a shard.+`PutRecord` and `PutRecords` honor `ExplicitHashKey` when it is provided. The+value must be a decimal integer in the Kinesis hash-key space, and records are+written to the open shard whose `HashKeyRange` contains that value.++Without `ExplicitHashKey`, Floci uses the partition key's Java hash modulo the+number of stored shards, including closed shards. If this selects a closed+shard, it follows split and merge lineage through any closed intermediate shards+to the first open descendant in breadth-first, stored-shard order. Writes never+append to a closed shard or fall back to an unrelated open shard. A lineage with+no reachable open shard fails the write. This simplified ordinary-key placement+differs from AWS's MD5 hash-range routing.## Shard Scaling (UpdateShardCount)diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.javaindex c67ab53068..a50383db94 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java@@ -23,8 +23,16 @@ import java.nio.charset.StandardCharsets;import java.time.Clock;import java.time.Duration;import java.time.Instant;-import java.util.*;+import java.util.ArrayDeque;import java.util.ArrayList;+import java.util.Base64;+import java.util.Comparator;+import java.util.Deque;+import java.util.HashMap;+import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.PriorityQueue;import java.util.Set;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.atomic.AtomicLong;@@ -1208,21 +1216,42 @@ public class KinesisService implements ResourceProvider {return selectShardByExplicitHashKey(stream, explicitHashKey);}- // Simple hash-based shard selection among ALL shards, then resolve to open one- int index = (int) (Math.abs((long) partitionKey.hashCode()) % stream.getShards().size());- KinesisShard shard = stream.getShards().get(index);+ // Preserve ordinary-key placement, then resolve historical parents to an open descendant.+ // The producer holds the stream lock, shared with splitShard and mergeShards.+ List<KinesisShard> shards = stream.getShards();+ int index = (int) (Math.abs((long) partitionKey.hashCode()) % shards.size());+ KinesisShard selected = shards.get(index);+ if (!selected.isClosed()) {+ return selected;+ }- // If closed, find the first open child (simplified)- while (shard.isClosed()) {- KinesisShard finalShard = shard;- shard = stream.getShards().stream()- .filter(s -> finalShard.getShardId().equals(s.getParentShardId()) || finalShard.getShardId().equals(s.getAdjacentParentShardId()))- .filter(s -> !s.isClosed())- .findFirst()- .orElse(shard); // Fallback to itself if no open child found- if (shard == finalShard) break; // prevent infinite loop+ List<KinesisShard> snapshot = List.copyOf(shards);+ Map<String, List<KinesisShard>> children = new HashMap<>();+ for (KinesisShard shard : snapshot) {+ if (shard.getParentShardId() != null) {+ children.computeIfAbsent(shard.getParentShardId(), ignored -> new ArrayList<>()).add(shard);+ }+ if (shard.getAdjacentParentShardId() != null) {+ children.computeIfAbsent(shard.getAdjacentParentShardId(), ignored -> new ArrayList<>()).add(shard);+ }+ }+ Deque<KinesisShard> pending = new ArrayDeque<>();+ Set<String> visited = new HashSet<>();+ pending.add(selected);+ visited.add(selected.getShardId());+ while (!pending.isEmpty()) {+ KinesisShard shard = pending.removeFirst();+ if (!shard.isClosed()) {+ return shard;+ }+ for (KinesisShard child : children.getOrDefault(shard.getShardId(), List.of())) {+ if (visited.add(child.getShardId())) {+ pending.addLast(child);+ }+ }}- return shard;+ throw new AwsException("InvalidArgumentException",+ "Selected shard " + selected.getShardId() + " has no open descendant.", 400);}private KinesisShard selectShardByExplicitHashKey(KinesisStream stream, String explicitHashKey) {diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisClosedShardRoutingServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisClosedShardRoutingServiceTest.javanew file mode 100644index 0000000000..b9cc5c2642--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisClosedShardRoutingServiceTest.java@@ -0,0 +1,195 @@+package io.github.hectorvent.floci.services.kinesis;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.math.BigInteger;+import java.time.Duration;+import java.util.List;+import java.util.Map;+import java.util.stream.Collectors;++import static org.junit.jupiter.api.Assertions.*;++class KinesisClosedShardRoutingServiceTest {++ private static final String REGION = "us-east-1";+ private static final String STREAM = "routing";+ private static final byte[] DATA = {1};++ private KinesisService service;++ @BeforeEach+ void setUp() {+ service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),+ new RegionResolver(REGION, "000000000000"));+ }++ @Test+ void putRecordFollowsAnOpenSplitChild() {+ KinesisStream stream = service.createStream(STREAM, 1, REGION);+ service.putRecord(STREAM, DATA, "c", REGION);+ split(stream.getShards().getFirst());++ assertOpenDestinationAndUnchangedClosedRecords(stream, "c", stream.getShards().get(1));+ }++ @Test+ void putRecordTraversesClosedSplitChildren() {+ KinesisStream stream = splitTwoGenerations();++ // "b" hashes to 98, selecting the original parent at index zero modulo seven.+ assertOpenDestinationAndUnchangedClosedRecords(stream, "b", stream.getShards().get(3));+ }++ @ParameterizedTest+ @ValueSource(ints = {1, 2})+ void putRecordPrefersAnOpenDirectChildOverADeeperDescendant(int closedChildIndex) {+ KinesisStream stream = service.createStream(STREAM, 1, REGION);+ split(stream.getShards().getFirst());+ split(stream.getShards().get(closedChildIndex));++ // "d" selects the root modulo five, with one direct child still open.+ assertOpenDestinationAndUnchangedClosedRecords(stream, "d", stream.getShards().get(3 - closedChildIndex));+ }++ @ParameterizedTest+ @ValueSource(strings = {"d", "e"})+ void putRecordTraversesEitherMergeParentThenAClosedChild(String partitionKey) {+ KinesisStream stream = service.createStream(STREAM, 2, REGION);+ service.mergeShards(STREAM, stream.getShards().get(0).getShardId(),+ stream.getShards().get(1).getShardId(), REGION);+ split(stream.getShards().get(2));++ // With five stored shards, these keys select each original merge parent.+ assertOpenDestinationAndUnchangedClosedRecords(stream, partitionKey, stream.getShards().get(3));+ }++ @Test+ void putRecordTraversesSplitThenMergeThenSplit() {+ KinesisStream stream = service.createStream(STREAM, 1, REGION);+ split(stream.getShards().getFirst());+ service.mergeShards(STREAM, stream.getShards().get(1).getShardId(),+ stream.getShards().get(2).getShardId(), REGION);+ split(stream.getShards().get(3));++ assertOpenDestinationAndUnchangedClosedRecords(stream, "f", stream.getShards().get(4));+ }++ @Test+ void putRecordUsesAReachableChildInsteadOfAnEarlierUnrelatedOpenShard() {+ KinesisStream stream = service.createStream(STREAM, 2, REGION);+ service.putRecordWithShardId(STREAM, DATA, "d", "0", REGION);+ split(stream.getShards().getFirst());++ // "d" selects the closed parent modulo four; shard one is unrelated and open.+ assertOpenDestinationAndUnchangedClosedRecords(stream, "d", stream.getShards().get(2));++ assertEquals(1, stream.getShards().get(0).recordCount());+ assertEquals(0, stream.getShards().get(1).recordCount());+ assertEquals(1, stream.getShards().get(2).recordCount());+ }++ @Test+ void putRecordRejectsAClosedShardWithoutDescendants() {+ KinesisStream stream = service.createStream(STREAM, 1, REGION);+ stream.getShards().getFirst().setClosed(true);++ assertNoDestination(stream, "b");+ }++ @Test+ void putRecordDoesNotFallBackToAnUnrelatedOpenShard() {+ KinesisStream stream = service.createStream(STREAM, 2, REGION);+ stream.getShards().getFirst().setClosed(true);++ assertNoDestination(stream, "b");+ }++ @Test+ void putRecordRejectsACycleWithoutHangingOrAppending() {+ KinesisStream stream = service.createStream(STREAM, 2, REGION);+ KinesisShard first = stream.getShards().get(0);+ KinesisShard second = stream.getShards().get(1);+ first.setClosed(true);+ second.setClosed(true);+ first.setParentShardId(second.getShardId());+ second.setAdjacentParentShardId(first.getShardId());++ assertTimeoutPreemptively(Duration.ofSeconds(2), () -> assertNoDestination(stream, "b"));+ }++ @Test+ void explicitHashKeysKeepRoutingToOpenRangeBoundariesAfterRepeatedSplits() {+ KinesisStream stream = splitTwoGenerations();+ List<KinesisShard> openShards = stream.getShards().stream().filter(shard -> !shard.isClosed()).toList();++ for (KinesisShard shard : openShards) {+ for (String hash : List.of(shard.getHashKeyRange().startingHashKey(),+ shard.getHashKeyRange().endingHashKey())) {+ KinesisService.PutRecordResult result = service.putRecordWithShardId(STREAM, DATA, "b", hash, REGION);+ assertEquals(shard.getShardId(), result.shardId());+ }+ }+ assertTrue(stream.getShards().stream().filter(KinesisShard::isClosed)+ .allMatch(shard -> shard.recordCount() == 0));+ }++ @Test+ void explicitAccountProducerAlsoTraversesClosedDescendants() {+ KinesisStream stream = splitTwoGenerations();++ String sequence = service.putRecordForAccount("000000000000", STREAM, DATA, "b", REGION);++ assertTrue(stream.getShards().stream().filter(KinesisShard::isClosed)+ .allMatch(shard -> shard.recordCount() == 0));+ assertTrue(stream.getShards().stream().filter(shard -> !shard.isClosed())+ .flatMap(shard -> shard.getRecords().stream())+ .anyMatch(record -> sequence.equals(record.getSequenceNumber())));+ }++ private KinesisStream splitTwoGenerations() {+ KinesisStream stream = service.createStream(STREAM, 1, REGION);+ split(stream.getShards().getFirst());+ List<KinesisShard> children = stream.getShards().stream().filter(shard -> !shard.isClosed()).toList();+ children.forEach(this::split);+ return stream;+ }++ private void split(KinesisShard shard) {+ BigInteger start = new BigInteger(shard.getHashKeyRange().startingHashKey());+ BigInteger end = new BigInteger(shard.getHashKeyRange().endingHashKey());+ service.splitShard(STREAM, shard.getShardId(), start.add(end).add(BigInteger.ONE)+ .divide(BigInteger.TWO).toString(), REGION);+ }++ private void assertOpenDestinationAndUnchangedClosedRecords(KinesisStream stream, String partitionKey,+ KinesisShard expectedDestination) {+ Map<String, Integer> closedCounts = stream.getShards().stream().filter(KinesisShard::isClosed)+ .collect(Collectors.toMap(KinesisShard::getShardId, KinesisShard::recordCount));+ KinesisService.PutRecordResult result = service.putRecordWithShardId(STREAM, DATA, partitionKey, REGION);+ KinesisShard destination = stream.getShards().stream()+ .filter(shard -> result.shardId().equals(shard.getShardId())).findFirst().orElseThrow();++ assertFalse(destination.isClosed());+ assertEquals(expectedDestination.getShardId(), result.shardId());+ assertEquals(result.sequenceNumber(), destination.getRecords().getLast().getSequenceNumber());+ stream.getShards().stream().filter(KinesisShard::isClosed)+ .forEach(shard -> assertEquals(closedCounts.get(shard.getShardId()).intValue(), shard.recordCount()));+ }++ private void assertNoDestination(KinesisStream stream, String partitionKey) {+ AwsException error = assertThrows(AwsException.class,+ () -> service.putRecordWithShardId(STREAM, DATA, partitionKey, REGION));+ assertEquals("InvalidArgumentException", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertTrue(stream.getShards().stream().allMatch(shard -> shard.recordCount() == 0));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.javaindex d851a742cb..082a92de10 100644--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java@@ -13,6 +13,8 @@ import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;import jakarta.ws.rs.core.Response;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;import java.math.BigDecimal;import java.time.Clock;@@ -84,6 +86,58 @@ class KinesisJsonHandlerTest {return (ObjectNode) response.getEntity();}+ @ParameterizedTest+ @ValueSource(strings = {"PutRecord", "PutRecords"})+ void writesReportAnOpenDescendantAfterRepeatedSplits(String action) {+ createStream("test-stream");+ KinesisStream stream = service.describeStream("test-stream", REGION);+ service.splitShard("test-stream", FIRST_SHARD_ID,+ "170141183460469231731687303715884105728", REGION);+ service.splitShard("test-stream", stream.getShards().get(1).getShardId(),+ "85070591730234615865843651857942052864", REGION);+ service.splitShard("test-stream", stream.getShards().get(2).getShardId(),+ "255211775190703847597530955573826158592", REGION);++ ObjectNode request = MAPPER.createObjectNode().put("StreamName", "test-stream");+ ObjectNode record = "PutRecords".equals(action) ? request.putArray("Records").addObject() : request;+ record.put("PartitionKey", "b").put("Data", "AQ==");+ ObjectNode response = responseEntity(handler.handle(action, request, REGION));+ JsonNode result = "PutRecords".equals(action) ? response.path("Records").get(0) : response;++ assertEquals(stream.getShards().get(3).getShardId(), result.path("ShardId").asText());+ assertFalse(result.path("SequenceNumber").asText().isEmpty());+ if ("PutRecords".equals(action)) {+ assertEquals(0, response.path("FailedRecordCount").asInt());+ }+ assertEquals(1, stream.getShards().get(3).recordCount());+ assertTrue(stream.getShards().stream().filter(KinesisShard::isClosed)+ .allMatch(shard -> shard.recordCount() == 0));+ }++ @ParameterizedTest+ @ValueSource(strings = {"PutRecord", "PutRecords"})+ void writesFailWithoutAppendingWhenNoOpenDescendantExists(String action) {+ createStream("test-stream");+ KinesisShard shard = service.describeStream("test-stream", REGION).getShards().getFirst();+ shard.setClosed(true);+ ObjectNode request = MAPPER.createObjectNode().put("StreamName", "test-stream");+ ObjectNode record = "PutRecords".equals(action) ? request.putArray("Records").addObject() : request;+ record.put("PartitionKey", "b").put("Data", "AQ==");++ if ("PutRecord".equals(action)) {+ AwsException error = assertThrows(AwsException.class, () -> handler.handle(action, request, REGION));+ assertEquals("InvalidArgumentException", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ } else {+ ObjectNode response = responseEntity(handler.handle(action, request, REGION));+ assertEquals(1, response.path("FailedRecordCount").asInt());+ assertEquals("InternalFailure", response.path("Records").get(0).path("ErrorCode").asText());+ assertFalse(response.path("Records").get(0).has("SequenceNumber"));+ assertFalse(response.path("Records").get(0).has("ShardId"));+ }+ assertEquals(0, shard.recordCount());+ }+@Testvoid describeStreamByName() {createStream("test-stream");