Light, fluffy, and always free - The AWS Local Emulator alternative
fix(kinesis): paginate ListShards with an opaque NextToken (#4098)
Daniël van Ziel8e6faffcaffe
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.javaindex 6c3b82eee1..7600465763 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/KinesisTest.java@@ -15,7 +15,12 @@ import software.amazon.awssdk.services.kinesis.KinesisClient;import software.amazon.awssdk.services.kinesis.model.CreateStreamRequest;import software.amazon.awssdk.services.kinesis.model.DeleteStreamRequest;import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryRequest;+import software.amazon.awssdk.services.kinesis.model.ListShardsRequest;+import software.amazon.awssdk.services.kinesis.model.ListShardsResponse;+import java.util.ArrayList;+import java.util.HashSet;+import java.util.List;import java.util.concurrent.atomic.AtomicReference;import static org.assertj.core.api.Assertions.assertThat;@@ -83,4 +88,44 @@ class KinesisTest {.build()));}}++ @Test+ @DisplayName("ListShards paginates through an opaque NextToken")+ void listShardsPaginatesThroughEveryPage() {+ KinesisClient client = TestFixtures.kinesisClient();+ String streamName = TestFixtures.uniqueName("sdk-v2-kinesis-pagination");++ try {+ assertDoesNotThrow(() -> client.createStream(CreateStreamRequest.builder()+ .streamName(streamName)+ .shardCount(3)+ .build()));++ List<String> seen = new ArrayList<>();+ String nextToken = null;+ int pages = 0;+ do {+ ListShardsRequest.Builder request = ListShardsRequest.builder().maxResults(1);+ if (nextToken == null) {+ // AWS documents NextToken as the only stream identifier once paging has started.+ request.streamName(streamName);+ } else {+ request.nextToken(nextToken);+ }+ ListShardsResponse page = assertDoesNotThrow(() -> client.listShards(request.build()));+ page.shards().forEach(shard -> seen.add(shard.shardId()));+ nextToken = page.nextToken();+ pages++;+ } while (nextToken != null && pages < 10);++ assertThat(seen).hasSize(3);+ assertThat(new HashSet<>(seen)).hasSize(3);+ assertThat(nextToken).isNull();+ } finally {+ assertDoesNotThrow(() -> client.deleteStream(DeleteStreamRequest.builder()+ .streamName(streamName)+ .build()));+ client.close();+ }+ }}diff --git a/docs/services/kinesis.md b/docs/services/kinesis.mdindex e5066020f4..18c96fb3a2 100644--- a/docs/services/kinesis.md+++ b/docs/services/kinesis.md@@ -96,6 +96,7 @@ action, so that limit is intentionally unenforced here.| Variable | Default | Description ||---|---|---|| `FLOCI_SERVICES_KINESIS_ENABLED` | `true` | Enable or disable the service |+| `FLOCI_SERVICES_KINESIS_LIST_SHARDS_NEXT_TOKEN_TTL_MILLIS` | `300000` | Lifetime of a `ListShards` `NextToken` in milliseconds; AWS expires tokens after 300000 ms. Lower it to exercise `ExpiredNextTokenException` without waiting |## Enhanced Fan-Out (EFO)diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex b2d5372fc7..5ea41b1c97 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -1699,6 +1699,14 @@ public interface EmulatorConfig {interface KinesisServiceConfig {@WithDefault("true")boolean enabled();++ /**+ * Lifetime of a ListShards NextToken, in milliseconds. AWS expires these tokens 300000+ * milliseconds after they are issued; lowering it lets tests exercise the expiry path+ * without waiting.+ */+ @WithDefault("300000")+ long listShardsNextTokenTtlMillis();}interface FirehoseServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.javaindex 656a55e811..8f84cb43f3 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.kinesis;+import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsErrorResponse;import io.github.hectorvent.floci.core.common.AwsEventStreamEncoder;import io.github.hectorvent.floci.core.common.AwsException;@@ -35,11 +36,17 @@ public class KinesisJsonHandler {private final KinesisService service;private final ObjectMapper objectMapper;+ private final long listShardsNextTokenTtlMillis;@Inject- public KinesisJsonHandler(KinesisService service, ObjectMapper objectMapper) {+ public KinesisJsonHandler(KinesisService service, ObjectMapper objectMapper, EmulatorConfig config) {+ this(service, objectMapper, config.services().kinesis().listShardsNextTokenTtlMillis());+ }++ KinesisJsonHandler(KinesisService service, ObjectMapper objectMapper, long listShardsNextTokenTtlMillis) {this.service = service;this.objectMapper = objectMapper;+ this.listShardsNextTokenTtlMillis = listShardsNextTokenTtlMillis;}public Response handle(String action, JsonNode request, String region) {@@ -696,7 +703,12 @@ public class KinesisJsonHandler {}private Response handleListShards(JsonNode request, String region) {- String resolvedStreamName = resolveStreamName(request);+ String nextToken = request.hasNonNull("NextToken") ? request.path("NextToken").asText() : null;+ ShardToken token = nextToken == null ? null : decodeShardToken(nextToken);+ String resolvedStreamName = token == null+ ? resolveStreamName(request)+ : streamNameFromToken(request, token);+KinesisStream stream = service.describeStream(resolvedStreamName, region);List<KinesisShard> shards = stream.getShards();@@ -709,7 +721,14 @@ public class KinesisJsonHandler {}int maxResults = request.has("MaxResults") ? request.path("MaxResults").asInt(1000) : 1000;- List<KinesisShard> page = paginateShards(shards, maxResults);++ List<KinesisShard> snapshot = List.copyOf(shards);+ int start = token == null ? 0 : resumeShardIndex(token, snapshot);+ List<KinesisShard> page = paginateShards(snapshot.subList(start, snapshot.size()), maxResults);+ String nextCursor = start + page.size() < snapshot.size() && !page.isEmpty()+ ? encodeShardToken(resolvedStreamName, page.get(page.size() - 1).getShardId(),+ System.currentTimeMillis())+ : null;ObjectNode response = objectMapper.createObjectNode();ArrayNode shardsArray = response.putArray("Shards");@@ -732,11 +751,88 @@ public class KinesisJsonHandler {}}- response.putNull("NextToken");+ if (nextCursor != null) {+ response.put("NextToken", nextCursor);+ }return Response.ok(response).build();}+ /**+ * Opaque, stream-bound cursor for ListShards. It carries the stream name, the last shard id of+ * the emitted page, and the issue time, so a client can resume from the token alone without+ * resending the stream name and without depending on a shard's position in the live list.+ */+ static String encodeShardToken(String streamName, String lastShardId, long issuedAtEpochMilli) {+ String raw = streamName + "|" + lastShardId + "|" + issuedAtEpochMilli;+ return Base64.getUrlEncoder().withoutPadding()+ .encodeToString(raw.getBytes(StandardCharsets.UTF_8));+ }++ ShardToken decodeShardToken(String nextToken) {+ String decoded;+ try {+ decoded = new String(Base64.getUrlDecoder().decode(nextToken), StandardCharsets.UTF_8);+ } catch (IllegalArgumentException e) {+ throw invalidNextToken();+ }+ String[] parts = decoded.split("\\|", -1);+ if (parts.length != 3 || parts[0].isEmpty() || parts[1].isEmpty()) {+ throw invalidNextToken();+ }+ long issuedAtEpochMilli;+ try {+ issuedAtEpochMilli = Long.parseLong(parts[2]);+ } catch (NumberFormatException e) {+ throw invalidNextToken();+ }+ if (System.currentTimeMillis() - issuedAtEpochMilli >= listShardsNextTokenTtlMillis) {+ throw new AwsException("ExpiredNextTokenException", "The NextToken has expired.", 400);+ }+ return new ShardToken(parts[0], parts[1], issuedAtEpochMilli);+ }++ /**+ * The reference says the {@code NextToken} unambiguously identifies the stream and that+ * {@code StreamName} cannot be sent with it, so a lone token is the documented paging form. A+ * {@code StreamARN} is tolerated when it names the same stream the token does, which keeps an+ * SDK that fills the ARN in for routing working, and rejected when it names a different one.+ */+ private String streamNameFromToken(JsonNode request, ShardToken token) {+ if (request.hasNonNull("StreamName")) {+ throw new AwsException("InvalidArgumentException",+ "StreamName cannot be specified together with NextToken.", 400);+ }+ if (request.hasNonNull("StreamARN")) {+ String arnStreamName = parseStreamNameFromArn(request.path("StreamARN").asText());+ if (!token.streamName().equals(arnStreamName)) {+ throw new AwsException("InvalidArgumentException",+ "The NextToken does not belong to the stream named by StreamARN.", 400);+ }+ }+ return token.streamName();+ }++ private static int resumeShardIndex(ShardToken token, List<KinesisShard> shards) {+ for (int i = 0; i < shards.size(); i++) {+ if (shards.get(i).getShardId().equals(token.lastShardId())) {+ return i + 1;+ }+ }+ throw invalidNextToken();+ }++ private static AwsException invalidNextToken() {+ return new AwsException("InvalidArgumentException", "The NextToken is not valid.", 400);+ }++ /**+ * Decoded ListShards cursor: the stream it belongs to, the last shard id it covered, and when it+ * was issued (epoch millis, for expiry).+ */+ record ShardToken(String streamName, String lastShardId, long issuedAtEpochMilli) {+ }+/*** Take a ListShards page, snapshotting first. The shard list is a live* {@link java.util.concurrent.CopyOnWriteArrayList}: its {@code subList} view is NOT an independentdiff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 9270131bbf..e9dcbb1e86 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -447,6 +447,7 @@ floci:rotation-tick-seconds: 60kinesis:enabled: true+ list-shards-next-token-ttl-millis: 300000firehose:enabled: truetick-interval-seconds: 10diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.javaindex c09e467244..fc04e18d19 100644--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java@@ -2,8 +2,10 @@ package io.github.hectorvent.floci.services.kinesis;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.MethodOrderer;import org.junit.jupiter.api.Order;@@ -12,6 +14,9 @@ import org.junit.jupiter.api.TestMethodOrder;import java.math.BigDecimal;import java.nio.ByteBuffer;+import java.util.ArrayList;+import java.util.HashSet;+import java.util.List;import java.util.regex.Matcher;import java.util.regex.Pattern;@@ -1384,4 +1389,84 @@ class KinesisIntegrationTest {.statusCode(400).body("__type", equalTo("ResourceNotFoundException"));}++ private static void createStream(String name, int shardCount) {+ given()+ .header("X-Amz-Target", "Kinesis_20131202.CreateStream")+ .contentType(KINESIS_CONTENT_TYPE)+ .body("{\"StreamName\": \"" + name + "\", \"ShardCount\": " + shardCount + "}")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ private static Response listShardsPage(String streamName, Integer maxResults, String nextToken) {+ ObjectNode request = MAPPER.createObjectNode();+ if (streamName != null) {+ request.put("StreamName", streamName);+ }+ if (maxResults != null) {+ request.put("MaxResults", maxResults);+ }+ if (nextToken != null) {+ request.put("NextToken", nextToken);+ }+ return given()+ .header("X-Amz-Target", "Kinesis_20131202.ListShards")+ .contentType(KINESIS_CONTENT_TYPE)+ .body(request.toString())+ .when()+ .post("/")+ .then()+ .extract().response();+ }++ @Test+ @Order(68)+ void listShardsPaginatesWithNextTokenAndExhausts() {+ createStream("kinesis-pagination-test", 3);++ List<String> seen = new ArrayList<>();+ String nextToken = null;+ int pages = 0;+ do {+ // AWS documents NextToken as the only stream identifier once paging has started, so only+ // the first page names the stream.+ Response response = listShardsPage(pages == 0 ? "kinesis-pagination-test" : null, 1, nextToken);+ response.then().statusCode(200);+ seen.addAll(response.jsonPath().getList("Shards.ShardId", String.class));+ nextToken = response.jsonPath().getString("NextToken");+ pages++;+ } while (nextToken != null && pages < 10);++ assertEquals(3, seen.size(), "every shard is seen exactly once across pages");+ assertEquals(3, new HashSet<>(seen).size(), "no shard repeats across pages");+ assertNull(nextToken, "the final page must omit NextToken");+ }++ @Test+ @Order(69)+ void listShardsRejectsStreamNameTogetherWithNextToken() {+ createStream("kinesis-pagination-a", 2);++ Response firstPage = listShardsPage("kinesis-pagination-a", 1, null);+ firstPage.then().statusCode(200);+ String token = firstPage.jsonPath().getString("NextToken");+ assertNotNull(token, "a partial first page must carry a NextToken");++ listShardsPage("kinesis-pagination-a", 1, token)+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidArgumentException"));+ }++ @Test+ @Order(70)+ void listShardsRejectsAMalformedToken() {+ listShardsPage("list-shards-test", 1, "not-a-valid-token!")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidArgumentException"));+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.javaindex fc754f7dfa..5659acce54 100644--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java@@ -52,7 +52,7 @@ class KinesisJsonHandlerTest {new InMemoryStorage<>(),new RegionResolver(REGION, ACCOUNT));- handler = new KinesisJsonHandler(service, MAPPER);+ handler = new KinesisJsonHandler(service, MAPPER, 300_000);}private void createStream(String name) {@@ -1584,6 +1584,97 @@ class KinesisJsonHandlerTest {assertEquals(Map.of("Foo", "Bar"), service.listTagsForStream("test-stream", REGION));}+ @Test+ void listShardsResumesFromTokenAloneWithoutTheStreamName() {+ createStream("test-stream", 3);++ ObjectNode first = MAPPER.createObjectNode();+ first.put("StreamName", "test-stream");+ first.put("MaxResults", 1);+ ObjectNode firstPage = responseEntity(handler.handle("ListShards", first, REGION));+ String token = firstPage.get("NextToken").asText();+ String firstShardId = firstPage.get("Shards").get(0).get("ShardId").asText();++ ObjectNode second = MAPPER.createObjectNode();+ second.put("NextToken", token);+ ObjectNode secondPage = responseEntity(handler.handle("ListShards", second, REGION));++ assertEquals(2, secondPage.get("Shards").size(), "the token alone resumes after the first shard");+ for (JsonNode shard : secondPage.get("Shards")) {+ assertFalse(shard.get("ShardId").asText().equals(firstShardId));+ }+ }++ @Test+ void listShardsRejectsStreamNameTogetherWithNextToken() {+ createStream("test-stream", 3);++ ObjectNode first = MAPPER.createObjectNode();+ first.put("StreamName", "test-stream");+ first.put("MaxResults", 1);+ String token = responseEntity(handler.handle("ListShards", first, REGION)).get("NextToken").asText();++ ObjectNode both = MAPPER.createObjectNode();+ both.put("StreamName", "test-stream");+ both.put("NextToken", token);+ AwsException ex = assertThrows(AwsException.class,+ () -> handler.handle("ListShards", both, REGION));+ assertEquals("InvalidArgumentException", ex.getErrorCode());+ }++ @Test+ void listShardsAcceptsAStreamArnThatNamesTheTokenStream() {+ createStream("test-stream", 3);++ ObjectNode first = MAPPER.createObjectNode();+ first.put("StreamName", "test-stream");+ first.put("MaxResults", 1);+ String token = responseEntity(handler.handle("ListShards", first, REGION)).get("NextToken").asText();++ ObjectNode second = MAPPER.createObjectNode();+ second.put("NextToken", token);+ second.put("StreamARN", STREAM_ARN);+ ObjectNode secondPage = responseEntity(handler.handle("ListShards", second, REGION));++ assertEquals(2, secondPage.get("Shards").size());+ }++ @Test+ void listShardsRejectsAStreamArnFromAnotherStream() {+ createStream("test-stream", 3);++ ObjectNode first = MAPPER.createObjectNode();+ first.put("StreamName", "test-stream");+ first.put("MaxResults", 1);+ String token = responseEntity(handler.handle("ListShards", first, REGION)).get("NextToken").asText();++ ObjectNode second = MAPPER.createObjectNode();+ second.put("NextToken", token);+ second.put("StreamARN", "arn:aws:kinesis:us-east-1:123456789012:stream/other-stream");+ AwsException ex = assertThrows(AwsException.class,+ () -> handler.handle("ListShards", second, REGION));+ assertEquals("InvalidArgumentException", ex.getErrorCode());+ }++ @Test+ void listShardsRejectsAnExpiredNextToken() {+ KinesisJsonHandler zeroTtlHandler = new KinesisJsonHandler(service, MAPPER, 0);+ createStream("test-stream", 3);++ ObjectNode first = MAPPER.createObjectNode();+ first.put("StreamName", "test-stream");+ first.put("MaxResults", 1);+ ObjectNode firstPage = (ObjectNode) zeroTtlHandler.handle("ListShards", first, REGION).getEntity();+ String token = firstPage.get("NextToken").asText();++ ObjectNode second = MAPPER.createObjectNode();+ second.put("NextToken", token);+ AwsException ex = assertThrows(AwsException.class,+ () -> zeroTtlHandler.handle("ListShards", second, REGION));+ assertEquals("ExpiredNextTokenException", ex.getErrorCode());+ assertEquals(400, ex.getHttpStatus());+ }+/** Every non-string tag value shape — number, boolean, null, object, array — plus a map mixing a valid and an invalid value. */private static List<ObjectNode> nonStringTagValues() {ObjectNode number = MAPPER.createObjectNode();diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisListShardsTokenExpiryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisListShardsTokenExpiryIntegrationTest.javanew file mode 100644index 0000000000..8ab143cfb5--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisListShardsTokenExpiryIntegrationTest.java@@ -0,0 +1,79 @@+package io.github.hectorvent.floci.services.kinesis;++import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import io.restassured.response.Response;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.util.Map;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;++/**+ * Proves the ListShards NextToken expiry is driven by the+ * {@code floci.services.kinesis.list-shards-next-token-ttl-millis} setting: with the TTL lowered to+ * zero, the second page is rejected instead of waiting out AWS's 300 second window.+ */+@QuarkusTest+@TestProfile(KinesisListShardsTokenExpiryIntegrationTest.ZeroTtlProfile.class)+class KinesisListShardsTokenExpiryIntegrationTest {++ private static final String KINESIS_CONTENT_TYPE = "application/x-amz-json-1.1";+ private static final ObjectMapper MAPPER = new ObjectMapper();++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void listShardsRejectsAnExpiredNextToken() {+ given()+ .header("X-Amz-Target", "Kinesis_20131202.CreateStream")+ .contentType(KINESIS_CONTENT_TYPE)+ .body("{\"StreamName\": \"kinesis-expiry-test\", \"ShardCount\": 2}")+ .when()+ .post("/")+ .then()+ .statusCode(200);++ ObjectNode first = MAPPER.createObjectNode();+ first.put("StreamName", "kinesis-expiry-test");+ first.put("MaxResults", 1);+ Response firstPage = given()+ .header("X-Amz-Target", "Kinesis_20131202.ListShards")+ .contentType(KINESIS_CONTENT_TYPE)+ .body(first.toString())+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().response();+ String token = firstPage.jsonPath().getString("NextToken");++ ObjectNode second = MAPPER.createObjectNode();+ second.put("NextToken", token);+ given()+ .header("X-Amz-Target", "Kinesis_20131202.ListShards")+ .contentType(KINESIS_CONTENT_TYPE)+ .body(second.toString())+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ExpiredNextTokenException"));+ }++ public static final class ZeroTtlProfile implements QuarkusTestProfile {+ @Override+ public Map<String, String> getConfigOverrides() {+ return Map.of("floci.services.kinesis.list-shards-next-token-ttl-millis", "0");+ }+ }+}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 44bf251d09..c264c2014f 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -235,6 +235,7 @@ floci:default-recovery-window-days: 30kinesis:enabled: true+ list-shards-next-token-ttl-millis: 300000firehose:enabled: truetick-interval-seconds: 10