Light, fluffy, and always free - The AWS Local Emulator alternative
fix(sqs): page ListQueues with MaxResults and NextToken (#6222)
Mohithf74d7a34a32c
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.javaindex 825d169449..4e8faf2474 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java@@ -390,4 +390,42 @@ class SqsTest {queueUrl = null;dlqUrl = null;}++ @Test+ @Order(22)+ void listQueuesPaginatorVisitsEveryQueueOnce() {+ List<String> created = new ArrayList<>();+ try {+ for (String suffix : List.of("a", "b", "c")) {+ created.add(sqs.createQueue(CreateQueueRequest.builder()+ .queueName("sdk-list-paging-" + suffix)+ .build()).queueUrl());+ }++ List<ListQueuesResponse> pages = new ArrayList<>();+ sqs.listQueuesPaginator(ListQueuesRequest.builder()+ .queueNamePrefix("sdk-list-paging-")+ .maxResults(1)+ .build()).forEach(pages::add);++ assertThat(pages).hasSize(3);+ assertThat(pages).allSatisfy(p -> assertThat(p.queueUrls()).hasSize(1));+ assertThat(pages.get(2).nextToken()).isNull();+ assertThat(pages.stream().flatMap(p -> p.queueUrls().stream()).toList())+ .containsExactlyElementsOf(created);++ assertThatThrownBy(() -> sqs.listQueues(ListQueuesRequest.builder().maxResults(1001).build()))+ .isInstanceOf(SqsException.class)+ .satisfies(e -> assertThat(((SqsException) e).awsErrorDetails().errorCode())+ .isEqualTo("InvalidParameterValue"));+ } finally {+ for (String url : created) {+ try {+ sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(url).build());+ } catch (QueueDoesNotExistException ignored) {+ // Already gone, which is the state cleanup wants; any other failure propagates.+ }+ }+ }+ }}diff --git a/docs/services/sqs.md b/docs/services/sqs.mdindex ac04cf8696..239cf7c295 100644--- a/docs/services/sqs.md+++ b/docs/services/sqs.md@@ -10,7 +10,7 @@| --- | --- || `CreateQueue` | Create a standard or FIFO queue || `DeleteQueue` | Delete a queue |-| `ListQueues` | List all queues |+| `ListQueues` | List queues, with `QueueNamePrefix` filtering and `MaxResults`/`NextToken` paging || `GetQueueUrl` | Look up a queue URL by name || `GetQueueAttributes` | Get queue configuration attributes || `SendMessage` | Send a message to a queue |diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.javaindex 31e06802a6..02d36d7e5f 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.common.AwsErrorResponse;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.PaginatedResult;import io.github.hectorvent.floci.services.sqs.model.Message;import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;import io.github.hectorvent.floci.services.sqs.model.Queue;@@ -151,12 +152,17 @@ public class SqsJsonHandler {private Response handleListQueues(JsonNode request, String region) {String prefix = request.path("QueueNamePrefix").asText(null);- List<Queue> queues = sqsService.listQueues(prefix, region);+ Integer maxResults = getOptionalIntField(request, "MaxResults");+ String nextToken = request.path("NextToken").asText(null);+ PaginatedResult<Queue> page = sqsService.listQueues(prefix, maxResults, nextToken, region);ObjectNode response = objectMapper.createObjectNode();ArrayNode queueUrls = response.putArray("QueueUrls");- for (Queue q : queues) {+ for (Queue q : page.items()) {queueUrls.add(q.getQueueUrl());}+ if (page.nextToken() != null) {+ response.put("NextToken", page.nextToken());+ }return Response.ok(response).build();}diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.javaindex 4777d52cbf..9fea7c15ff 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.core.common.AwsErrorMessages;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.AwsNamespaces;import io.github.hectorvent.floci.core.common.AwsQueryResponse;+import io.github.hectorvent.floci.core.common.PaginatedResult;import io.github.hectorvent.floci.core.common.XmlBuilder;import io.github.hectorvent.floci.services.sqs.model.Message;import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;@@ -173,12 +174,17 @@ public class SqsQueryHandler {private Response handleListQueues(MultivaluedMap<String, String> params, String region) {String prefix = getParam(params, "QueueNamePrefix");- List<Queue> queues = sqsService.listQueues(prefix, region);+ Integer maxResults = getOptionalIntParam(params, "MaxResults");+ String nextToken = getParam(params, "NextToken");+ PaginatedResult<Queue> page = sqsService.listQueues(prefix, maxResults, nextToken, region);XmlBuilder xml = new XmlBuilder();- for (Queue q : queues) {+ for (Queue q : page.items()) {xml.elem("QueueUrl", q.getQueueUrl());}+ if (page.nextToken() != null) {+ xml.elem("NextToken", page.nextToken());+ }return Response.ok(AwsQueryResponse.envelope("ListQueues", null, xml.build())).build();}diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.javaindex 35264d4c94..e373fb381a 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java@@ -12,6 +12,8 @@ import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsEndpoints;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.AwsRegions;+import io.github.hectorvent.floci.core.common.PaginatedResult;+import io.github.hectorvent.floci.core.common.Pagination;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.common.RequestContext;import io.github.hectorvent.floci.core.common.Resettable;@@ -67,6 +69,8 @@ public class SqsService implements Resettable, ResourceProvider {private static final int DEDUP_LOCK_STRIPES = 256;private static final int MAX_RECEIVE_WAIT_TIME_SECONDS = 20;private static final int MAX_BATCH_ENTRIES = 10;+ private static final int MAX_LIST_QUEUES_RESULTS = 1000;+ private static final String LIST_QUEUES_TOKEN_NAMESPACE = "sqs-list-queues";private static final int MAX_TERMINAL_MOVE_TASKS = 10;private static final Duration TERMINAL_MOVE_TASK_TTL = Duration.ofHours(1);/** AWS accepts MaximumMessageSize between 1 KiB and 1 MiB; 1 MiB is also the default.@@ -851,6 +855,28 @@ public class SqsService implements Resettable, ResourceProvider {return queues;}+ /**+ * ListQueues with AWS paging: without {@code maxResults} the response holds at most 1,000+ * queues and never a token; with it, a token is returned only while more queues remain.+ * Queues are ordered by name so consecutive pages neither skip nor repeat a queue.+ */+ public PaginatedResult<Queue> listQueues(String namePrefix, Integer maxResults, String nextToken,+ String region) {+ if (maxResults != null && (maxResults < 1 || maxResults > MAX_LIST_QUEUES_RESULTS)) {+ throw new AwsException("InvalidParameterValue",+ "Value for parameter MaxResults is invalid. Reason: MaxResults must be an integer between 1 and "+ + MAX_LIST_QUEUES_RESULTS + ".", 400);+ }+ int limit = maxResults != null ? maxResults : MAX_LIST_QUEUES_RESULTS;+ PaginatedResult<Queue> page = Pagination.paginate(listQueues(namePrefix, region), Queue::getQueueName,+ limit, nextToken, LIST_QUEUES_TOKEN_NAMESPACE,+ token -> new AwsException("InvalidParameterValue", "Invalid NextToken value.", 400));+ if (maxResults == null) {+ return new PaginatedResult<>(page.items(), null);+ }+ return page;+ }+public String getQueueUrl(String queueName, String region) {String accountId = regionResolver.getAccountId();String queueUrl = buildQueueUrl(region, accountId, queueName);diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsListQueuesPaginationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsListQueuesPaginationIntegrationTest.javanew file mode 100644index 0000000000..ac204947f2--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsListQueuesPaginationIntegrationTest.java@@ -0,0 +1,189 @@+package io.github.hectorvent.floci.services.sqs;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.path.json.JsonPath;+import io.restassured.path.xml.XmlPath;+import io.restassured.response.Response;+import io.restassured.specification.RequestSpecification;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.ArrayList;+import java.util.List;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasKey;+import static org.hamcrest.Matchers.not;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;++@QuarkusTest+class SqsListQueuesPaginationIntegrationTest {++ private static final String JSON_CONTENT_TYPE = "application/x-amz-json-1.0";+ private static final String FORM_CONTENT_TYPE = "application/x-www-form-urlencoded";+ private static final String PREFIX = "list-paging-";+ private static final List<String> QUEUE_NAMES =+ List.of(PREFIX + "a", PREFIX + "b", PREFIX + "c");++ private final List<String> queueUrls = new ArrayList<>();+ private String otherQueueUrl;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @BeforeEach+ void createQueues() {+ otherQueueUrl = given().contentType(FORM_CONTENT_TYPE)+ .formParam("Action", "CreateQueue")+ .formParam("QueueName", "other-list-paging")+ .post("/").then().statusCode(200)+ .extract().xmlPath().getString("CreateQueueResponse.CreateQueueResult.QueueUrl");+ for (String name : QUEUE_NAMES) {+ queueUrls.add(given().contentType(FORM_CONTENT_TYPE)+ .formParam("Action", "CreateQueue")+ .formParam("QueueName", name)+ .post("/").then().statusCode(200)+ .extract().xmlPath().getString("CreateQueueResponse.CreateQueueResult.QueueUrl"));+ }+ }++ @AfterEach+ void deleteQueues() {+ List<String> all = new ArrayList<>(queueUrls);+ all.add(otherQueueUrl);+ for (String url : all) {+ given().contentType(FORM_CONTENT_TYPE)+ .formParam("Action", "DeleteQueue")+ .formParam("QueueUrl", url)+ .post("/");+ }+ }++ @Test+ void queryProtocolPagesThroughEveryQueueOnce() {+ List<String> collected = new ArrayList<>();+ String token = null;+ do {+ XmlPath xml = queryListQueues("1", token).then().statusCode(200).extract().xmlPath();+ List<String> page = xml.getList("ListQueuesResponse.ListQueuesResult.QueueUrl");+ assertEquals(1, page.size());+ collected.addAll(page);+ token = xml.getString("ListQueuesResponse.ListQueuesResult.NextToken");+ if (token != null && token.isEmpty()) {+ token = null;+ }+ } while (token != null);++ assertEquals(queueUrls, collected);+ }++ @Test+ void queryProtocolOmitsNextTokenWithoutMaxResults() {+ queryListQueues(null, null)+ .then()+ .statusCode(200)+ .body(not(containsString("<NextToken>")))+ .body(containsString(PREFIX + "c"));+ }++ @Test+ void queryProtocolRejectsOutOfRangeMaxResults() {+ queryListQueues("1001", null)+ .then()+ .statusCode(400)+ .body(containsString("<Code>InvalidParameterValue</Code>"));+ queryListQueues("0", null)+ .then()+ .statusCode(400)+ .body(containsString("<Code>InvalidParameterValue</Code>"));+ }++ @Test+ void queryProtocolRejectsInvalidNextToken() {+ queryListQueues("1", "garbage-token!")+ .then()+ .statusCode(400)+ .body(containsString("<Code>InvalidParameterValue</Code>"));+ }++ @Test+ void jsonProtocolPagesThroughEveryQueueOnce() {+ List<String> collected = new ArrayList<>();+ String token = null;+ do {+ String body = "{\"QueueNamePrefix\":\"" + PREFIX + "\",\"MaxResults\":1"+ + (token != null ? ",\"NextToken\":\"" + token + "\"" : "") + "}";+ JsonPath json = jsonListQueues(body).then().statusCode(200).extract().jsonPath();+ List<String> page = json.getList("QueueUrls");+ assertEquals(1, page.size());+ collected.addAll(page);+ token = json.getString("NextToken");+ } while (token != null);++ assertEquals(queueUrls, collected);+ }++ @Test+ void jsonProtocolReturnsNextTokenOnlyWhileMoreQueuesRemain() {+ String first = jsonListQueues("{\"QueueNamePrefix\":\"" + PREFIX + "\",\"MaxResults\":2}")+ .then().statusCode(200).extract().jsonPath().getString("NextToken");+ assertNotNull(first);++ JsonPath last = jsonListQueues("{\"QueueNamePrefix\":\"" + PREFIX + "\",\"MaxResults\":2,"+ + "\"NextToken\":\"" + first + "\"}").then().statusCode(200).extract().jsonPath();+ assertEquals(List.of(queueUrls.get(2)), last.getList("QueueUrls"));+ assertNull(last.getString("NextToken"));++ jsonListQueues("{\"QueueNamePrefix\":\"" + PREFIX + "\"}")+ .then()+ .statusCode(200)+ .body("$", not(hasKey("NextToken")));+ }++ @Test+ void jsonProtocolRejectsOutOfRangeMaxResults() {+ jsonListQueues("{\"MaxResults\":1001}")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidParameterValue"));+ }++ @Test+ void jsonProtocolRejectsInvalidNextToken() {+ jsonListQueues("{\"MaxResults\":1,\"NextToken\":\"garbage-token!\"}")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidParameterValue"));+ }++ private static Response queryListQueues(String maxResults, String nextToken) {+ RequestSpecification spec = given()+ .contentType(FORM_CONTENT_TYPE)+ .formParam("Action", "ListQueues")+ .formParam("QueueNamePrefix", PREFIX);+ if (maxResults != null) {+ spec.formParam("MaxResults", maxResults);+ }+ if (nextToken != null) {+ spec.formParam("NextToken", nextToken);+ }+ return spec.post("/");+ }++ private static Response jsonListQueues(String body) {+ return given()+ .contentType(JSON_CONTENT_TYPE)+ .header("X-Amz-Target", "AmazonSQS.ListQueues")+ .body(body)+ .post("/");+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.javaindex 1a7196207b..0a67cbad1b 100644--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.sqs;import io.github.hectorvent.floci.config.EmulatorConfig.SqsServiceConfig.EndpointStrategy;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.PaginatedResult;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.common.RequestContext;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;@@ -385,6 +386,120 @@ class SqsServiceTest {assertEquals(2, alpha.size());}+ @Test+ void listQueuesPagesEveryQueueExactlyOnce() {+ String region = "eu-west-1";+ sqsService.createQueue("page-c", null, region);+ sqsService.createQueue("page-a", null, region);+ sqsService.createQueue("page-b", null, region);++ List<String> names = new ArrayList<>();+ String token = null;+ int pages = 0;+ do {+ PaginatedResult<Queue> page = sqsService.listQueues(null, 1, token, region);+ assertEquals(1, page.items().size());+ names.add(page.items().getFirst().getQueueName());+ token = page.nextToken();+ pages++;+ } while (token != null);++ assertEquals(3, pages);+ assertEquals(List.of("page-a", "page-b", "page-c"), names);+ }++ @Test+ void listQueuesOmitsNextTokenOnLastPage() {+ String region = "eu-west-1";+ sqsService.createQueue("last-a", null, region);+ sqsService.createQueue("last-b", null, region);++ PaginatedResult<Queue> first = sqsService.listQueues(null, 1, null, region);+ assertNotNull(first.nextToken());+ PaginatedResult<Queue> second = sqsService.listQueues(null, 1, first.nextToken(), region);+ assertEquals("last-b", second.items().getFirst().getQueueName());+ assertNull(second.nextToken());++ assertNull(sqsService.listQueues(null, 2, null, region).nextToken());+ }++ @Test+ void listQueuesWithoutMaxResultsNeverReturnsNextToken() {+ String region = "eu-west-1";+ sqsService.createQueue("nomax-a", null, region);+ sqsService.createQueue("nomax-b", null, region);++ PaginatedResult<Queue> page = sqsService.listQueues(null, null, null, region);+ assertEquals(2, page.items().size());+ assertNull(page.nextToken());+ }++ @Test+ void listQueuesResumesAfterCursorQueueIsDeleted() {+ String region = "eu-west-1";+ sqsService.createQueue("gone-a", null, region);+ Queue cursorQueue = sqsService.createQueue("gone-b", null, region);+ sqsService.createQueue("gone-c", null, region);+ sqsService.createQueue("gone-d", null, region);++ PaginatedResult<Queue> first = sqsService.listQueues(null, 2, null, region);+ assertEquals(List.of("gone-a", "gone-b"), first.items().stream().map(Queue::getQueueName).toList());++ sqsService.deleteQueue(cursorQueue.getQueueUrl(), region);++ PaginatedResult<Queue> second = sqsService.listQueues(null, 2, first.nextToken(), region);+ assertEquals(List.of("gone-c", "gone-d"), second.items().stream().map(Queue::getQueueName).toList());+ assertNull(second.nextToken());+ }++ @Test+ void listQueuesWithoutMaxResultsCapsAtOneThousand() {+ String region = "eu-west-1";+ for (int i = 0; i < 1001; i++) {+ sqsService.createQueue(String.format("cap-%04d", i), null, region);+ }++ PaginatedResult<Queue> page = sqsService.listQueues(null, null, null, region);++ assertEquals(1000, page.items().size());+ assertEquals("cap-0000", page.items().getFirst().getQueueName());+ assertEquals("cap-0999", page.items().getLast().getQueueName());+ assertNull(page.nextToken());+ }++ @Test+ void listQueuesPagesWithinPrefix() {+ String region = "eu-west-1";+ sqsService.createQueue("pfx-one", null, region);+ sqsService.createQueue("other-queue", null, region);+ sqsService.createQueue("pfx-two", null, region);++ PaginatedResult<Queue> first = sqsService.listQueues("pfx-", 1, null, region);+ PaginatedResult<Queue> second = sqsService.listQueues("pfx-", 1, first.nextToken(), region);++ assertEquals("pfx-one", first.items().getFirst().getQueueName());+ assertEquals("pfx-two", second.items().getFirst().getQueueName());+ assertNull(second.nextToken());+ }++ @ParameterizedTest+ @ValueSource(ints = {0, -1, 1001})+ void listQueuesRejectsOutOfRangeMaxResults(int maxResults) {+ AwsException e = assertThrows(AwsException.class,+ () -> sqsService.listQueues(null, maxResults, null, "eu-west-1"));+ assertEquals("InvalidParameterValue", e.getErrorCode());+ assertEquals(400, e.getHttpStatus());+ }++ @ParameterizedTest+ @ValueSource(strings = {"not-a-token!", "Z2FyYmFnZQ"})+ void listQueuesRejectsInvalidNextToken(String token) {+ sqsService.createQueue("token-queue", null, "eu-west-1");+ AwsException e = assertThrows(AwsException.class,+ () -> sqsService.listQueues(null, 1, token, "eu-west-1"));+ assertEquals("InvalidParameterValue", e.getErrorCode());+ }+@Testvoid getQueueUrl() {String region = "eu-west-1";