Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(sqs): page ListQueues with MaxResults and NextToken (#6222)

Mohithf74d7a34a32c
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java
index 825d169449..4e8faf2474 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java
@@ -390,4 +390,42 @@ class SqsTest {
queueUrl = null;
dlqUrl = null;
}
+
+ @Test
+ @Order(22)
+ void listQueuesPaginatorVisitsEveryQueueOnce() {
+ List<String> created = new ArrayList<>();
+ try {
+ for (String suffix : List.of("a", "b", "c")) {
+ created.add(sqs.createQueue(CreateQueueRequest.builder()
+ .queueName("sdk-list-paging-" + suffix)
+ .build()).queueUrl());
+ }
+
+ List<ListQueuesResponse> pages = new ArrayList<>();
+ sqs.listQueuesPaginator(ListQueuesRequest.builder()
+ .queueNamePrefix("sdk-list-paging-")
+ .maxResults(1)
+ .build()).forEach(pages::add);
+
+ assertThat(pages).hasSize(3);
+ assertThat(pages).allSatisfy(p -> assertThat(p.queueUrls()).hasSize(1));
+ assertThat(pages.get(2).nextToken()).isNull();
+ assertThat(pages.stream().flatMap(p -> p.queueUrls().stream()).toList())
+ .containsExactlyElementsOf(created);
+
+ assertThatThrownBy(() -> sqs.listQueues(ListQueuesRequest.builder().maxResults(1001).build()))
+ .isInstanceOf(SqsException.class)
+ .satisfies(e -> assertThat(((SqsException) e).awsErrorDetails().errorCode())
+ .isEqualTo("InvalidParameterValue"));
+ } finally {
+ for (String url : created) {
+ try {
+ sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(url).build());
+ } catch (QueueDoesNotExistException ignored) {
+ // Already gone, which is the state cleanup wants; any other failure propagates.
+ }
+ }
+ }
+ }
}
diff --git a/docs/services/sqs.md b/docs/services/sqs.md
index ac04cf8696..239cf7c295 100644
--- a/docs/services/sqs.md
+++ b/docs/services/sqs.md
@@ -10,7 +10,7 @@
| --- | --- |
| `CreateQueue` | Create a standard or FIFO queue |
| `DeleteQueue` | Delete a queue |
-| `ListQueues` | List all queues |
+| `ListQueues` | List queues, with `QueueNamePrefix` filtering and `MaxResults`/`NextToken` paging |
| `GetQueueUrl` | Look up a queue URL by name |
| `GetQueueAttributes` | Get queue configuration attributes |
| `SendMessage` | Send a message to a queue |
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java
index 31e06802a6..02d36d7e5f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsErrorResponse;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.PaginatedResult;
import io.github.hectorvent.floci.services.sqs.model.Message;
import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;
import io.github.hectorvent.floci.services.sqs.model.Queue;
@@ -151,12 +152,17 @@ public class SqsJsonHandler {
private Response handleListQueues(JsonNode request, String region) {
String prefix = request.path("QueueNamePrefix").asText(null);
- List<Queue> queues = sqsService.listQueues(prefix, region);
+ Integer maxResults = getOptionalIntField(request, "MaxResults");
+ String nextToken = request.path("NextToken").asText(null);
+ PaginatedResult<Queue> page = sqsService.listQueues(prefix, maxResults, nextToken, region);
ObjectNode response = objectMapper.createObjectNode();
ArrayNode queueUrls = response.putArray("QueueUrls");
- for (Queue q : queues) {
+ for (Queue q : page.items()) {
queueUrls.add(q.getQueueUrl());
}
+ if (page.nextToken() != null) {
+ response.put("NextToken", page.nextToken());
+ }
return Response.ok(response).build();
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java
index 4777d52cbf..9fea7c15ff 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java
@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.core.common.AwsErrorMessages;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.AwsNamespaces;
import io.github.hectorvent.floci.core.common.AwsQueryResponse;
+import io.github.hectorvent.floci.core.common.PaginatedResult;
import io.github.hectorvent.floci.core.common.XmlBuilder;
import io.github.hectorvent.floci.services.sqs.model.Message;
import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;
@@ -173,12 +174,17 @@ public class SqsQueryHandler {
private Response handleListQueues(MultivaluedMap<String, String> params, String region) {
String prefix = getParam(params, "QueueNamePrefix");
- List<Queue> queues = sqsService.listQueues(prefix, region);
+ Integer maxResults = getOptionalIntParam(params, "MaxResults");
+ String nextToken = getParam(params, "NextToken");
+ PaginatedResult<Queue> page = sqsService.listQueues(prefix, maxResults, nextToken, region);
XmlBuilder xml = new XmlBuilder();
- for (Queue q : queues) {
+ for (Queue q : page.items()) {
xml.elem("QueueUrl", q.getQueueUrl());
}
+ if (page.nextToken() != null) {
+ xml.elem("NextToken", page.nextToken());
+ }
return Response.ok(AwsQueryResponse.envelope("ListQueues", null, xml.build())).build();
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
index 35264d4c94..e373fb381a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
@@ -12,6 +12,8 @@ import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsEndpoints;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.AwsRegions;
+import io.github.hectorvent.floci.core.common.PaginatedResult;
+import io.github.hectorvent.floci.core.common.Pagination;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.RequestContext;
import io.github.hectorvent.floci.core.common.Resettable;
@@ -67,6 +69,8 @@ public class SqsService implements Resettable, ResourceProvider {
private static final int DEDUP_LOCK_STRIPES = 256;
private static final int MAX_RECEIVE_WAIT_TIME_SECONDS = 20;
private static final int MAX_BATCH_ENTRIES = 10;
+ private static final int MAX_LIST_QUEUES_RESULTS = 1000;
+ private static final String LIST_QUEUES_TOKEN_NAMESPACE = "sqs-list-queues";
private static final int MAX_TERMINAL_MOVE_TASKS = 10;
private static final Duration TERMINAL_MOVE_TASK_TTL = Duration.ofHours(1);
/** AWS accepts MaximumMessageSize between 1 KiB and 1 MiB; 1 MiB is also the default.
@@ -851,6 +855,28 @@ public class SqsService implements Resettable, ResourceProvider {
return queues;
}
+ /**
+ * ListQueues with AWS paging: without {@code maxResults} the response holds at most 1,000
+ * queues and never a token; with it, a token is returned only while more queues remain.
+ * Queues are ordered by name so consecutive pages neither skip nor repeat a queue.
+ */
+ public PaginatedResult<Queue> listQueues(String namePrefix, Integer maxResults, String nextToken,
+ String region) {
+ if (maxResults != null && (maxResults < 1 || maxResults > MAX_LIST_QUEUES_RESULTS)) {
+ throw new AwsException("InvalidParameterValue",
+ "Value for parameter MaxResults is invalid. Reason: MaxResults must be an integer between 1 and "
+ + MAX_LIST_QUEUES_RESULTS + ".", 400);
+ }
+ int limit = maxResults != null ? maxResults : MAX_LIST_QUEUES_RESULTS;
+ PaginatedResult<Queue> page = Pagination.paginate(listQueues(namePrefix, region), Queue::getQueueName,
+ limit, nextToken, LIST_QUEUES_TOKEN_NAMESPACE,
+ token -> new AwsException("InvalidParameterValue", "Invalid NextToken value.", 400));
+ if (maxResults == null) {
+ return new PaginatedResult<>(page.items(), null);
+ }
+ return page;
+ }
+
public String getQueueUrl(String queueName, String region) {
String accountId = regionResolver.getAccountId();
String queueUrl = buildQueueUrl(region, accountId, queueName);
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsListQueuesPaginationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsListQueuesPaginationIntegrationTest.java
new file mode 100644
index 0000000000..ac204947f2
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsListQueuesPaginationIntegrationTest.java
@@ -0,0 +1,189 @@
+package io.github.hectorvent.floci.services.sqs;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.path.json.JsonPath;
+import io.restassured.path.xml.XmlPath;
+import io.restassured.response.Response;
+import io.restassured.specification.RequestSpecification;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+@QuarkusTest
+class SqsListQueuesPaginationIntegrationTest {
+
+ private static final String JSON_CONTENT_TYPE = "application/x-amz-json-1.0";
+ private static final String FORM_CONTENT_TYPE = "application/x-www-form-urlencoded";
+ private static final String PREFIX = "list-paging-";
+ private static final List<String> QUEUE_NAMES =
+ List.of(PREFIX + "a", PREFIX + "b", PREFIX + "c");
+
+ private final List<String> queueUrls = new ArrayList<>();
+ private String otherQueueUrl;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @BeforeEach
+ void createQueues() {
+ otherQueueUrl = given().contentType(FORM_CONTENT_TYPE)
+ .formParam("Action", "CreateQueue")
+ .formParam("QueueName", "other-list-paging")
+ .post("/").then().statusCode(200)
+ .extract().xmlPath().getString("CreateQueueResponse.CreateQueueResult.QueueUrl");
+ for (String name : QUEUE_NAMES) {
+ queueUrls.add(given().contentType(FORM_CONTENT_TYPE)
+ .formParam("Action", "CreateQueue")
+ .formParam("QueueName", name)
+ .post("/").then().statusCode(200)
+ .extract().xmlPath().getString("CreateQueueResponse.CreateQueueResult.QueueUrl"));
+ }
+ }
+
+ @AfterEach
+ void deleteQueues() {
+ List<String> all = new ArrayList<>(queueUrls);
+ all.add(otherQueueUrl);
+ for (String url : all) {
+ given().contentType(FORM_CONTENT_TYPE)
+ .formParam("Action", "DeleteQueue")
+ .formParam("QueueUrl", url)
+ .post("/");
+ }
+ }
+
+ @Test
+ void queryProtocolPagesThroughEveryQueueOnce() {
+ List<String> collected = new ArrayList<>();
+ String token = null;
+ do {
+ XmlPath xml = queryListQueues("1", token).then().statusCode(200).extract().xmlPath();
+ List<String> page = xml.getList("ListQueuesResponse.ListQueuesResult.QueueUrl");
+ assertEquals(1, page.size());
+ collected.addAll(page);
+ token = xml.getString("ListQueuesResponse.ListQueuesResult.NextToken");
+ if (token != null && token.isEmpty()) {
+ token = null;
+ }
+ } while (token != null);
+
+ assertEquals(queueUrls, collected);
+ }
+
+ @Test
+ void queryProtocolOmitsNextTokenWithoutMaxResults() {
+ queryListQueues(null, null)
+ .then()
+ .statusCode(200)
+ .body(not(containsString("<NextToken>")))
+ .body(containsString(PREFIX + "c"));
+ }
+
+ @Test
+ void queryProtocolRejectsOutOfRangeMaxResults() {
+ queryListQueues("1001", null)
+ .then()
+ .statusCode(400)
+ .body(containsString("<Code>InvalidParameterValue</Code>"));
+ queryListQueues("0", null)
+ .then()
+ .statusCode(400)
+ .body(containsString("<Code>InvalidParameterValue</Code>"));
+ }
+
+ @Test
+ void queryProtocolRejectsInvalidNextToken() {
+ queryListQueues("1", "garbage-token!")
+ .then()
+ .statusCode(400)
+ .body(containsString("<Code>InvalidParameterValue</Code>"));
+ }
+
+ @Test
+ void jsonProtocolPagesThroughEveryQueueOnce() {
+ List<String> collected = new ArrayList<>();
+ String token = null;
+ do {
+ String body = "{\"QueueNamePrefix\":\"" + PREFIX + "\",\"MaxResults\":1"
+ + (token != null ? ",\"NextToken\":\"" + token + "\"" : "") + "}";
+ JsonPath json = jsonListQueues(body).then().statusCode(200).extract().jsonPath();
+ List<String> page = json.getList("QueueUrls");
+ assertEquals(1, page.size());
+ collected.addAll(page);
+ token = json.getString("NextToken");
+ } while (token != null);
+
+ assertEquals(queueUrls, collected);
+ }
+
+ @Test
+ void jsonProtocolReturnsNextTokenOnlyWhileMoreQueuesRemain() {
+ String first = jsonListQueues("{\"QueueNamePrefix\":\"" + PREFIX + "\",\"MaxResults\":2}")
+ .then().statusCode(200).extract().jsonPath().getString("NextToken");
+ assertNotNull(first);
+
+ JsonPath last = jsonListQueues("{\"QueueNamePrefix\":\"" + PREFIX + "\",\"MaxResults\":2,"
+ + "\"NextToken\":\"" + first + "\"}").then().statusCode(200).extract().jsonPath();
+ assertEquals(List.of(queueUrls.get(2)), last.getList("QueueUrls"));
+ assertNull(last.getString("NextToken"));
+
+ jsonListQueues("{\"QueueNamePrefix\":\"" + PREFIX + "\"}")
+ .then()
+ .statusCode(200)
+ .body("$", not(hasKey("NextToken")));
+ }
+
+ @Test
+ void jsonProtocolRejectsOutOfRangeMaxResults() {
+ jsonListQueues("{\"MaxResults\":1001}")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidParameterValue"));
+ }
+
+ @Test
+ void jsonProtocolRejectsInvalidNextToken() {
+ jsonListQueues("{\"MaxResults\":1,\"NextToken\":\"garbage-token!\"}")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidParameterValue"));
+ }
+
+ private static Response queryListQueues(String maxResults, String nextToken) {
+ RequestSpecification spec = given()
+ .contentType(FORM_CONTENT_TYPE)
+ .formParam("Action", "ListQueues")
+ .formParam("QueueNamePrefix", PREFIX);
+ if (maxResults != null) {
+ spec.formParam("MaxResults", maxResults);
+ }
+ if (nextToken != null) {
+ spec.formParam("NextToken", nextToken);
+ }
+ return spec.post("/");
+ }
+
+ private static Response jsonListQueues(String body) {
+ return given()
+ .contentType(JSON_CONTENT_TYPE)
+ .header("X-Amz-Target", "AmazonSQS.ListQueues")
+ .body(body)
+ .post("/");
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
index 1a7196207b..0a67cbad1b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.sqs;
import io.github.hectorvent.floci.config.EmulatorConfig.SqsServiceConfig.EndpointStrategy;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.PaginatedResult;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.RequestContext;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
@@ -385,6 +386,120 @@ class SqsServiceTest {
assertEquals(2, alpha.size());
}
+ @Test
+ void listQueuesPagesEveryQueueExactlyOnce() {
+ String region = "eu-west-1";
+ sqsService.createQueue("page-c", null, region);
+ sqsService.createQueue("page-a", null, region);
+ sqsService.createQueue("page-b", null, region);
+
+ List<String> names = new ArrayList<>();
+ String token = null;
+ int pages = 0;
+ do {
+ PaginatedResult<Queue> page = sqsService.listQueues(null, 1, token, region);
+ assertEquals(1, page.items().size());
+ names.add(page.items().getFirst().getQueueName());
+ token = page.nextToken();
+ pages++;
+ } while (token != null);
+
+ assertEquals(3, pages);
+ assertEquals(List.of("page-a", "page-b", "page-c"), names);
+ }
+
+ @Test
+ void listQueuesOmitsNextTokenOnLastPage() {
+ String region = "eu-west-1";
+ sqsService.createQueue("last-a", null, region);
+ sqsService.createQueue("last-b", null, region);
+
+ PaginatedResult<Queue> first = sqsService.listQueues(null, 1, null, region);
+ assertNotNull(first.nextToken());
+ PaginatedResult<Queue> second = sqsService.listQueues(null, 1, first.nextToken(), region);
+ assertEquals("last-b", second.items().getFirst().getQueueName());
+ assertNull(second.nextToken());
+
+ assertNull(sqsService.listQueues(null, 2, null, region).nextToken());
+ }
+
+ @Test
+ void listQueuesWithoutMaxResultsNeverReturnsNextToken() {
+ String region = "eu-west-1";
+ sqsService.createQueue("nomax-a", null, region);
+ sqsService.createQueue("nomax-b", null, region);
+
+ PaginatedResult<Queue> page = sqsService.listQueues(null, null, null, region);
+ assertEquals(2, page.items().size());
+ assertNull(page.nextToken());
+ }
+
+ @Test
+ void listQueuesResumesAfterCursorQueueIsDeleted() {
+ String region = "eu-west-1";
+ sqsService.createQueue("gone-a", null, region);
+ Queue cursorQueue = sqsService.createQueue("gone-b", null, region);
+ sqsService.createQueue("gone-c", null, region);
+ sqsService.createQueue("gone-d", null, region);
+
+ PaginatedResult<Queue> first = sqsService.listQueues(null, 2, null, region);
+ assertEquals(List.of("gone-a", "gone-b"), first.items().stream().map(Queue::getQueueName).toList());
+
+ sqsService.deleteQueue(cursorQueue.getQueueUrl(), region);
+
+ PaginatedResult<Queue> second = sqsService.listQueues(null, 2, first.nextToken(), region);
+ assertEquals(List.of("gone-c", "gone-d"), second.items().stream().map(Queue::getQueueName).toList());
+ assertNull(second.nextToken());
+ }
+
+ @Test
+ void listQueuesWithoutMaxResultsCapsAtOneThousand() {
+ String region = "eu-west-1";
+ for (int i = 0; i < 1001; i++) {
+ sqsService.createQueue(String.format("cap-%04d", i), null, region);
+ }
+
+ PaginatedResult<Queue> page = sqsService.listQueues(null, null, null, region);
+
+ assertEquals(1000, page.items().size());
+ assertEquals("cap-0000", page.items().getFirst().getQueueName());
+ assertEquals("cap-0999", page.items().getLast().getQueueName());
+ assertNull(page.nextToken());
+ }
+
+ @Test
+ void listQueuesPagesWithinPrefix() {
+ String region = "eu-west-1";
+ sqsService.createQueue("pfx-one", null, region);
+ sqsService.createQueue("other-queue", null, region);
+ sqsService.createQueue("pfx-two", null, region);
+
+ PaginatedResult<Queue> first = sqsService.listQueues("pfx-", 1, null, region);
+ PaginatedResult<Queue> second = sqsService.listQueues("pfx-", 1, first.nextToken(), region);
+
+ assertEquals("pfx-one", first.items().getFirst().getQueueName());
+ assertEquals("pfx-two", second.items().getFirst().getQueueName());
+ assertNull(second.nextToken());
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {0, -1, 1001})
+ void listQueuesRejectsOutOfRangeMaxResults(int maxResults) {
+ AwsException e = assertThrows(AwsException.class,
+ () -> sqsService.listQueues(null, maxResults, null, "eu-west-1"));
+ assertEquals("InvalidParameterValue", e.getErrorCode());
+ assertEquals(400, e.getHttpStatus());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"not-a-token!", "Z2FyYmFnZQ"})
+ void listQueuesRejectsInvalidNextToken(String token) {
+ sqsService.createQueue("token-queue", null, "eu-west-1");
+ AwsException e = assertThrows(AwsException.class,
+ () -> sqsService.listQueues(null, 1, token, "eu-west-1"));
+ assertEquals("InvalidParameterValue", e.getErrorCode());
+ }
+
@Test
void getQueueUrl() {
String region = "eu-west-1";