Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(codebuild): paginate ListProjects responses (#6163)

Jimmy Liu Jiacc08eab2f7f7
diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildJsonHandler.java
index 90277f96d6..bde33874c4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildJsonHandler.java
@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.codebuild;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.codebuild.CodeBuildService.ProjectPage;
import io.github.hectorvent.floci.services.codebuild.model.Build;
import io.github.hectorvent.floci.services.codebuild.model.Project;
import io.github.hectorvent.floci.services.codebuild.model.ProjectArtifacts;
@@ -130,9 +131,20 @@ public class CodeBuildJsonHandler {
}
private Response listProjects(JsonNode req, String region) {
- List<String> names = service.listProjects(region,
- req.path("sortBy").asText(null), req.path("sortOrder").asText(null));
- return Response.ok(Map.of("projects", names)).build();
+ String sortBy = req.path("sortBy").asText(null);
+ String sortOrder = req.path("sortOrder").asText(null);
+ JsonNode tokenNode = req.get("nextToken");
+ if (tokenNode != null && !tokenNode.isTextual()) {
+ throw new AwsException("InvalidInputException", "Invalid nextToken", 400);
+ }
+ String nextToken = tokenNode == null ? null : tokenNode.asText();
+ ProjectPage page = service.listProjectsPage(region, sortBy, sortOrder, nextToken);
+ Map<String, Object> result = new HashMap<>();
+ result.put("projects", page.projects());
+ if (page.nextToken() != null) {
+ result.put("nextToken", page.nextToken());
+ }
+ return Response.ok(result).build();
}
private Response createReportGroup(JsonNode req, String region, String account) throws Exception {
diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java
index b2c01ed81d..0a36f5ad39 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java
@@ -1,9 +1,13 @@
package io.github.hectorvent.floci.services.codebuild;
import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.Base64Url;
import io.github.hectorvent.floci.core.storage.StorageBackedMap;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.codebuild.model.Build;
@@ -18,11 +22,14 @@ import jakarta.annotation.PostConstruct;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
+import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
+import java.util.Base64;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
@@ -32,6 +39,9 @@ import java.util.stream.Collectors;
@ApplicationScoped
public class CodeBuildService {
+ private static final int LIST_PROJECTS_PAGE_SIZE = 100;
+ private static final ObjectMapper PAGE_TOKEN_MAPPER = new ObjectMapper();
+
// key: region -> name -> project
private Map<String, Map<String, Project>> projects = new ConcurrentHashMap<>();
// key: region -> arn -> report group
@@ -254,6 +264,95 @@ public class CodeBuildService {
if (sortBy == null && sortOrder == null) {
return listProjects(region);
}
+ return listProjectRecords(region, sortBy, sortOrder).stream()
+ .map(Project::getName)
+ .toList();
+ }
+
+ public ProjectPage listProjectsPage(String region, String sortBy, String sortOrder, String nextToken) {
+ if (sortBy == null && sortOrder == null && nextToken == null) {
+ List<String> names = listProjects(region);
+ if (names.size() <= LIST_PROJECTS_PAGE_SIZE) {
+ return new ProjectPage(names, null);
+ }
+ }
+
+ List<Project> projects = listProjectRecords(region, sortBy, sortOrder);
+ int start = 0;
+ if (nextToken != null) {
+ Project anchor = decodeListProjectsToken(nextToken, region, sortBy, sortOrder);
+ Comparator<Project> comparator = projectComparator(sortBy, sortOrder);
+ while (start < projects.size() && comparator.compare(projects.get(start), anchor) <= 0) {
+ start++;
+ }
+ }
+
+ int end = Math.min(start + LIST_PROJECTS_PAGE_SIZE, projects.size());
+ List<String> names = projects.subList(start, end).stream().map(Project::getName).toList();
+ String resultToken = end < projects.size()
+ ? encodeListProjectsToken(projects.get(end - 1), region, sortBy, sortOrder)
+ : null;
+ return new ProjectPage(names, resultToken);
+ }
+
+ private Project decodeListProjectsToken(String token, String region, String sortBy, String sortOrder) {
+ if (token.isEmpty()) {
+ throw new AwsException("InvalidInputException", "Invalid nextToken", 400);
+ }
+ JsonNode cursor;
+ try {
+ cursor = PAGE_TOKEN_MAPPER.readTree(Base64.getUrlDecoder().decode(token));
+ } catch (IllegalArgumentException | IOException e) {
+ throw new AwsException("InvalidInputException", "Invalid nextToken", 400);
+ }
+ JsonNode lastName = cursor == null ? null : cursor.path("lastName");
+ JsonNode sortValue = cursor == null ? null : cursor.path("sortValue");
+ if (cursor == null || !"ListProjects".equals(cursor.path("action").asText(null))
+ || !region.equals(cursor.path("region").asText(null))
+ || !cursor.has("sortBy") || !Objects.equals(sortBy, cursor.path("sortBy").asText(null))
+ || !cursor.has("sortOrder") || !Objects.equals(sortOrder, cursor.path("sortOrder").asText(null))
+ || !lastName.isTextual() || lastName.asText().isEmpty()
+ || (sortBy == null || "NAME".equals(sortBy)) && !sortValue.isNull()
+ || !sortValue.isNull() && !sortValue.isNumber()) {
+ throw new AwsException("InvalidInputException", "Invalid nextToken", 400);
+ }
+ Double value = sortValue.isNull() ? null : sortValue.asDouble();
+ if (value != null && !Double.isFinite(value)) {
+ throw new AwsException("InvalidInputException", "Invalid nextToken", 400);
+ }
+ Project anchor = new Project();
+ anchor.setName(lastName.asText());
+ anchor.setCreated(value);
+ anchor.setLastModified(value);
+ return anchor;
+ }
+
+ private String encodeListProjectsToken(Project last, String region, String sortBy, String sortOrder) {
+ Double value = switch (sortBy == null ? "NAME" : sortBy) {
+ case "CREATED_TIME" -> last.getCreated();
+ case "LAST_MODIFIED_TIME" -> last.getLastModified();
+ default -> null;
+ };
+ ObjectNode cursor = PAGE_TOKEN_MAPPER.createObjectNode();
+ cursor.put("action", "ListProjects");
+ cursor.put("region", region);
+ cursor.put("sortBy", sortBy);
+ cursor.put("sortOrder", sortOrder);
+ cursor.put("lastName", last.getName());
+ cursor.put("sortValue", value);
+ return Base64Url.encode(cursor.toString());
+ }
+
+ public record ProjectPage(List<String> projects, String nextToken) {
+ }
+
+ private List<Project> listProjectRecords(String region, String sortBy, String sortOrder) {
+ return projectsFor(region).values().stream()
+ .sorted(projectComparator(sortBy, sortOrder))
+ .toList();
+ }
+
+ private Comparator<Project> projectComparator(String sortBy, String sortOrder) {
Comparator<Project> comparator = switch (sortBy == null ? "NAME" : sortBy) {
case "NAME" -> Comparator.comparing(Project::getName);
case "CREATED_TIME" -> Comparator.comparing(Project::getCreated,
@@ -265,10 +364,7 @@ public class CodeBuildService {
if (isDescendingSortOrder(sortOrder, false)) {
comparator = comparator.reversed();
}
- return projectsFor(region).values().stream()
- .sorted(comparator.thenComparing(Project::getName))
- .map(Project::getName)
- .toList();
+ return comparator.thenComparing(Project::getName);
}
// ---- Report Groups ----
diff --git a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsPaginationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsPaginationIntegrationTest.java
new file mode 100644
index 0000000000..d6bd25e3d6
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsPaginationIntegrationTest.java
@@ -0,0 +1,84 @@
+package io.github.hectorvent.floci.services.codebuild;
+
+import io.github.hectorvent.floci.services.codebuild.model.ProjectArtifacts;
+import io.github.hectorvent.floci.services.codebuild.model.ProjectEnvironment;
+import io.github.hectorvent.floci.services.codebuild.model.ProjectSource;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.IntStream;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@QuarkusTest
+class CodeBuildListProjectsPaginationIntegrationTest {
+
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT = "000000000000";
+
+ @Inject CodeBuildService service;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void listProjectsReturnsAtMostOneHundredNamesPerPage() {
+ String prefix = "page-project-" + UUID.randomUUID() + "-";
+ List<String> names = IntStream.range(0, 101)
+ .mapToObj(index -> prefix + "%03d".formatted(index))
+ .toList();
+ List<String> created = new ArrayList<>();
+ try {
+ ProjectSource source = new ProjectSource();
+ source.setType("NO_SOURCE");
+ ProjectArtifacts artifacts = new ProjectArtifacts();
+ artifacts.setType("NO_ARTIFACTS");
+ for (String name : names) {
+ service.createProject(REGION, ACCOUNT, name, null, source, null, null,
+ artifacts, null, new ProjectEnvironment(), "arn:aws:iam::000000000000:role/codebuild",
+ null, null, null, null, null, null, null);
+ created.add(name);
+ }
+
+ List<String> allNames = new ArrayList<>();
+ String nextToken = null;
+ int pages = 0;
+ do {
+ Map<String, String> request = nextToken == null
+ ? Map.of("sortBy", "NAME", "sortOrder", "ASCENDING")
+ : Map.of("sortBy", "NAME", "sortOrder", "ASCENDING", "nextToken", nextToken);
+ Response response = given().header("X-Amz-Target", "CodeBuild_20161006.ListProjects")
+ .contentType(CONTENT_TYPE).body(request)
+ .post("/").then().statusCode(200).extract().response();
+ List<String> page = response.jsonPath().getList("projects", String.class);
+ assertTrue(page.size() <= 100, "AWS ListProjects returns at most 100 names per page");
+ allNames.addAll(page);
+ nextToken = response.jsonPath().getString("nextToken");
+ if (pages++ == 0) {
+ assertNotNull(nextToken, "101 projects must produce a continuation token");
+ }
+ assertTrue(pages < 10, "pagination must eventually end");
+ } while (nextToken != null);
+
+ assertEquals(names, allNames.stream().filter(name -> name.startsWith(prefix)).toList());
+ } finally {
+ for (String name : created) {
+ service.deleteProject(REGION, name);
+ }
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsTest.java b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsTest.java
index 5836f4e22d..4c4702980a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsTest.java
@@ -1,8 +1,10 @@
package io.github.hectorvent.floci.services.codebuild;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.Base64Url;
import io.github.hectorvent.floci.services.codebuild.model.Project;
import io.github.hectorvent.floci.services.codebuild.model.ProjectArtifacts;
import io.github.hectorvent.floci.services.codebuild.model.ProjectEnvironment;
@@ -11,11 +13,19 @@ import jakarta.ws.rs.core.Response;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.IntStream;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.mock;
@@ -79,11 +89,153 @@ class CodeBuildListProjectsTest {
assertEquals(List.of("alpha", "zeta"), list(Map.of("sortBy", sortBy, "sortOrder", "ASCENDING")));
}
+ @Test
+ void paginatesWithoutChangingExistingSortOrder() throws Exception {
+ IntStream.range(0, 101).forEach(index -> createProject(REGION,
+ "project-%03d".formatted(index), (double) (101 - index), (double) index));
+
+ assertPagesMatchServiceOrder(Map.of(), service.listProjects(REGION).stream().sorted().toList());
+ assertPagesMatchServiceOrder(Map.of("sortBy", "NAME", "sortOrder", "DESCENDING"),
+ service.listProjects(REGION, "NAME", "DESCENDING"));
+ assertPagesMatchServiceOrder(Map.of("sortBy", "CREATED_TIME", "sortOrder", "ASCENDING"),
+ service.listProjects(REGION, "CREATED_TIME", "ASCENDING"));
+ assertPagesMatchServiceOrder(Map.of("sortBy", "LAST_MODIFIED_TIME", "sortOrder", "DESCENDING"),
+ service.listProjects(REGION, "LAST_MODIFIED_TIME", "DESCENDING"));
+ }
+
+ @Test
+ void onlyReturnsContinuationWhenMoreThanOneHundredProjectsExist() throws Exception {
+ assertFalse(listPage(Map.of()).containsKey("nextToken"));
+ IntStream.range(0, 100).forEach(index -> createProject(REGION,
+ "project-%03d".formatted(index), null, null));
+
+ Map<String, Object> page = listPage(Map.of("sortBy", "NAME"));
+ assertEquals(100, ((List<?>) page.get("projects")).size());
+ assertFalse(page.containsKey("nextToken"));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ void continuesThroughThreePages() throws Exception {
+ IntStream.range(0, 201).forEach(index -> createProject(REGION,
+ "project-%03d".formatted(index), null, null));
+
+ List<String> names = new ArrayList<>();
+ String token = null;
+ for (int pageIndex = 0; pageIndex < 3; pageIndex++) {
+ Map<String, String> request = new HashMap<>(Map.of("sortBy", "NAME"));
+ if (token != null) {
+ request.put("nextToken", token);
+ }
+ Map<String, Object> page = listPage(request);
+ List<String> projects = (List<String>) page.get("projects");
+ assertEquals(pageIndex < 2 ? 100 : 1, projects.size());
+ names.addAll(projects);
+ token = (String) page.get("nextToken");
+ }
+ assertNull(token);
+ assertEquals(service.listProjects(REGION, "NAME", null), names);
+ }
+
+ @Test
+ void rejectsInvalidAndMismatchedContinuationTokens() throws Exception {
+ IntStream.range(0, 101).forEach(index -> createProject(REGION,
+ "project-%03d".formatted(index), null, null));
+ String token = (String) listPage(Map.of("sortBy", "NAME", "sortOrder", "ASCENDING"))
+ .get("nextToken");
+ assertNotNull(token);
+
+ assertInvalidToken(Map.of("sortBy", "NAME", "sortOrder", "ASCENDING", "nextToken", "invalid?"),
+ REGION);
+ assertInvalidToken(Map.of("sortBy", "NAME", "sortOrder", "ASCENDING", "nextToken", ""),
+ REGION);
+ assertInvalidToken(Map.of("sortBy", "NAME", "sortOrder", "DESCENDING", "nextToken", token),
+ REGION);
+ assertInvalidToken(Map.of("sortBy", "NAME", "sortOrder", "ASCENDING", "nextToken", token),
+ "eu-west-1");
+
+ ObjectNode cursor = (ObjectNode) mapper.readTree(Base64.getUrlDecoder().decode(token));
+ cursor.put("sortValue", 1.0);
+ assertInvalidToken(Map.of("sortBy", "NAME", "sortOrder", "ASCENDING", "nextToken",
+ Base64Url.encode(mapper.writeValueAsBytes(cursor))), REGION);
+
+ String timeToken = (String) listPage(Map.of("sortBy", "CREATED_TIME"))
+ .get("nextToken");
+ ObjectNode timeCursor = (ObjectNode) mapper.readTree(Base64.getUrlDecoder().decode(timeToken));
+ timeCursor.put("sortValue", "invalid");
+ assertInvalidToken(Map.of("sortBy", "CREATED_TIME", "nextToken",
+ Base64Url.encode(mapper.writeValueAsBytes(timeCursor))), REGION);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"null", "123", "{}"})
+ void rejectsNonTextualContinuationTokens(String value) throws Exception {
+ ObjectNode request = mapper.createObjectNode();
+ request.set("nextToken", mapper.readTree(value));
+
+ AwsException error = assertThrows(AwsException.class,
+ () -> handler.handle("ListProjects", request, REGION, ACCOUNT));
+ assertEquals("InvalidInputException", error.getErrorCode());
+ assertEquals(400, error.getHttpStatus());
+ }
+
+ @ParameterizedTest
+ @CsvSource({"DEFAULT", "NAME", "CREATED_TIME"})
+ void continuesAfterDeletingTheLastProjectOnThePreviousPage(String sortBy) throws Exception {
+ IntStream.range(0, 101).forEach(index -> createProject(REGION,
+ "project-%03d".formatted(index), 1.0, null));
+ Map<String, String> request = switch (sortBy) {
+ case "NAME" -> Map.of("sortBy", "NAME", "sortOrder", "ASCENDING");
+ case "CREATED_TIME" -> Map.of("sortBy", "CREATED_TIME", "sortOrder", "DESCENDING");
+ default -> Map.of();
+ };
+ String token = (String) listPage(request).get("nextToken");
+ assertNotNull(token);
+
+ service.deleteProject(REGION, "project-099");
+ Map<String, String> nextRequest = new HashMap<>(request);
+ nextRequest.put("nextToken", token);
+ Map<String, Object> nextPage = listPage(nextRequest);
+ assertEquals(List.of("project-100"), nextPage.get("projects"));
+ assertFalse(nextPage.containsKey("nextToken"));
+ }
+
+ @SuppressWarnings("unchecked")
+ private void assertPagesMatchServiceOrder(Map<String, String> options, List<String> expected) throws Exception {
+ Map<String, Object> first = listPage(options);
+ assertEquals(100, ((List<?>) first.get("projects")).size());
+ String token = (String) first.get("nextToken");
+ assertNotNull(token);
+
+ Map<String, String> nextRequest = new HashMap<>(options);
+ nextRequest.put("nextToken", token);
+ Map<String, Object> second = listPage(nextRequest);
+ assertFalse(second.containsKey("nextToken"));
+ List<String> names = new ArrayList<>((List<String>) first.get("projects"));
+ names.addAll((List<String>) second.get("projects"));
+ assertEquals(expected, names);
+ }
+
+ private void assertInvalidToken(Map<String, String> request, String region) {
+ AwsException error = assertThrows(AwsException.class, () -> listPage(request, region));
+ assertEquals("InvalidInputException", error.getErrorCode());
+ assertEquals(400, error.getHttpStatus());
+ }
+
@SuppressWarnings("unchecked")
private List<String> list(Map<String, String> request) throws Exception {
- try (Response response = handler.handle("ListProjects", mapper.valueToTree(request), REGION, ACCOUNT)) {
+ return (List<String>) listPage(request).get("projects");
+ }
+
+ private Map<String, Object> listPage(Map<String, String> request) throws Exception {
+ return listPage(request, REGION);
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, Object> listPage(Map<String, String> request, String region) throws Exception {
+ try (Response response = handler.handle("ListProjects", mapper.valueToTree(request), region, ACCOUNT)) {
assertEquals(200, response.getStatus());
- return (List<String>) ((Map<?, ?>) response.getEntity()).get("projects");
+ return (Map<String, Object>) response.getEntity();
}
}