Light, fluffy, and always free - The AWS Local Emulator alternative
fix(codebuild): paginate ListProjects responses (#6163)
Jimmy Liu Jiacc08eab2f7f7
diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildJsonHandler.javaindex 90277f96d6..bde33874c4 100644--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildJsonHandler.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.codebuild;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.codebuild.CodeBuildService.ProjectPage;import io.github.hectorvent.floci.services.codebuild.model.Build;import io.github.hectorvent.floci.services.codebuild.model.Project;import io.github.hectorvent.floci.services.codebuild.model.ProjectArtifacts;@@ -130,9 +131,20 @@ public class CodeBuildJsonHandler {}private Response listProjects(JsonNode req, String region) {- List<String> names = service.listProjects(region,- req.path("sortBy").asText(null), req.path("sortOrder").asText(null));- return Response.ok(Map.of("projects", names)).build();+ String sortBy = req.path("sortBy").asText(null);+ String sortOrder = req.path("sortOrder").asText(null);+ JsonNode tokenNode = req.get("nextToken");+ if (tokenNode != null && !tokenNode.isTextual()) {+ throw new AwsException("InvalidInputException", "Invalid nextToken", 400);+ }+ String nextToken = tokenNode == null ? null : tokenNode.asText();+ ProjectPage page = service.listProjectsPage(region, sortBy, sortOrder, nextToken);+ Map<String, Object> result = new HashMap<>();+ result.put("projects", page.projects());+ if (page.nextToken() != null) {+ result.put("nextToken", page.nextToken());+ }+ return Response.ok(result).build();}private Response createReportGroup(JsonNode req, String region, String account) throws Exception {diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.javaindex b2c01ed81d..0a36f5ad39 100644--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java@@ -1,9 +1,13 @@package io.github.hectorvent.floci.services.codebuild;import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.Base64Url;import io.github.hectorvent.floci.core.storage.StorageBackedMap;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.codebuild.model.Build;@@ -18,11 +22,14 @@ import jakarta.annotation.PostConstruct;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;+import java.io.IOException;import java.time.Instant;import java.util.ArrayList;+import java.util.Base64;import java.util.Comparator;import java.util.List;import java.util.Map;+import java.util.Objects;import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.CopyOnWriteArrayList;@@ -32,6 +39,9 @@ import java.util.stream.Collectors;@ApplicationScopedpublic class CodeBuildService {+ private static final int LIST_PROJECTS_PAGE_SIZE = 100;+ private static final ObjectMapper PAGE_TOKEN_MAPPER = new ObjectMapper();+// key: region -> name -> projectprivate Map<String, Map<String, Project>> projects = new ConcurrentHashMap<>();// key: region -> arn -> report group@@ -254,6 +264,95 @@ public class CodeBuildService {if (sortBy == null && sortOrder == null) {return listProjects(region);}+ return listProjectRecords(region, sortBy, sortOrder).stream()+ .map(Project::getName)+ .toList();+ }++ public ProjectPage listProjectsPage(String region, String sortBy, String sortOrder, String nextToken) {+ if (sortBy == null && sortOrder == null && nextToken == null) {+ List<String> names = listProjects(region);+ if (names.size() <= LIST_PROJECTS_PAGE_SIZE) {+ return new ProjectPage(names, null);+ }+ }++ List<Project> projects = listProjectRecords(region, sortBy, sortOrder);+ int start = 0;+ if (nextToken != null) {+ Project anchor = decodeListProjectsToken(nextToken, region, sortBy, sortOrder);+ Comparator<Project> comparator = projectComparator(sortBy, sortOrder);+ while (start < projects.size() && comparator.compare(projects.get(start), anchor) <= 0) {+ start++;+ }+ }++ int end = Math.min(start + LIST_PROJECTS_PAGE_SIZE, projects.size());+ List<String> names = projects.subList(start, end).stream().map(Project::getName).toList();+ String resultToken = end < projects.size()+ ? encodeListProjectsToken(projects.get(end - 1), region, sortBy, sortOrder)+ : null;+ return new ProjectPage(names, resultToken);+ }++ private Project decodeListProjectsToken(String token, String region, String sortBy, String sortOrder) {+ if (token.isEmpty()) {+ throw new AwsException("InvalidInputException", "Invalid nextToken", 400);+ }+ JsonNode cursor;+ try {+ cursor = PAGE_TOKEN_MAPPER.readTree(Base64.getUrlDecoder().decode(token));+ } catch (IllegalArgumentException | IOException e) {+ throw new AwsException("InvalidInputException", "Invalid nextToken", 400);+ }+ JsonNode lastName = cursor == null ? null : cursor.path("lastName");+ JsonNode sortValue = cursor == null ? null : cursor.path("sortValue");+ if (cursor == null || !"ListProjects".equals(cursor.path("action").asText(null))+ || !region.equals(cursor.path("region").asText(null))+ || !cursor.has("sortBy") || !Objects.equals(sortBy, cursor.path("sortBy").asText(null))+ || !cursor.has("sortOrder") || !Objects.equals(sortOrder, cursor.path("sortOrder").asText(null))+ || !lastName.isTextual() || lastName.asText().isEmpty()+ || (sortBy == null || "NAME".equals(sortBy)) && !sortValue.isNull()+ || !sortValue.isNull() && !sortValue.isNumber()) {+ throw new AwsException("InvalidInputException", "Invalid nextToken", 400);+ }+ Double value = sortValue.isNull() ? null : sortValue.asDouble();+ if (value != null && !Double.isFinite(value)) {+ throw new AwsException("InvalidInputException", "Invalid nextToken", 400);+ }+ Project anchor = new Project();+ anchor.setName(lastName.asText());+ anchor.setCreated(value);+ anchor.setLastModified(value);+ return anchor;+ }++ private String encodeListProjectsToken(Project last, String region, String sortBy, String sortOrder) {+ Double value = switch (sortBy == null ? "NAME" : sortBy) {+ case "CREATED_TIME" -> last.getCreated();+ case "LAST_MODIFIED_TIME" -> last.getLastModified();+ default -> null;+ };+ ObjectNode cursor = PAGE_TOKEN_MAPPER.createObjectNode();+ cursor.put("action", "ListProjects");+ cursor.put("region", region);+ cursor.put("sortBy", sortBy);+ cursor.put("sortOrder", sortOrder);+ cursor.put("lastName", last.getName());+ cursor.put("sortValue", value);+ return Base64Url.encode(cursor.toString());+ }++ public record ProjectPage(List<String> projects, String nextToken) {+ }++ private List<Project> listProjectRecords(String region, String sortBy, String sortOrder) {+ return projectsFor(region).values().stream()+ .sorted(projectComparator(sortBy, sortOrder))+ .toList();+ }++ private Comparator<Project> projectComparator(String sortBy, String sortOrder) {Comparator<Project> comparator = switch (sortBy == null ? "NAME" : sortBy) {case "NAME" -> Comparator.comparing(Project::getName);case "CREATED_TIME" -> Comparator.comparing(Project::getCreated,@@ -265,10 +364,7 @@ public class CodeBuildService {if (isDescendingSortOrder(sortOrder, false)) {comparator = comparator.reversed();}- return projectsFor(region).values().stream()- .sorted(comparator.thenComparing(Project::getName))- .map(Project::getName)- .toList();+ return comparator.thenComparing(Project::getName);}// ---- Report Groups ----diff --git a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsPaginationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsPaginationIntegrationTest.javanew file mode 100644index 0000000000..d6bd25e3d6--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsPaginationIntegrationTest.java@@ -0,0 +1,84 @@+package io.github.hectorvent.floci.services.codebuild;++import io.github.hectorvent.floci.services.codebuild.model.ProjectArtifacts;+import io.github.hectorvent.floci.services.codebuild.model.ProjectEnvironment;+import io.github.hectorvent.floci.services.codebuild.model.ProjectSource;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import jakarta.inject.Inject;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.UUID;+import java.util.stream.IntStream;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertTrue;++@QuarkusTest+class CodeBuildListProjectsPaginationIntegrationTest {++ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";+ private static final String REGION = "us-east-1";+ private static final String ACCOUNT = "000000000000";++ @Inject CodeBuildService service;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void listProjectsReturnsAtMostOneHundredNamesPerPage() {+ String prefix = "page-project-" + UUID.randomUUID() + "-";+ List<String> names = IntStream.range(0, 101)+ .mapToObj(index -> prefix + "%03d".formatted(index))+ .toList();+ List<String> created = new ArrayList<>();+ try {+ ProjectSource source = new ProjectSource();+ source.setType("NO_SOURCE");+ ProjectArtifacts artifacts = new ProjectArtifacts();+ artifacts.setType("NO_ARTIFACTS");+ for (String name : names) {+ service.createProject(REGION, ACCOUNT, name, null, source, null, null,+ artifacts, null, new ProjectEnvironment(), "arn:aws:iam::000000000000:role/codebuild",+ null, null, null, null, null, null, null);+ created.add(name);+ }++ List<String> allNames = new ArrayList<>();+ String nextToken = null;+ int pages = 0;+ do {+ Map<String, String> request = nextToken == null+ ? Map.of("sortBy", "NAME", "sortOrder", "ASCENDING")+ : Map.of("sortBy", "NAME", "sortOrder", "ASCENDING", "nextToken", nextToken);+ Response response = given().header("X-Amz-Target", "CodeBuild_20161006.ListProjects")+ .contentType(CONTENT_TYPE).body(request)+ .post("/").then().statusCode(200).extract().response();+ List<String> page = response.jsonPath().getList("projects", String.class);+ assertTrue(page.size() <= 100, "AWS ListProjects returns at most 100 names per page");+ allNames.addAll(page);+ nextToken = response.jsonPath().getString("nextToken");+ if (pages++ == 0) {+ assertNotNull(nextToken, "101 projects must produce a continuation token");+ }+ assertTrue(pages < 10, "pagination must eventually end");+ } while (nextToken != null);++ assertEquals(names, allNames.stream().filter(name -> name.startsWith(prefix)).toList());+ } finally {+ for (String name : created) {+ service.deleteProject(REGION, name);+ }+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsTest.java b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsTest.javaindex 5836f4e22d..4c4702980a 100644--- a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildListProjectsTest.java@@ -1,8 +1,10 @@package io.github.hectorvent.floci.services.codebuild;import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.Base64Url;import io.github.hectorvent.floci.services.codebuild.model.Project;import io.github.hectorvent.floci.services.codebuild.model.ProjectArtifacts;import io.github.hectorvent.floci.services.codebuild.model.ProjectEnvironment;@@ -11,11 +13,19 @@ import jakarta.ws.rs.core.Response;import org.junit.jupiter.api.Test;import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.CsvSource;+import org.junit.jupiter.params.provider.ValueSource;+import java.util.ArrayList;+import java.util.Base64;+import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.stream.IntStream;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.mockito.Mockito.mock;@@ -79,11 +89,153 @@ class CodeBuildListProjectsTest {assertEquals(List.of("alpha", "zeta"), list(Map.of("sortBy", sortBy, "sortOrder", "ASCENDING")));}+ @Test+ void paginatesWithoutChangingExistingSortOrder() throws Exception {+ IntStream.range(0, 101).forEach(index -> createProject(REGION,+ "project-%03d".formatted(index), (double) (101 - index), (double) index));++ assertPagesMatchServiceOrder(Map.of(), service.listProjects(REGION).stream().sorted().toList());+ assertPagesMatchServiceOrder(Map.of("sortBy", "NAME", "sortOrder", "DESCENDING"),+ service.listProjects(REGION, "NAME", "DESCENDING"));+ assertPagesMatchServiceOrder(Map.of("sortBy", "CREATED_TIME", "sortOrder", "ASCENDING"),+ service.listProjects(REGION, "CREATED_TIME", "ASCENDING"));+ assertPagesMatchServiceOrder(Map.of("sortBy", "LAST_MODIFIED_TIME", "sortOrder", "DESCENDING"),+ service.listProjects(REGION, "LAST_MODIFIED_TIME", "DESCENDING"));+ }++ @Test+ void onlyReturnsContinuationWhenMoreThanOneHundredProjectsExist() throws Exception {+ assertFalse(listPage(Map.of()).containsKey("nextToken"));+ IntStream.range(0, 100).forEach(index -> createProject(REGION,+ "project-%03d".formatted(index), null, null));++ Map<String, Object> page = listPage(Map.of("sortBy", "NAME"));+ assertEquals(100, ((List<?>) page.get("projects")).size());+ assertFalse(page.containsKey("nextToken"));+ }++ @Test+ @SuppressWarnings("unchecked")+ void continuesThroughThreePages() throws Exception {+ IntStream.range(0, 201).forEach(index -> createProject(REGION,+ "project-%03d".formatted(index), null, null));++ List<String> names = new ArrayList<>();+ String token = null;+ for (int pageIndex = 0; pageIndex < 3; pageIndex++) {+ Map<String, String> request = new HashMap<>(Map.of("sortBy", "NAME"));+ if (token != null) {+ request.put("nextToken", token);+ }+ Map<String, Object> page = listPage(request);+ List<String> projects = (List<String>) page.get("projects");+ assertEquals(pageIndex < 2 ? 100 : 1, projects.size());+ names.addAll(projects);+ token = (String) page.get("nextToken");+ }+ assertNull(token);+ assertEquals(service.listProjects(REGION, "NAME", null), names);+ }++ @Test+ void rejectsInvalidAndMismatchedContinuationTokens() throws Exception {+ IntStream.range(0, 101).forEach(index -> createProject(REGION,+ "project-%03d".formatted(index), null, null));+ String token = (String) listPage(Map.of("sortBy", "NAME", "sortOrder", "ASCENDING"))+ .get("nextToken");+ assertNotNull(token);++ assertInvalidToken(Map.of("sortBy", "NAME", "sortOrder", "ASCENDING", "nextToken", "invalid?"),+ REGION);+ assertInvalidToken(Map.of("sortBy", "NAME", "sortOrder", "ASCENDING", "nextToken", ""),+ REGION);+ assertInvalidToken(Map.of("sortBy", "NAME", "sortOrder", "DESCENDING", "nextToken", token),+ REGION);+ assertInvalidToken(Map.of("sortBy", "NAME", "sortOrder", "ASCENDING", "nextToken", token),+ "eu-west-1");++ ObjectNode cursor = (ObjectNode) mapper.readTree(Base64.getUrlDecoder().decode(token));+ cursor.put("sortValue", 1.0);+ assertInvalidToken(Map.of("sortBy", "NAME", "sortOrder", "ASCENDING", "nextToken",+ Base64Url.encode(mapper.writeValueAsBytes(cursor))), REGION);++ String timeToken = (String) listPage(Map.of("sortBy", "CREATED_TIME"))+ .get("nextToken");+ ObjectNode timeCursor = (ObjectNode) mapper.readTree(Base64.getUrlDecoder().decode(timeToken));+ timeCursor.put("sortValue", "invalid");+ assertInvalidToken(Map.of("sortBy", "CREATED_TIME", "nextToken",+ Base64Url.encode(mapper.writeValueAsBytes(timeCursor))), REGION);+ }++ @ParameterizedTest+ @ValueSource(strings = {"null", "123", "{}"})+ void rejectsNonTextualContinuationTokens(String value) throws Exception {+ ObjectNode request = mapper.createObjectNode();+ request.set("nextToken", mapper.readTree(value));++ AwsException error = assertThrows(AwsException.class,+ () -> handler.handle("ListProjects", request, REGION, ACCOUNT));+ assertEquals("InvalidInputException", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ }++ @ParameterizedTest+ @CsvSource({"DEFAULT", "NAME", "CREATED_TIME"})+ void continuesAfterDeletingTheLastProjectOnThePreviousPage(String sortBy) throws Exception {+ IntStream.range(0, 101).forEach(index -> createProject(REGION,+ "project-%03d".formatted(index), 1.0, null));+ Map<String, String> request = switch (sortBy) {+ case "NAME" -> Map.of("sortBy", "NAME", "sortOrder", "ASCENDING");+ case "CREATED_TIME" -> Map.of("sortBy", "CREATED_TIME", "sortOrder", "DESCENDING");+ default -> Map.of();+ };+ String token = (String) listPage(request).get("nextToken");+ assertNotNull(token);++ service.deleteProject(REGION, "project-099");+ Map<String, String> nextRequest = new HashMap<>(request);+ nextRequest.put("nextToken", token);+ Map<String, Object> nextPage = listPage(nextRequest);+ assertEquals(List.of("project-100"), nextPage.get("projects"));+ assertFalse(nextPage.containsKey("nextToken"));+ }++ @SuppressWarnings("unchecked")+ private void assertPagesMatchServiceOrder(Map<String, String> options, List<String> expected) throws Exception {+ Map<String, Object> first = listPage(options);+ assertEquals(100, ((List<?>) first.get("projects")).size());+ String token = (String) first.get("nextToken");+ assertNotNull(token);++ Map<String, String> nextRequest = new HashMap<>(options);+ nextRequest.put("nextToken", token);+ Map<String, Object> second = listPage(nextRequest);+ assertFalse(second.containsKey("nextToken"));+ List<String> names = new ArrayList<>((List<String>) first.get("projects"));+ names.addAll((List<String>) second.get("projects"));+ assertEquals(expected, names);+ }++ private void assertInvalidToken(Map<String, String> request, String region) {+ AwsException error = assertThrows(AwsException.class, () -> listPage(request, region));+ assertEquals("InvalidInputException", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ }+@SuppressWarnings("unchecked")private List<String> list(Map<String, String> request) throws Exception {- try (Response response = handler.handle("ListProjects", mapper.valueToTree(request), REGION, ACCOUNT)) {+ return (List<String>) listPage(request).get("projects");+ }++ private Map<String, Object> listPage(Map<String, String> request) throws Exception {+ return listPage(request, REGION);+ }++ @SuppressWarnings("unchecked")+ private Map<String, Object> listPage(Map<String, String> request, String region) throws Exception {+ try (Response response = handler.handle("ListProjects", mapper.valueToTree(request), region, ACCOUNT)) {assertEquals(200, response.getStatus());- return (List<String>) ((Map<?, ?>) response.getEntity()).get("projects");+ return (Map<String, Object>) response.getEntity();}}