Light, fluffy, and always free - The AWS Local Emulator alternative
fix(batch): paginate multi-node job listings (#6512)
Aniket Shukladdb94d6c151d
diff --git a/docs/services/batch.md b/docs/services/batch.mdindex 47b5104946..e9a9283bcb 100644--- a/docs/services/batch.md+++ b/docs/services/batch.md@@ -103,6 +103,8 @@ does not yet preempt the remaining nodes (see Limitations). `DescribeJobs`/queuereturn one entry for the wholejob (with `nodeProperties`, no per-node `container`); pass `multiNodeJobId` to `ListJobs` for aone-entry-per-node breakdown with each node's own `container` and `nodeProperties.nodeIndex`.+Use `maxResults` and `nextToken` to page this breakdown, with up to 1,000 nodes per page.+The final page omits `nextToken`.`containerOverrides` and array jobs are not supported on MNP job definitions; use `nodeOverrides`instead.@@ -171,5 +173,3 @@ correctness, deliberately deferred):- `ListJobs` with `arrayJobId` ignores the `filters` parameter (only `jobStatus` is honored forchildren). This matches AWS's own note that filters don't apply to child jobs, but it means arraychildren can't be filtered by `JOB_NAME`/`JOB_DEFINITION`/etc.-- `ListJobs` with `multiNodeJobId` has no pagination (`maxResults`/`nextToken`); it always returns- every node in one response.diff --git a/pom.xml b/pom.xmlindex b3dbb1732e..a59b4b9818 100644--- a/pom.xml+++ b/pom.xml@@ -348,6 +348,12 @@<version>${aws.sdk.version}</version><scope>test</scope></dependency>+ <dependency>+ <groupId>software.amazon.awssdk</groupId>+ <artifactId>batch</artifactId>+ <version>${aws.sdk.version}</version>+ <scope>test</scope>+ </dependency><dependency><groupId>software.amazon.awssdk</groupId><artifactId>bedrock</artifactId>diff --git a/src/main/java/io/github/hectorvent/floci/services/batch/BatchService.java b/src/main/java/io/github/hectorvent/floci/services/batch/BatchService.javaindex e5e6176b22..dafbaa3d80 100644--- a/src/main/java/io/github/hectorvent/floci/services/batch/BatchService.java+++ b/src/main/java/io/github/hectorvent/floci/services/batch/BatchService.java@@ -893,7 +893,7 @@ public class BatchService {return listArrayChildren(arrayJobId, request);}if (multiNodeJobId != null) {- return listMultiNodeJobNodes(multiNodeJobId);+ return listMultiNodeJobNodes(multiNodeJobId, request);}BatchJobQueue queue = resolveJobQueue(jobQueueRef);@@ -959,18 +959,26 @@ public class BatchService {return out;}- // No pagination: realistic node counts are small, so every node is always returned.- private ObjectNode listMultiNodeJobNodes(String jobId) {+ private ObjectNode listMultiNodeJobNodes(String jobId, JsonNode request) {+ int maxResults = request.hasNonNull("maxResults")+ ? Math.max(1, Math.min(DEFAULT_MAX_RESULTS, request.path("maxResults").asInt()))+ : DEFAULT_MAX_RESULTS;+ int offset = parseNextToken(textOrNull(request, "nextToken"));ArrayNode summaries = objectMapper.createArrayNode();+ ObjectNode out = objectMapper.createObjectNode();+ out.set("jobSummaryList", summaries);getJob(jobId).ifPresent(job -> {if (job.getNodeProperties() != null) {- for (BatchNodeExecution execution : job.getNodeExecutions()) {- summaries.add(nodeJobSummary(job, execution));+ List<BatchNodeExecution> executions = job.getNodeExecutions();+ int end = (int) Math.min(executions.size(), (long) offset + maxResults);+ for (int index = offset; index < end; index++) {+ summaries.add(nodeJobSummary(job, executions.get(index)));+ }+ if (end < executions.size()) {+ out.put("nextToken", String.valueOf(end));}}});- ObjectNode out = objectMapper.createObjectNode();- out.set("jobSummaryList", summaries);return out;}diff --git a/src/test/java/io/github/hectorvent/floci/services/batch/BatchMultiNodePaginationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/batch/BatchMultiNodePaginationIntegrationTest.javanew file mode 100644index 0000000000..af9508faf8--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/batch/BatchMultiNodePaginationIntegrationTest.java@@ -0,0 +1,174 @@+package io.github.hectorvent.floci.services.batch;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.batch.model.BatchJob;+import io.github.hectorvent.floci.services.batch.model.BatchNodeExecution;+import io.github.hectorvent.floci.services.batch.model.BatchNodeProperties;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.RestAssured;+import jakarta.inject.Inject;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;+import software.amazon.awssdk.regions.Region;+import software.amazon.awssdk.services.batch.BatchClient;+import software.amazon.awssdk.services.batch.model.ClientException;+import software.amazon.awssdk.services.batch.model.KeyValuesPair;+import software.amazon.awssdk.services.batch.model.ListJobsResponse;++import java.net.URI;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.UUID;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++@QuarkusTest+class BatchMultiNodePaginationIntegrationTest {++ private static final String ACCOUNT = "741648900001";+ private static final Region REGION = Region.US_EAST_1;++ @Inject+ StorageFactory storageFactory;++ private AccountAwareStorageBackend<BatchJob> jobs;+ private final List<String> seededJobs = new ArrayList<>();++ @BeforeEach+ void setUp() {+ jobs = storageFactory.create("batch", "batch-jobs.json", new TypeReference<Map<String, BatchJob>>() {});+ }++ @AfterEach+ void cleanUp() {+ seededJobs.forEach(id -> jobs.deleteForAccount(ACCOUNT, id));+ }++ @Test+ void sdkPagesMultiNodeJobsAndItsPaginatorVisitsEveryNodeOnce() {+ String jobId = seedJob(3);+ try (BatchClient client = client(ACCOUNT)) {+ ListJobsResponse first = client.listJobs(request -> request.multiNodeJobId(jobId).maxResults(1));+ assertEquals(1, first.jobSummaryList().size());+ assertEquals(0, first.jobSummaryList().getFirst().nodeProperties().nodeIndex());+ assertTrue(first.jobSummaryList().getFirst().nodeProperties().isMainNode());+ assertNotNull(first.nextToken());+ ListJobsResponse second = client.listJobs(request -> request.multiNodeJobId(jobId)+ .maxResults(1).nextToken(first.nextToken()));+ assertEquals(1, second.jobSummaryList().size());+ assertEquals(1, second.jobSummaryList().getFirst().nodeProperties().nodeIndex());+ assertNotNull(second.nextToken());+ ListJobsResponse last = client.listJobs(request -> request.multiNodeJobId(jobId)+ .maxResults(1).nextToken(second.nextToken()));+ assertEquals(1, last.jobSummaryList().size());+ assertEquals(2, last.jobSummaryList().getFirst().nodeProperties().nodeIndex());+ assertNull(last.nextToken());+ assertEquals(List.of(0, 1, 2), client.listJobsPaginator(request -> request+ .multiNodeJobId(jobId).maxResults(2)).jobSummaryList().stream()+ .map(summary -> summary.nodeProperties().nodeIndex()).toList());+ }+ }++ @Test+ void sdkUsesTheExistingThousandResultLimitForMultiNodeJobs() {+ String jobId = seedJob(1001);+ try (BatchClient client = client(ACCOUNT)) {+ ListJobsResponse first = client.listJobs(request -> request.multiNodeJobId(jobId));+ assertEquals(1000, first.jobSummaryList().size());+ assertNotNull(first.nextToken());+ ListJobsResponse last = client.listJobs(request -> request.multiNodeJobId(jobId)+ .nextToken(first.nextToken()));+ assertEquals(1, last.jobSummaryList().size());+ assertEquals(1000, last.jobSummaryList().getFirst().nodeProperties().nodeIndex());+ assertNull(last.nextToken());+ assertEquals(1000, client.listJobs(request -> request.multiNodeJobId(jobId)+ .maxResults(Integer.MAX_VALUE)).jobSummaryList().size());+ assertEquals(1000, client.listJobs(request -> request.multiNodeJobId(jobId)+ .filters(KeyValuesPair.builder().name("JOB_NAME").values("does-not-match").build()))+ .jobSummaryList().size());+ }+ }++ @ParameterizedTest+ @ValueSource(strings = {"not-a-token", "-1", "2147483648"})+ void sdkRejectsInvalidMultiNodeTokens(String token) {+ String jobId = seedJob(3);+ try (BatchClient client = client(ACCOUNT)) {+ ClientException failure = assertThrows(ClientException.class,+ () -> client.listJobs(request -> request.multiNodeJobId(jobId).nextToken(token)));+ assertEquals(400, failure.statusCode());+ }+ }++ @ParameterizedTest+ @ValueSource(ints = {3, Integer.MAX_VALUE})+ void sdkReturnsAnEmptyTerminalPageAtOrBeyondTheNodeCount(int offset) {+ String jobId = seedJob(3);+ try (BatchClient client = client(ACCOUNT)) {+ ListJobsResponse response = client.listJobs(request -> request.multiNodeJobId(jobId)+ .maxResults(1).nextToken(Integer.toString(offset)));+ assertTrue(response.jobSummaryList().isEmpty());+ assertNull(response.nextToken());+ }+ }++ @Test+ void sdkKeepsMissingAndNonMultiNodeJobsEmptyAndDoesNotExposeAnotherAccount() {+ String plainJob = seedJob(0);+ String multiNodeJob = seedJob(3);+ try (BatchClient client = client(ACCOUNT); BatchClient otherAccount = client("741648900002")) {+ assertTrue(client.listJobs(request -> request.multiNodeJobId("missing")).jobSummaryList().isEmpty());+ assertTrue(client.listJobs(request -> request.multiNodeJobId(plainJob)).jobSummaryList().isEmpty());+ assertTrue(otherAccount.listJobs(request -> request.multiNodeJobId(multiNodeJob))+ .jobSummaryList().isEmpty());+ }+ }++ private String seedJob(int nodes) {+ BatchJob job = new BatchJob();+ String id = UUID.randomUUID().toString();+ job.setJobId(id);+ job.setJobArn(AwsArnUtils.Arn.of("batch", REGION.id(), ACCOUNT, "job/" + id).toString());+ job.setJobName("pagination-" + id);+ job.setStatus("SUCCEEDED");+ job.setRegion(REGION.id());+ job.setAccountId(ACCOUNT);+ job.setCreatedAt(System.currentTimeMillis());+ if (nodes > 0) {+ BatchNodeProperties properties = new BatchNodeProperties();+ properties.setNumNodes(nodes);+ properties.setMainNode(0);+ job.setNodeProperties(properties);+ for (int index = 0; index < nodes; index++) {+ BatchNodeExecution execution = new BatchNodeExecution();+ execution.setNodeIndex(index);+ execution.setMainNode(index == 0);+ job.getNodeExecutions().add(execution);+ }+ }+ jobs.putForAccount(ACCOUNT, id, job);+ seededJobs.add(id);+ return id;+ }++ private static BatchClient client(String account) {+ return BatchClient.builder().endpointOverride(URI.create("http://localhost:" + RestAssured.port))+ .httpClientBuilder(UrlConnectionHttpClient.builder()).region(REGION)+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(account, "test")))+ .build();+ }+}