Light, fluffy, and always free - The AWS Local Emulator alternative
feat(sfn): apply ItemBatcher to distributed Map children (#3604)
Ovidiu (Ovi)917e94914ff7
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.mdindex de6e2a87f9..9519a82bba 100644--- a/docs/services/step-functions.md+++ b/docs/services/step-functions.md@@ -124,6 +124,24 @@ Results remain in input order even when iterations finish out of order. If an itthe Map state fails promptly, cancels its active sibling iterations, and does not start queuediterations.+## Distributed Map ItemBatcher++`ItemBatcher` hands each child execution a batch of items instead of a single item. The child input+is `{"BatchInput": ..., "Items": [...]}`, with `BatchInput` present only when the state declares it.+`ItemSelector` still runs per item, before the items are grouped.++A batch closes on `MaxItemsPerBatch`, on `MaxInputBytesPerBatch`, or on the 256 KiB child-input+ceiling AWS applies whether or not a byte limit is declared. Either limit may be given as a+`...Path` field, or as an expression in a JSONata state machine. With neither declared, items fill+one batch up to that ceiling. The size measured is the serialized child payload, envelope and+`BatchInput` included, not the items alone. An item that would exceed the ceiling on its own can+never start a child execution, so the state fails with `States.DataLimitExceeded` rather than+building a batch AWS would reject: reduce the item with `ItemSelector` first.++`MaxConcurrency` then bounds concurrent batches, and the Map result has one entry per batch rather+than per item. `DescribeMapRun` reports items under `itemCounts` and batches under+`executionCounts`.+## Retry policies`Task`, `Parallel`, and `Map` states honor their `Retry` field. `ErrorEquals` matchingdiff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.javaindex a2ca923bf3..1e071c5ed9 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java@@ -112,6 +112,9 @@ import java.util.stream.Collectors;@ApplicationScopedpublic class AslExecutor {+ /** AWS starts no child execution with an input over 256 KiB, batched or not. */+ private static final int MAX_BATCH_INPUT_BYTES = 256 * 1024;+private enum MapItemsSource {DEFAULT,ITEM_READER_ARRAY,@@ -2352,43 +2355,46 @@ public class AslExecutor {ArrayNode results = objectMapper.createArrayNode();int itemCount = items.size();- JsonNode[] childInputsByIndex = hasResultWriter ? new JsonNode[itemCount] : null;- long[][] childTimingsByIndex = hasResultWriter ? new long[itemCount][] : null;- int requestedConcurrency = resolveMapMaxConcurrency(- stateDef, mapInput, jsonata, context, variables);+ // An ItemBatcher gives each child execution a batch of items rather than one item, so the+ // children the scheduler runs are the batches. ItemSelector has already been applied to+ // each item inside them, as on AWS.+ List<JsonNode> batches = stateDef.has("ItemBatcher")+ ? buildItemBatches(stateDef, items, resolvedItems, itemTransform, mapInput, jsonata,+ context, variables)+ : null;+ int childCount = batches == null ? itemCount : batches.size();+ JsonNode[] childInputsByIndex = hasResultWriter ? new JsonNode[childCount] : null;+ long[][] childTimingsByIndex = hasResultWriter ? new long[childCount][] : null;+ int requestedConcurrency = resolveMapIntegerField(+ stateDef, "MaxConcurrency", 0, mapInput, jsonata, context, variables);int effectiveConcurrency = effectiveMapConcurrency(- itemCount, requestedConcurrency, distributed);+ childCount, requestedConcurrency, distributed);chain.publish("MapStateStarted", Map.of("length", itemCount));MapRunIdentity mapRun = null;MapRun mapRunRecord = null;if (distributed) {mapRun = newMapRunIdentity(stateDef, sm, context);- mapRunRecord = newMapRun(mapRun, context, itemCount, requestedConcurrency);+ mapRunRecord = newMapRun(mapRun, context, itemCount, childCount, requestedConcurrency);chain.publish("MapRunStarted", Map.of("mapRunArn", mapRun.arn()));}var succeededItems = new AtomicInteger();var failedItems = new AtomicInteger();- var iterationChains = new ArrayList<HistoryChain>(itemCount);- for (var i = 0; i < itemCount; i++) {+ AtomicInteger succeededExecutions = new AtomicInteger();+ AtomicInteger failedExecutions = new AtomicInteger();+ var iterationChains = new ArrayList<HistoryChain>(childCount);+ for (var i = 0; i < childCount; i++) {iterationChains.add(distributed ? HistoryChain.ofChildExecution() : chain.fork());}java.util.function.IntFunction<Callable<JsonNode>> makeTask = (i) -> () -> {var iterationChain = iterationChains.get(i);- JsonNode item = items.get(i);- ObjectNode iterContext = ((ObjectNode) context).deepCopy();- ObjectNode mapCtx = objectMapper.createObjectNode();- ObjectNode mapItem = objectMapper.createObjectNode();- mapItem.put("Index", i);- if (resolvedItems.source() == MapItemsSource.ITEM_READER_OBJECT) {- mapItem.put("Key", item.path("Key").asText());- mapItem.set("Value", item.get("Value"));- } else {- mapItem.set("Value", item);- }- mapCtx.set("Item", mapItem);- iterContext.set("Map", mapCtx);+ boolean batchedChild = batches != null;+ JsonNode item = batchedChild ? batches.get(i) : items.get(i);+ int itemsInChild = batchedChild ? item.path("Items").size() : 1;+ ObjectNode iterContext = batchedChild+ ? ((ObjectNode) context).deepCopy()+ : mapItemContext(context, resolvedItems, items.get(i), i);long startMs = hasResultWriter ? System.currentTimeMillis() : 0L;JsonNode branchOutput;@@ -2397,7 +2403,7 @@ public class AslExecutor {iterationChain.publish("MapIterationStarted", Map.of("name", name, "index", i));}JsonNode iterInput = item;- if (itemTransform != null) {+ if (!batchedChild && itemTransform != null) {// $ in ItemSelector resolves against the Map state's effective input, not the item.iterInput = resolveParameters(itemTransform, mapInput, iterContext);}@@ -2412,13 +2418,15 @@ public class AslExecutor {branchOutput = executeBranch(startAt, iteratorStates, iterInput, iterationChain, sm,topLevelQueryLanguage, iterContext, variables.deepCopy());} catch (FailStateException e) {- failedItems.incrementAndGet();+ failedItems.addAndGet(itemsInChild);+ failedExecutions.incrementAndGet();if (!distributed && !e.isRuntimeError()) {iterationChain.publishAside("MapIterationFailed", Map.of("name", name, "index", i));}throw new IterationFailure(i, e);}- succeededItems.incrementAndGet();+ succeededItems.addAndGet(itemsInChild);+ succeededExecutions.incrementAndGet();if (!distributed) {iterationChain.publish("MapIterationSucceeded", Map.of("name", name, "index", i));}@@ -2428,11 +2436,11 @@ public class AslExecutor {return branchOutput;};- if (itemCount > 0) {+ if (childCount > 0) {List<JsonNode> itemOutputs;try {itemOutputs = MapIterationScheduler.execute(- itemCount, Math.max(1, effectiveConcurrency),+ childCount, Math.max(1, effectiveConcurrency),i -> () -> callUnderExecutionAccount(sm, makeTask.apply(i)),executionDeadlineNanos);} catch (java.util.concurrent.TimeoutException e) {@@ -2445,7 +2453,8 @@ public class AslExecutor {chain.continueFrom(iterationChains.get(e.index).lastEventId());} else {publishMapRunFailedEvent(chain, e.failure);- recordMapRun(mapRunRecord, "FAILED", succeededItems.get(), failedItems.get());+ recordMapRun(mapRunRecord, "FAILED", succeededItems.get(), failedItems.get(),+ succeededExecutions.get(), failedExecutions.get());}throw e.failure;} finally {@@ -2457,8 +2466,8 @@ public class AslExecutor {JsonNode mapResult = results;if (hasResultWriter) {ArrayNode childInputs = objectMapper.createArrayNode();- List<long[]> childTimings = new ArrayList<>(itemCount);- for (int i = 0; i < itemCount; i++) {+ List<long[]> childTimings = new ArrayList<>(childCount);+ for (int i = 0; i < childCount; i++) {childInputs.add(childInputsByIndex[i]);childTimings.add(childTimingsByIndex[i]);}@@ -2468,13 +2477,15 @@ public class AslExecutor {} catch (FailStateException e) {// A ResultWriter failure fails the Map run on AWS.publishMapRunFailedEvent(chain, e);- recordMapRun(mapRunRecord, "FAILED", succeededItems.get(), failedItems.get());+ recordMapRun(mapRunRecord, "FAILED", succeededItems.get(), failedItems.get(),+ succeededExecutions.get(), failedExecutions.get());throw e;}}if (distributed) {- recordMapRun(mapRunRecord, "SUCCEEDED", succeededItems.get(), failedItems.get());+ recordMapRun(mapRunRecord, "SUCCEEDED", succeededItems.get(), failedItems.get(),+ succeededExecutions.get(), failedExecutions.get());chain.publishAside("MapRunSucceeded", null);} else {chain.continueAfter(iterationChains);@@ -2495,29 +2506,35 @@ public class AslExecutor {return new StateResult(output, stateDef.path("Next").asText(null));}- private int resolveMapMaxConcurrency(JsonNode stateDef, JsonNode mapInput, boolean jsonata,- JsonNode context, ObjectNode variables) {+ /**+ * Resolves an integer Map field from its literal, {@code <field>Path} or JSONata expression form,+ * as MaxConcurrency and the ItemBatcher limits all take. An absent field is 0. {@code minimum} is+ * the smallest accepted value, which is what separates MaxConcurrency, where 0 means the service+ * ceiling, from a batch limit, where it is meaningless.+ */+ private int resolveMapIntegerField(JsonNode container, String field, int minimum, JsonNode mapInput,+ boolean jsonata, JsonNode context, ObjectNode variables) {JsonNode value;boolean jsonataExpression = false;- if (stateDef.has("MaxConcurrencyPath")) {- value = resolvePath(stateDef.get("MaxConcurrencyPath").asText(), mapInput);- } else if (stateDef.has("MaxConcurrency")) {- value = stateDef.get("MaxConcurrency");+ if (container.has(field + "Path")) {+ value = resolvePath(container.get(field + "Path").asText(), mapInput);+ } else if (container.has(field)) {+ value = container.get(field);if (jsonata && value.isTextual() && JsonataEvaluator.isExpression(value.asText())) {jsonataExpression = true;JsonNode statesVar = buildStatesVar(mapInput, null, context);- value = jsonataEvaluator.evaluateField(value.asText(), "MaxConcurrency", statesVar, variables);+ value = jsonataEvaluator.evaluateField(value.asText(), field, statesVar, variables);}} else {return 0;}- if (!value.isIntegralNumber() || value.bigIntegerValue().signum() < 0) {+ if (!value.isIntegralNumber() || value.bigIntegerValue().compareTo(BigInteger.valueOf(minimum)) < 0) {throw new FailStateException(jsonataExpression ? "States.QueryEvaluationError" : "States.Runtime",- "MaxConcurrency must resolve to a non-negative integer", "MaxConcurrency");+ field + " must resolve to an integer of " + minimum + " or more", field);}- return value.bigIntegerValue().compareTo(java.math.BigInteger.valueOf(Integer.MAX_VALUE)) > 0+ return value.bigIntegerValue().compareTo(BigInteger.valueOf(Integer.MAX_VALUE)) > 0? Integer.MAX_VALUE: value.intValue();}@@ -2543,12 +2560,13 @@ public class AslExecutor {* stops at that same instant.*/private static MapRun newMapRun(MapRunIdentity identity, JsonNode context, int itemCount,- int requestedConcurrency) {+ int executionCount, int requestedConcurrency) {var mapRun = new MapRun();mapRun.setMapRunArn(identity.arn());mapRun.setExecutionArn(context.path("Execution").path("Id").asText(null));mapRun.setStartDate(System.currentTimeMillis() / 1000.0);mapRun.setItemCount(itemCount);+ mapRun.setExecutionCount(executionCount);// ASL spells an unbounded Map as MaxConcurrency 0, or by omitting it; DescribeMapRun// reports that same run as Integer.MAX_VALUE.mapRun.setMaxConcurrency(@@ -2557,14 +2575,113 @@ public class AslExecutor {}/** Kept for every Distributed Map, so the mapRunArn in the history resolves through DescribeMapRun. */- private void recordMapRun(MapRun mapRun, String status, int succeededItems, int failedItems) {+ private void recordMapRun(MapRun mapRun, String status, int succeededItems, int failedItems,+ int succeededExecutions, int failedExecutions) {mapRun.setStopDate(System.currentTimeMillis() / 1000.0);mapRun.setStatus(status);mapRun.setSucceededCount(succeededItems);mapRun.setFailedCount(failedItems);+ mapRun.setSucceededExecutionCount(succeededExecutions);+ mapRun.setFailedExecutionCount(failedExecutions);sfnService.get().recordMapRun(mapRun);}+ /** The $$.Map.Item context one iteration sees: its index, its value, and its key for an object dataset. */+ private ObjectNode mapItemContext(JsonNode context, ResolvedMapItems resolvedItems, JsonNode item, int index) {+ ObjectNode iterContext = ((ObjectNode) context).deepCopy();+ ObjectNode mapCtx = objectMapper.createObjectNode();+ ObjectNode mapItem = objectMapper.createObjectNode();+ mapItem.put("Index", index);+ if (resolvedItems.source() == MapItemsSource.ITEM_READER_OBJECT) {+ mapItem.put("Key", item.path("Key").asText());+ mapItem.set("Value", item.get("Value"));+ } else {+ mapItem.set("Value", item);+ }+ mapCtx.set("Item", mapItem);+ iterContext.set("Map", mapCtx);+ return iterContext;+ }++ /**+ * Groups the items into batches of {@code {"BatchInput": ..., "Items": [...]}}, closing a batch on+ * MaxItemsPerBatch, on MaxInputBytesPerBatch, or on the 256 KiB child-input ceiling AWS applies+ * whether or not a byte limit is declared. The size measured is the serialized child payload,+ * envelope and BatchInput included, not the items alone.+ */+ private List<JsonNode> buildItemBatches(JsonNode stateDef, JsonNode items, ResolvedMapItems resolvedItems,+ JsonNode itemTransform, JsonNode mapInput, boolean jsonata,+ JsonNode context, ObjectNode variables) throws Exception {+ JsonNode batcher = stateDef.get("ItemBatcher");+ int maxItemsPerBatch = resolveMapIntegerField(+ batcher, "MaxItemsPerBatch", 1, mapInput, jsonata, context, variables);+ int maxBytesPerBatch = resolveMapIntegerField(+ batcher, "MaxInputBytesPerBatch", 1, mapInput, jsonata, context, variables);++ JsonNode batchInput = null;+ if (batcher.has("BatchInput")) {+ batchInput = jsonata+ ? jsonataEvaluator.resolveTemplate(batcher.get("BatchInput"), "ItemBatcher/BatchInput",+ buildStatesVar(mapInput, null, context), variables)+ : resolveParameters(batcher.get("BatchInput"), mapInput, context);+ }++ int byteCeiling = maxBytesPerBatch > 0+ ? Math.min(maxBytesPerBatch, MAX_BATCH_INPUT_BYTES)+ : MAX_BATCH_INPUT_BYTES;+ int envelopeBytes = serializedBytes(newBatch(batchInput, objectMapper.createArrayNode()));++ List<JsonNode> batches = new ArrayList<>();+ ArrayNode current = objectMapper.createArrayNode();+ int currentBytes = envelopeBytes;+ for (int i = 0; i < items.size(); i++) {+ JsonNode item = items.get(i);+ JsonNode childItem = item;+ if (itemTransform != null) {+ childItem = resolveParameters(itemTransform, mapInput,+ mapItemContext(context, resolvedItems, item, i));+ }+ // An item that cannot fit a batch even on its own can never start a child execution, so+ // the run fails rather than exporting a batch AWS would reject.+ int aloneBytes = envelopeBytes + serializedBytes(childItem);+ if (aloneBytes > MAX_BATCH_INPUT_BYTES) {+ throw new FailStateException("States.DataLimitExceeded",+ "The item at index " + i + " is " + aloneBytes + " bytes as a child input, over the "+ + MAX_BATCH_INPUT_BYTES + " byte maximum. Reduce it with ItemSelector.");+ }+ // The separator this item adds once it is not the first element of the array.+ int itemBytes = serializedBytes(childItem) + (current.size() > 0 ? 1 : 0);+ boolean itemsFull = maxItemsPerBatch > 0 && current.size() >= maxItemsPerBatch;+ boolean bytesFull = currentBytes + itemBytes > byteCeiling;+ if (current.size() > 0 && (itemsFull || bytesFull)) {+ batches.add(newBatch(batchInput, current));+ current = objectMapper.createArrayNode();+ currentBytes = envelopeBytes;+ itemBytes = serializedBytes(childItem);+ }+ current.add(childItem);+ currentBytes += itemBytes;+ }+ if (current.size() > 0) {+ batches.add(newBatch(batchInput, current));+ }+ return batches;+ }++ private int serializedBytes(JsonNode node) {+ return node.toString().getBytes(StandardCharsets.UTF_8).length;+ }++ private JsonNode newBatch(JsonNode batchInput, ArrayNode batchItems) {+ ObjectNode batch = objectMapper.createObjectNode();+ if (batchInput != null) {+ batch.set("BatchInput", batchInput);+ }+ batch.set("Items", batchItems);+ return batch;+ }++/*** Emulates a Distributed Map state's {@code ResultWriter}* (<a href="https://docs.aws.amazon.com/step-functions/latest/dg/input-output-resultwriter.html">AWS docs</a>).diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.javaindex 368b000c3d..04ae29f710 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java@@ -406,24 +406,24 @@ public class StepFunctionsJsonHandler {response.put("maxConcurrency", mapRun.getMaxConcurrency());response.put("toleratedFailurePercentage", 0.0);response.put("toleratedFailureCount", 0);- putMapRunCounts(response.putObject("itemCounts"), mapRun);- // One child execution per item: ItemBatcher is not applied, so no execution covers a batch.- putMapRunCounts(response.putObject("executionCounts"), mapRun);+ putMapRunCounts(response.putObject("itemCounts"), mapRun.getItemCount(),+ mapRun.getSucceededCount(), mapRun.getFailedCount());+ // An ItemBatcher run has one execution per batch, so the two blocks differ there.+ putMapRunCounts(response.putObject("executionCounts"), mapRun.getExecutionCount(),+ mapRun.getSucceededExecutionCount(), mapRun.getFailedExecutionCount());response.put("redriveCount", 0);return response;}/** A failed item's result counts as written, as on AWS. */- private static void putMapRunCounts(ObjectNode counts, MapRun mapRun) {- var succeeded = mapRun.getSucceededCount();- var failed = mapRun.getFailedCount();+ private static void putMapRunCounts(ObjectNode counts, int total, int succeeded, int failed) {counts.put("pending", 0);counts.put("running", 0);counts.put("succeeded", succeeded);counts.put("failed", failed);counts.put("timedOut", 0);- counts.put("aborted", mapRun.getItemCount() - succeeded - failed);- counts.put("total", mapRun.getItemCount());+ counts.put("aborted", total - succeeded - failed);+ counts.put("total", total);counts.put("resultsWritten", succeeded + failed);counts.put("failuresNotRedrivable", 0);counts.put("pendingRedrive", 0);diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/MapRun.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/MapRun.javaindex a45a5e8bd2..1e5244c161 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/MapRun.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/MapRun.java@@ -23,6 +23,10 @@ public class MapRun {private int succeededCount;private int failedCount;/** The Map's declared MaxConcurrency, with an unbounded Map held as Integer.MAX_VALUE. */+ private int executionCount;+ private int succeededExecutionCount;+ private int failedExecutionCount;+private int maxConcurrency;public String getMapRunArn() { return mapRunArn; }@@ -47,6 +51,13 @@ public class MapRun {public int getFailedCount() { return failedCount; }public void setFailedCount(int failedCount) { this.failedCount = failedCount; }+ public int getExecutionCount() { return executionCount; }+ public void setExecutionCount(int executionCount) { this.executionCount = executionCount; }+ public int getSucceededExecutionCount() { return status == null ? executionCount : succeededExecutionCount; }+ public void setSucceededExecutionCount(int count) { this.succeededExecutionCount = count; }+ public int getFailedExecutionCount() { return failedExecutionCount; }+ public void setFailedExecutionCount(int count) { this.failedExecutionCount = count; }+public int getMaxConcurrency() { return maxConcurrency; }public void setMaxConcurrency(int maxConcurrency) { this.maxConcurrency = maxConcurrency; }}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsDescribeMapRunIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsDescribeMapRunIntegrationTest.javaindex d87f041b5a..4029c98c9d 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsDescribeMapRunIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsDescribeMapRunIntegrationTest.java@@ -292,6 +292,48 @@ class StepFunctionsDescribeMapRunIntegrationTest {assertEquals(10, counts.size(), "a counter set carries exactly ten fields");}+ /**+ * itemCounts counts items and executionCounts counts child executions, which an ItemBatcher run+ * is the only shape to tell apart. The split follows the documented ItemBatcher semantics rather+ * than a measured run.+ */+ @Test+ @Order(12)+ void describeMapRunCountsBatchedItemsAndExecutionsSeparately() throws Exception {+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "End": true,+ "Items": [{"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}, {"n": 5}],+ "ItemBatcher": {"MaxItemsPerBatch": 2},+ "ItemProcessor": {+ "ProcessorConfig": {"Mode": "DISTRIBUTED", "ExecutionType": "STANDARD"},+ "StartAt": "Keep",+ "States": {"Keep": {"Type": "Pass", "End": true}}+ }+ }+ }+ }+ """;++ String smArn = createStateMachine("describe-map-run-batched", definition);+ String execArn = startExecution(smArn, "{}");+ Response describe = waitForTerminalState(execArn);+ assertEquals("SUCCEEDED", describe.jsonPath().getString("status"),+ "cause: " + describe.jsonPath().getString("cause"));++ Response response = describeMapRun(mapRunArnFromHistory(execArn));+ response.then().statusCode(200);+ assertEquals(5, response.jsonPath().getInt("itemCounts.total"));+ assertEquals(5, response.jsonPath().getInt("itemCounts.succeeded"));+ assertEquals(3, response.jsonPath().getInt("executionCounts.total"));+ assertEquals(3, response.jsonPath().getInt("executionCounts.succeeded"));+ }+private static String distributedMap(String items, String bucket, Integer maxConcurrency) {return """{diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.javaindex cea67f39b3..60eeaae2e6 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java@@ -942,6 +942,291 @@ class StepFunctionsJsonataIntegrationTest {assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));}+ @Test+ void distributedMapWithItemBatcher_groupsItemsIntoBatchesOfMaxItemsPerBatch() throws Exception {+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "Items": [{"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}, {"n": 5}],+ "ItemBatcher": {"MaxItemsPerBatch": 2},+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "Keep",+ "States": {+ "Keep": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-item-batcher-max-items", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);+ JsonNode batches = new ObjectMapper().readTree(output);++ assertEquals(3, batches.size());+ assertEquals(2, batches.get(0).path("Items").size());+ assertEquals(2, batches.get(1).path("Items").size());+ assertEquals(1, batches.get(2).path("Items").size());+ assertEquals(1, batches.get(0).path("Items").get(0).path("n").asInt());+ }++ @Test+ void distributedMapWithItemBatcher_mergesBatchInputIntoEveryBatch() throws Exception {+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "Items": [{"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}, {"n": 5}],+ "ItemBatcher": {"MaxItemsPerBatch": 2, "BatchInput": {"runType": "nightly"}},+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "Keep",+ "States": {+ "Keep": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-item-batcher-batch-input", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);+ JsonNode batches = new ObjectMapper().readTree(output);++ assertEquals(3, batches.size());+ for (JsonNode batch : batches) {+ assertEquals("nightly", batch.path("BatchInput").path("runType").asText());+ }+ }++ @Test+ void distributedMapWithItemBatcher_startsANewBatchOnMaxInputBytesPerBatch() throws Exception {+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "Items": [{"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}, {"n": 5}],+ "ItemBatcher": {"MaxInputBytesPerBatch": 20},+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "Keep",+ "States": {+ "Keep": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-item-batcher-max-bytes", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);+ JsonNode batches = new ObjectMapper().readTree(output);++ assertTrue(batches.size() > 1, "expected the byte limit to split the items: " + output);+ for (JsonNode batch : batches) {+ assertTrue(batch.path("Items").size() >= 1);+ }+ }++ @Test+ void distributedMapWithoutItemBatcher_stillGivesEachChildOneItem() throws Exception {+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "Items": [{"n": 1}, {"n": 2}],+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "Keep",+ "States": {+ "Keep": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-item-batcher-absent", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);+ JsonNode children = new ObjectMapper().readTree(output);++ assertEquals(2, children.size());+ assertEquals(1, children.get(0).path("n").asInt());+ assertTrue(children.get(0).path("Items").isMissingNode());+ }++ @Test+ void distributedMapWithItemBatcher_capsABatchAtTheChildInputCeiling() throws Exception {+ createBucket("map-inputs-large");+ StringBuilder dataset = new StringBuilder("[");+ for (int i = 0; i < 40; i++) {+ if (i > 0) {+ dataset.append(",");+ }+ dataset.append("{\"pad\":\"").append("x".repeat(10_000)).append("\"}");+ }+ dataset.append("]");+ putObject("map-inputs-large", "large.json", dataset.toString());++ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "ItemReader": {+ "Resource": "arn:aws:states:::s3:getObject",+ "ReaderConfig": {+ "InputType": "JSON"+ },+ "Arguments": {+ "Bucket": "map-inputs-large",+ "Key": "large.json"+ }+ },+ "ItemBatcher": {+ "MaxItemsPerBatch": 100+ },+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "Count",+ "States": {+ "Count": {+ "Type": "Pass",+ "Output": "{% { 'n': $count($states.input.Items) } %}",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-item-batcher-ceiling", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);+ JsonNode batches = new ObjectMapper().readTree(output);++ // 40 items of about 10 KB each exceed 256 KiB, so MaxItemsPerBatch 100 cannot be the only limit.+ assertTrue(batches.size() > 1, "the ceiling must split the run: " + batches.size() + " batches");+ int counted = 0;+ for (JsonNode batch : batches) {+ int size = batch.path("n").asInt();+ assertTrue(size * 10_000 < 256 * 1024, "batch of " + size + " items exceeds the ceiling");+ counted += size;+ }+ assertEquals(40, counted);+ }++ @Test+ void distributedMapWithItemBatcher_failsWhenASingleItemExceedsTheCeiling() throws Exception {+ createBucket("map-inputs-oversized");+ putObject("map-inputs-oversized", "oversized.json",+ "[{\"pad\":\"" + "x".repeat(300_000) + "\"}]");++ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "ItemReader": {+ "Resource": "arn:aws:states:::s3:getObject",+ "ReaderConfig": {+ "InputType": "JSON"+ },+ "Arguments": {+ "Bucket": "map-inputs-oversized",+ "Key": "oversized.json"+ }+ },+ "ItemBatcher": {+ "MaxItemsPerBatch": 10+ },+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "Keep",+ "States": {+ "Keep": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-item-batcher-oversized", definition);+ String execArn = startExecution(smArn, "{}");+ Response failure = waitForExecutionFailure(execArn);++ assertEquals("FAILED", failure.jsonPath().getString("status"));+ assertEquals("States.DataLimitExceeded", failure.jsonPath().getString("error"));+ }+@Testvoid distributedMapWithS3JsonItemReader_objectIteratesKeyValuePairs() throws Exception {createBucket("map-inputs-object");