Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(sfn): apply ItemBatcher to distributed Map children (#3604)

Ovidiu (Ovi)917e94914ff7
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.md
index de6e2a87f9..9519a82bba 100644
--- a/docs/services/step-functions.md
+++ b/docs/services/step-functions.md
@@ -124,6 +124,24 @@ Results remain in input order even when iterations finish out of order. If an it
the Map state fails promptly, cancels its active sibling iterations, and does not start queued
iterations.
+## Distributed Map ItemBatcher
+
+`ItemBatcher` hands each child execution a batch of items instead of a single item. The child input
+is `{"BatchInput": ..., "Items": [...]}`, with `BatchInput` present only when the state declares it.
+`ItemSelector` still runs per item, before the items are grouped.
+
+A batch closes on `MaxItemsPerBatch`, on `MaxInputBytesPerBatch`, or on the 256 KiB child-input
+ceiling AWS applies whether or not a byte limit is declared. Either limit may be given as a
+`...Path` field, or as an expression in a JSONata state machine. With neither declared, items fill
+one batch up to that ceiling. The size measured is the serialized child payload, envelope and
+`BatchInput` included, not the items alone. An item that would exceed the ceiling on its own can
+never start a child execution, so the state fails with `States.DataLimitExceeded` rather than
+building a batch AWS would reject: reduce the item with `ItemSelector` first.
+
+`MaxConcurrency` then bounds concurrent batches, and the Map result has one entry per batch rather
+than per item. `DescribeMapRun` reports items under `itemCounts` and batches under
+`executionCounts`.
+
## Retry policies
`Task`, `Parallel`, and `Map` states honor their `Retry` field. `ErrorEquals` matching
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index a2ca923bf3..1e071c5ed9 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -112,6 +112,9 @@ import java.util.stream.Collectors;
@ApplicationScoped
public class AslExecutor {
+ /** AWS starts no child execution with an input over 256 KiB, batched or not. */
+ private static final int MAX_BATCH_INPUT_BYTES = 256 * 1024;
+
private enum MapItemsSource {
DEFAULT,
ITEM_READER_ARRAY,
@@ -2352,43 +2355,46 @@ public class AslExecutor {
ArrayNode results = objectMapper.createArrayNode();
int itemCount = items.size();
- JsonNode[] childInputsByIndex = hasResultWriter ? new JsonNode[itemCount] : null;
- long[][] childTimingsByIndex = hasResultWriter ? new long[itemCount][] : null;
- int requestedConcurrency = resolveMapMaxConcurrency(
- stateDef, mapInput, jsonata, context, variables);
+ // An ItemBatcher gives each child execution a batch of items rather than one item, so the
+ // children the scheduler runs are the batches. ItemSelector has already been applied to
+ // each item inside them, as on AWS.
+ List<JsonNode> batches = stateDef.has("ItemBatcher")
+ ? buildItemBatches(stateDef, items, resolvedItems, itemTransform, mapInput, jsonata,
+ context, variables)
+ : null;
+ int childCount = batches == null ? itemCount : batches.size();
+ JsonNode[] childInputsByIndex = hasResultWriter ? new JsonNode[childCount] : null;
+ long[][] childTimingsByIndex = hasResultWriter ? new long[childCount][] : null;
+ int requestedConcurrency = resolveMapIntegerField(
+ stateDef, "MaxConcurrency", 0, mapInput, jsonata, context, variables);
int effectiveConcurrency = effectiveMapConcurrency(
- itemCount, requestedConcurrency, distributed);
+ childCount, requestedConcurrency, distributed);
chain.publish("MapStateStarted", Map.of("length", itemCount));
MapRunIdentity mapRun = null;
MapRun mapRunRecord = null;
if (distributed) {
mapRun = newMapRunIdentity(stateDef, sm, context);
- mapRunRecord = newMapRun(mapRun, context, itemCount, requestedConcurrency);
+ mapRunRecord = newMapRun(mapRun, context, itemCount, childCount, requestedConcurrency);
chain.publish("MapRunStarted", Map.of("mapRunArn", mapRun.arn()));
}
var succeededItems = new AtomicInteger();
var failedItems = new AtomicInteger();
- var iterationChains = new ArrayList<HistoryChain>(itemCount);
- for (var i = 0; i < itemCount; i++) {
+ AtomicInteger succeededExecutions = new AtomicInteger();
+ AtomicInteger failedExecutions = new AtomicInteger();
+ var iterationChains = new ArrayList<HistoryChain>(childCount);
+ for (var i = 0; i < childCount; i++) {
iterationChains.add(distributed ? HistoryChain.ofChildExecution() : chain.fork());
}
java.util.function.IntFunction<Callable<JsonNode>> makeTask = (i) -> () -> {
var iterationChain = iterationChains.get(i);
- JsonNode item = items.get(i);
- ObjectNode iterContext = ((ObjectNode) context).deepCopy();
- ObjectNode mapCtx = objectMapper.createObjectNode();
- ObjectNode mapItem = objectMapper.createObjectNode();
- mapItem.put("Index", i);
- if (resolvedItems.source() == MapItemsSource.ITEM_READER_OBJECT) {
- mapItem.put("Key", item.path("Key").asText());
- mapItem.set("Value", item.get("Value"));
- } else {
- mapItem.set("Value", item);
- }
- mapCtx.set("Item", mapItem);
- iterContext.set("Map", mapCtx);
+ boolean batchedChild = batches != null;
+ JsonNode item = batchedChild ? batches.get(i) : items.get(i);
+ int itemsInChild = batchedChild ? item.path("Items").size() : 1;
+ ObjectNode iterContext = batchedChild
+ ? ((ObjectNode) context).deepCopy()
+ : mapItemContext(context, resolvedItems, items.get(i), i);
long startMs = hasResultWriter ? System.currentTimeMillis() : 0L;
JsonNode branchOutput;
@@ -2397,7 +2403,7 @@ public class AslExecutor {
iterationChain.publish("MapIterationStarted", Map.of("name", name, "index", i));
}
JsonNode iterInput = item;
- if (itemTransform != null) {
+ if (!batchedChild && itemTransform != null) {
// $ in ItemSelector resolves against the Map state's effective input, not the item.
iterInput = resolveParameters(itemTransform, mapInput, iterContext);
}
@@ -2412,13 +2418,15 @@ public class AslExecutor {
branchOutput = executeBranch(startAt, iteratorStates, iterInput, iterationChain, sm,
topLevelQueryLanguage, iterContext, variables.deepCopy());
} catch (FailStateException e) {
- failedItems.incrementAndGet();
+ failedItems.addAndGet(itemsInChild);
+ failedExecutions.incrementAndGet();
if (!distributed && !e.isRuntimeError()) {
iterationChain.publishAside("MapIterationFailed", Map.of("name", name, "index", i));
}
throw new IterationFailure(i, e);
}
- succeededItems.incrementAndGet();
+ succeededItems.addAndGet(itemsInChild);
+ succeededExecutions.incrementAndGet();
if (!distributed) {
iterationChain.publish("MapIterationSucceeded", Map.of("name", name, "index", i));
}
@@ -2428,11 +2436,11 @@ public class AslExecutor {
return branchOutput;
};
- if (itemCount > 0) {
+ if (childCount > 0) {
List<JsonNode> itemOutputs;
try {
itemOutputs = MapIterationScheduler.execute(
- itemCount, Math.max(1, effectiveConcurrency),
+ childCount, Math.max(1, effectiveConcurrency),
i -> () -> callUnderExecutionAccount(sm, makeTask.apply(i)),
executionDeadlineNanos);
} catch (java.util.concurrent.TimeoutException e) {
@@ -2445,7 +2453,8 @@ public class AslExecutor {
chain.continueFrom(iterationChains.get(e.index).lastEventId());
} else {
publishMapRunFailedEvent(chain, e.failure);
- recordMapRun(mapRunRecord, "FAILED", succeededItems.get(), failedItems.get());
+ recordMapRun(mapRunRecord, "FAILED", succeededItems.get(), failedItems.get(),
+ succeededExecutions.get(), failedExecutions.get());
}
throw e.failure;
} finally {
@@ -2457,8 +2466,8 @@ public class AslExecutor {
JsonNode mapResult = results;
if (hasResultWriter) {
ArrayNode childInputs = objectMapper.createArrayNode();
- List<long[]> childTimings = new ArrayList<>(itemCount);
- for (int i = 0; i < itemCount; i++) {
+ List<long[]> childTimings = new ArrayList<>(childCount);
+ for (int i = 0; i < childCount; i++) {
childInputs.add(childInputsByIndex[i]);
childTimings.add(childTimingsByIndex[i]);
}
@@ -2468,13 +2477,15 @@ public class AslExecutor {
} catch (FailStateException e) {
// A ResultWriter failure fails the Map run on AWS.
publishMapRunFailedEvent(chain, e);
- recordMapRun(mapRunRecord, "FAILED", succeededItems.get(), failedItems.get());
+ recordMapRun(mapRunRecord, "FAILED", succeededItems.get(), failedItems.get(),
+ succeededExecutions.get(), failedExecutions.get());
throw e;
}
}
if (distributed) {
- recordMapRun(mapRunRecord, "SUCCEEDED", succeededItems.get(), failedItems.get());
+ recordMapRun(mapRunRecord, "SUCCEEDED", succeededItems.get(), failedItems.get(),
+ succeededExecutions.get(), failedExecutions.get());
chain.publishAside("MapRunSucceeded", null);
} else {
chain.continueAfter(iterationChains);
@@ -2495,29 +2506,35 @@ public class AslExecutor {
return new StateResult(output, stateDef.path("Next").asText(null));
}
- private int resolveMapMaxConcurrency(JsonNode stateDef, JsonNode mapInput, boolean jsonata,
- JsonNode context, ObjectNode variables) {
+ /**
+ * Resolves an integer Map field from its literal, {@code <field>Path} or JSONata expression form,
+ * as MaxConcurrency and the ItemBatcher limits all take. An absent field is 0. {@code minimum} is
+ * the smallest accepted value, which is what separates MaxConcurrency, where 0 means the service
+ * ceiling, from a batch limit, where it is meaningless.
+ */
+ private int resolveMapIntegerField(JsonNode container, String field, int minimum, JsonNode mapInput,
+ boolean jsonata, JsonNode context, ObjectNode variables) {
JsonNode value;
boolean jsonataExpression = false;
- if (stateDef.has("MaxConcurrencyPath")) {
- value = resolvePath(stateDef.get("MaxConcurrencyPath").asText(), mapInput);
- } else if (stateDef.has("MaxConcurrency")) {
- value = stateDef.get("MaxConcurrency");
+ if (container.has(field + "Path")) {
+ value = resolvePath(container.get(field + "Path").asText(), mapInput);
+ } else if (container.has(field)) {
+ value = container.get(field);
if (jsonata && value.isTextual() && JsonataEvaluator.isExpression(value.asText())) {
jsonataExpression = true;
JsonNode statesVar = buildStatesVar(mapInput, null, context);
- value = jsonataEvaluator.evaluateField(value.asText(), "MaxConcurrency", statesVar, variables);
+ value = jsonataEvaluator.evaluateField(value.asText(), field, statesVar, variables);
}
} else {
return 0;
}
- if (!value.isIntegralNumber() || value.bigIntegerValue().signum() < 0) {
+ if (!value.isIntegralNumber() || value.bigIntegerValue().compareTo(BigInteger.valueOf(minimum)) < 0) {
throw new FailStateException(
jsonataExpression ? "States.QueryEvaluationError" : "States.Runtime",
- "MaxConcurrency must resolve to a non-negative integer", "MaxConcurrency");
+ field + " must resolve to an integer of " + minimum + " or more", field);
}
- return value.bigIntegerValue().compareTo(java.math.BigInteger.valueOf(Integer.MAX_VALUE)) > 0
+ return value.bigIntegerValue().compareTo(BigInteger.valueOf(Integer.MAX_VALUE)) > 0
? Integer.MAX_VALUE
: value.intValue();
}
@@ -2543,12 +2560,13 @@ public class AslExecutor {
* stops at that same instant.
*/
private static MapRun newMapRun(MapRunIdentity identity, JsonNode context, int itemCount,
- int requestedConcurrency) {
+ int executionCount, int requestedConcurrency) {
var mapRun = new MapRun();
mapRun.setMapRunArn(identity.arn());
mapRun.setExecutionArn(context.path("Execution").path("Id").asText(null));
mapRun.setStartDate(System.currentTimeMillis() / 1000.0);
mapRun.setItemCount(itemCount);
+ mapRun.setExecutionCount(executionCount);
// ASL spells an unbounded Map as MaxConcurrency 0, or by omitting it; DescribeMapRun
// reports that same run as Integer.MAX_VALUE.
mapRun.setMaxConcurrency(
@@ -2557,14 +2575,113 @@ public class AslExecutor {
}
/** Kept for every Distributed Map, so the mapRunArn in the history resolves through DescribeMapRun. */
- private void recordMapRun(MapRun mapRun, String status, int succeededItems, int failedItems) {
+ private void recordMapRun(MapRun mapRun, String status, int succeededItems, int failedItems,
+ int succeededExecutions, int failedExecutions) {
mapRun.setStopDate(System.currentTimeMillis() / 1000.0);
mapRun.setStatus(status);
mapRun.setSucceededCount(succeededItems);
mapRun.setFailedCount(failedItems);
+ mapRun.setSucceededExecutionCount(succeededExecutions);
+ mapRun.setFailedExecutionCount(failedExecutions);
sfnService.get().recordMapRun(mapRun);
}
+ /** The $$.Map.Item context one iteration sees: its index, its value, and its key for an object dataset. */
+ private ObjectNode mapItemContext(JsonNode context, ResolvedMapItems resolvedItems, JsonNode item, int index) {
+ ObjectNode iterContext = ((ObjectNode) context).deepCopy();
+ ObjectNode mapCtx = objectMapper.createObjectNode();
+ ObjectNode mapItem = objectMapper.createObjectNode();
+ mapItem.put("Index", index);
+ if (resolvedItems.source() == MapItemsSource.ITEM_READER_OBJECT) {
+ mapItem.put("Key", item.path("Key").asText());
+ mapItem.set("Value", item.get("Value"));
+ } else {
+ mapItem.set("Value", item);
+ }
+ mapCtx.set("Item", mapItem);
+ iterContext.set("Map", mapCtx);
+ return iterContext;
+ }
+
+ /**
+ * Groups the items into batches of {@code {"BatchInput": ..., "Items": [...]}}, closing a batch on
+ * MaxItemsPerBatch, on MaxInputBytesPerBatch, or on the 256 KiB child-input ceiling AWS applies
+ * whether or not a byte limit is declared. The size measured is the serialized child payload,
+ * envelope and BatchInput included, not the items alone.
+ */
+ private List<JsonNode> buildItemBatches(JsonNode stateDef, JsonNode items, ResolvedMapItems resolvedItems,
+ JsonNode itemTransform, JsonNode mapInput, boolean jsonata,
+ JsonNode context, ObjectNode variables) throws Exception {
+ JsonNode batcher = stateDef.get("ItemBatcher");
+ int maxItemsPerBatch = resolveMapIntegerField(
+ batcher, "MaxItemsPerBatch", 1, mapInput, jsonata, context, variables);
+ int maxBytesPerBatch = resolveMapIntegerField(
+ batcher, "MaxInputBytesPerBatch", 1, mapInput, jsonata, context, variables);
+
+ JsonNode batchInput = null;
+ if (batcher.has("BatchInput")) {
+ batchInput = jsonata
+ ? jsonataEvaluator.resolveTemplate(batcher.get("BatchInput"), "ItemBatcher/BatchInput",
+ buildStatesVar(mapInput, null, context), variables)
+ : resolveParameters(batcher.get("BatchInput"), mapInput, context);
+ }
+
+ int byteCeiling = maxBytesPerBatch > 0
+ ? Math.min(maxBytesPerBatch, MAX_BATCH_INPUT_BYTES)
+ : MAX_BATCH_INPUT_BYTES;
+ int envelopeBytes = serializedBytes(newBatch(batchInput, objectMapper.createArrayNode()));
+
+ List<JsonNode> batches = new ArrayList<>();
+ ArrayNode current = objectMapper.createArrayNode();
+ int currentBytes = envelopeBytes;
+ for (int i = 0; i < items.size(); i++) {
+ JsonNode item = items.get(i);
+ JsonNode childItem = item;
+ if (itemTransform != null) {
+ childItem = resolveParameters(itemTransform, mapInput,
+ mapItemContext(context, resolvedItems, item, i));
+ }
+ // An item that cannot fit a batch even on its own can never start a child execution, so
+ // the run fails rather than exporting a batch AWS would reject.
+ int aloneBytes = envelopeBytes + serializedBytes(childItem);
+ if (aloneBytes > MAX_BATCH_INPUT_BYTES) {
+ throw new FailStateException("States.DataLimitExceeded",
+ "The item at index " + i + " is " + aloneBytes + " bytes as a child input, over the "
+ + MAX_BATCH_INPUT_BYTES + " byte maximum. Reduce it with ItemSelector.");
+ }
+ // The separator this item adds once it is not the first element of the array.
+ int itemBytes = serializedBytes(childItem) + (current.size() > 0 ? 1 : 0);
+ boolean itemsFull = maxItemsPerBatch > 0 && current.size() >= maxItemsPerBatch;
+ boolean bytesFull = currentBytes + itemBytes > byteCeiling;
+ if (current.size() > 0 && (itemsFull || bytesFull)) {
+ batches.add(newBatch(batchInput, current));
+ current = objectMapper.createArrayNode();
+ currentBytes = envelopeBytes;
+ itemBytes = serializedBytes(childItem);
+ }
+ current.add(childItem);
+ currentBytes += itemBytes;
+ }
+ if (current.size() > 0) {
+ batches.add(newBatch(batchInput, current));
+ }
+ return batches;
+ }
+
+ private int serializedBytes(JsonNode node) {
+ return node.toString().getBytes(StandardCharsets.UTF_8).length;
+ }
+
+ private JsonNode newBatch(JsonNode batchInput, ArrayNode batchItems) {
+ ObjectNode batch = objectMapper.createObjectNode();
+ if (batchInput != null) {
+ batch.set("BatchInput", batchInput);
+ }
+ batch.set("Items", batchItems);
+ return batch;
+ }
+
+
/**
* Emulates a Distributed Map state's {@code ResultWriter}
* (<a href="https://docs.aws.amazon.com/step-functions/latest/dg/input-output-resultwriter.html">AWS docs</a>).
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java
index 368b000c3d..04ae29f710 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java
@@ -406,24 +406,24 @@ public class StepFunctionsJsonHandler {
response.put("maxConcurrency", mapRun.getMaxConcurrency());
response.put("toleratedFailurePercentage", 0.0);
response.put("toleratedFailureCount", 0);
- putMapRunCounts(response.putObject("itemCounts"), mapRun);
- // One child execution per item: ItemBatcher is not applied, so no execution covers a batch.
- putMapRunCounts(response.putObject("executionCounts"), mapRun);
+ putMapRunCounts(response.putObject("itemCounts"), mapRun.getItemCount(),
+ mapRun.getSucceededCount(), mapRun.getFailedCount());
+ // An ItemBatcher run has one execution per batch, so the two blocks differ there.
+ putMapRunCounts(response.putObject("executionCounts"), mapRun.getExecutionCount(),
+ mapRun.getSucceededExecutionCount(), mapRun.getFailedExecutionCount());
response.put("redriveCount", 0);
return response;
}
/** A failed item's result counts as written, as on AWS. */
- private static void putMapRunCounts(ObjectNode counts, MapRun mapRun) {
- var succeeded = mapRun.getSucceededCount();
- var failed = mapRun.getFailedCount();
+ private static void putMapRunCounts(ObjectNode counts, int total, int succeeded, int failed) {
counts.put("pending", 0);
counts.put("running", 0);
counts.put("succeeded", succeeded);
counts.put("failed", failed);
counts.put("timedOut", 0);
- counts.put("aborted", mapRun.getItemCount() - succeeded - failed);
- counts.put("total", mapRun.getItemCount());
+ counts.put("aborted", total - succeeded - failed);
+ counts.put("total", total);
counts.put("resultsWritten", succeeded + failed);
counts.put("failuresNotRedrivable", 0);
counts.put("pendingRedrive", 0);
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/MapRun.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/MapRun.java
index a45a5e8bd2..1e5244c161 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/MapRun.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/MapRun.java
@@ -23,6 +23,10 @@ public class MapRun {
private int succeededCount;
private int failedCount;
/** The Map's declared MaxConcurrency, with an unbounded Map held as Integer.MAX_VALUE. */
+ private int executionCount;
+ private int succeededExecutionCount;
+ private int failedExecutionCount;
+
private int maxConcurrency;
public String getMapRunArn() { return mapRunArn; }
@@ -47,6 +51,13 @@ public class MapRun {
public int getFailedCount() { return failedCount; }
public void setFailedCount(int failedCount) { this.failedCount = failedCount; }
+ public int getExecutionCount() { return executionCount; }
+ public void setExecutionCount(int executionCount) { this.executionCount = executionCount; }
+ public int getSucceededExecutionCount() { return status == null ? executionCount : succeededExecutionCount; }
+ public void setSucceededExecutionCount(int count) { this.succeededExecutionCount = count; }
+ public int getFailedExecutionCount() { return failedExecutionCount; }
+ public void setFailedExecutionCount(int count) { this.failedExecutionCount = count; }
+
public int getMaxConcurrency() { return maxConcurrency; }
public void setMaxConcurrency(int maxConcurrency) { this.maxConcurrency = maxConcurrency; }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsDescribeMapRunIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsDescribeMapRunIntegrationTest.java
index d87f041b5a..4029c98c9d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsDescribeMapRunIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsDescribeMapRunIntegrationTest.java
@@ -292,6 +292,48 @@ class StepFunctionsDescribeMapRunIntegrationTest {
assertEquals(10, counts.size(), "a counter set carries exactly ten fields");
}
+ /**
+ * itemCounts counts items and executionCounts counts child executions, which an ItemBatcher run
+ * is the only shape to tell apart. The split follows the documented ItemBatcher semantics rather
+ * than a measured run.
+ */
+ @Test
+ @Order(12)
+ void describeMapRunCountsBatchedItemsAndExecutionsSeparately() throws Exception {
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Fan",
+ "States": {
+ "Fan": {
+ "Type": "Map",
+ "End": true,
+ "Items": [{"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}, {"n": 5}],
+ "ItemBatcher": {"MaxItemsPerBatch": 2},
+ "ItemProcessor": {
+ "ProcessorConfig": {"Mode": "DISTRIBUTED", "ExecutionType": "STANDARD"},
+ "StartAt": "Keep",
+ "States": {"Keep": {"Type": "Pass", "End": true}}
+ }
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("describe-map-run-batched", definition);
+ String execArn = startExecution(smArn, "{}");
+ Response describe = waitForTerminalState(execArn);
+ assertEquals("SUCCEEDED", describe.jsonPath().getString("status"),
+ "cause: " + describe.jsonPath().getString("cause"));
+
+ Response response = describeMapRun(mapRunArnFromHistory(execArn));
+ response.then().statusCode(200);
+ assertEquals(5, response.jsonPath().getInt("itemCounts.total"));
+ assertEquals(5, response.jsonPath().getInt("itemCounts.succeeded"));
+ assertEquals(3, response.jsonPath().getInt("executionCounts.total"));
+ assertEquals(3, response.jsonPath().getInt("executionCounts.succeeded"));
+ }
+
private static String distributedMap(String items, String bucket, Integer maxConcurrency) {
return """
{
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
index cea67f39b3..60eeaae2e6 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
@@ -942,6 +942,291 @@ class StepFunctionsJsonataIntegrationTest {
assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));
}
+ @Test
+ void distributedMapWithItemBatcher_groupsItemsIntoBatchesOfMaxItemsPerBatch() throws Exception {
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Fan",
+ "States": {
+ "Fan": {
+ "Type": "Map",
+ "Items": [{"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}, {"n": 5}],
+ "ItemBatcher": {"MaxItemsPerBatch": 2},
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "Keep",
+ "States": {
+ "Keep": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-item-batcher-max-items", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+ JsonNode batches = new ObjectMapper().readTree(output);
+
+ assertEquals(3, batches.size());
+ assertEquals(2, batches.get(0).path("Items").size());
+ assertEquals(2, batches.get(1).path("Items").size());
+ assertEquals(1, batches.get(2).path("Items").size());
+ assertEquals(1, batches.get(0).path("Items").get(0).path("n").asInt());
+ }
+
+ @Test
+ void distributedMapWithItemBatcher_mergesBatchInputIntoEveryBatch() throws Exception {
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Fan",
+ "States": {
+ "Fan": {
+ "Type": "Map",
+ "Items": [{"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}, {"n": 5}],
+ "ItemBatcher": {"MaxItemsPerBatch": 2, "BatchInput": {"runType": "nightly"}},
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "Keep",
+ "States": {
+ "Keep": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-item-batcher-batch-input", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+ JsonNode batches = new ObjectMapper().readTree(output);
+
+ assertEquals(3, batches.size());
+ for (JsonNode batch : batches) {
+ assertEquals("nightly", batch.path("BatchInput").path("runType").asText());
+ }
+ }
+
+ @Test
+ void distributedMapWithItemBatcher_startsANewBatchOnMaxInputBytesPerBatch() throws Exception {
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Fan",
+ "States": {
+ "Fan": {
+ "Type": "Map",
+ "Items": [{"n": 1}, {"n": 2}, {"n": 3}, {"n": 4}, {"n": 5}],
+ "ItemBatcher": {"MaxInputBytesPerBatch": 20},
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "Keep",
+ "States": {
+ "Keep": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-item-batcher-max-bytes", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+ JsonNode batches = new ObjectMapper().readTree(output);
+
+ assertTrue(batches.size() > 1, "expected the byte limit to split the items: " + output);
+ for (JsonNode batch : batches) {
+ assertTrue(batch.path("Items").size() >= 1);
+ }
+ }
+
+ @Test
+ void distributedMapWithoutItemBatcher_stillGivesEachChildOneItem() throws Exception {
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Fan",
+ "States": {
+ "Fan": {
+ "Type": "Map",
+ "Items": [{"n": 1}, {"n": 2}],
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "Keep",
+ "States": {
+ "Keep": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-item-batcher-absent", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+ JsonNode children = new ObjectMapper().readTree(output);
+
+ assertEquals(2, children.size());
+ assertEquals(1, children.get(0).path("n").asInt());
+ assertTrue(children.get(0).path("Items").isMissingNode());
+ }
+
+ @Test
+ void distributedMapWithItemBatcher_capsABatchAtTheChildInputCeiling() throws Exception {
+ createBucket("map-inputs-large");
+ StringBuilder dataset = new StringBuilder("[");
+ for (int i = 0; i < 40; i++) {
+ if (i > 0) {
+ dataset.append(",");
+ }
+ dataset.append("{\"pad\":\"").append("x".repeat(10_000)).append("\"}");
+ }
+ dataset.append("]");
+ putObject("map-inputs-large", "large.json", dataset.toString());
+
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Fan",
+ "States": {
+ "Fan": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Arguments": {
+ "Bucket": "map-inputs-large",
+ "Key": "large.json"
+ }
+ },
+ "ItemBatcher": {
+ "MaxItemsPerBatch": 100
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "Count",
+ "States": {
+ "Count": {
+ "Type": "Pass",
+ "Output": "{% { 'n': $count($states.input.Items) } %}",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-item-batcher-ceiling", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+ JsonNode batches = new ObjectMapper().readTree(output);
+
+ // 40 items of about 10 KB each exceed 256 KiB, so MaxItemsPerBatch 100 cannot be the only limit.
+ assertTrue(batches.size() > 1, "the ceiling must split the run: " + batches.size() + " batches");
+ int counted = 0;
+ for (JsonNode batch : batches) {
+ int size = batch.path("n").asInt();
+ assertTrue(size * 10_000 < 256 * 1024, "batch of " + size + " items exceeds the ceiling");
+ counted += size;
+ }
+ assertEquals(40, counted);
+ }
+
+ @Test
+ void distributedMapWithItemBatcher_failsWhenASingleItemExceedsTheCeiling() throws Exception {
+ createBucket("map-inputs-oversized");
+ putObject("map-inputs-oversized", "oversized.json",
+ "[{\"pad\":\"" + "x".repeat(300_000) + "\"}]");
+
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Fan",
+ "States": {
+ "Fan": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Arguments": {
+ "Bucket": "map-inputs-oversized",
+ "Key": "oversized.json"
+ }
+ },
+ "ItemBatcher": {
+ "MaxItemsPerBatch": 10
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "Keep",
+ "States": {
+ "Keep": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-item-batcher-oversized", definition);
+ String execArn = startExecution(smArn, "{}");
+ Response failure = waitForExecutionFailure(execArn);
+
+ assertEquals("FAILED", failure.jsonPath().getString("status"));
+ assertEquals("States.DataLimitExceeded", failure.jsonPath().getString("error"));
+ }
+
@Test
void distributedMapWithS3JsonItemReader_objectIteratesKeyValuePairs() throws Exception {
createBucket("map-inputs-object");