Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(stepfunctions): run Map iterations concurrently honoring MaxConcurrency (#1810)

Alex Bannaa7da1b98566c
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.md
index f773052c6e..456448d61b 100644
--- a/docs/services/step-functions.md
+++ b/docs/services/step-functions.md
@@ -35,6 +35,17 @@
| `UntagResource` | - |
<!-- floci:actions:end -->
+## Map concurrency
+
+Map states honor `MaxConcurrency` and, for JSONPath state machines, `MaxConcurrencyPath`.
+JSONata state machines may supply `MaxConcurrency` as an expression. A value of `0`, or an
+omitted value, uses the AWS service ceiling: 40 concurrent iterations for Inline Map states and
+10,000 for Distributed Map states. `MaxConcurrency: 1` runs iterations sequentially.
+
+Results remain in input order even when iterations finish out of order. If an iteration fails,
+the Map state fails promptly, cancels its active sibling iterations, and does not start queued
+iterations.
+
## Configuration
| Variable | Default | Description |
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index b986e88c9f..77b2251e5c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -93,6 +93,8 @@ public class AslExecutor {
private static final Logger LOG = Logger.getLogger(AslExecutor.class);
private static final int MAX_WAIT_SECONDS = 30;
+ private static final int INLINE_MAP_MAX_CONCURRENCY = 40;
+ private static final int DISTRIBUTED_MAP_MAX_CONCURRENCY = 10_000;
// ecs:runTask.sync polling — wait up to ~60s for the task to reach STOPPED.
private static final int ECS_SYNC_POLL_ATTEMPTS = 600;
@@ -1450,20 +1452,27 @@ public class AslExecutor {
: Long.MAX_VALUE;
ArrayNode results = objectMapper.createArrayNode();
- for (Future<JsonNode> future : futures) {
- long remainingNanos = deadlineNanos - System.nanoTime();
- if (remainingNanos <= 0) {
- futures.forEach(f -> f.cancel(true));
- throw new FailStateException("States.Timeout",
- "Parallel state timed out after " + timeoutSeconds + " seconds");
- }
- try {
- results.add(future.get(remainingNanos, TimeUnit.NANOSECONDS));
- } catch (java.util.concurrent.TimeoutException e) {
- futures.forEach(f -> f.cancel(true));
- throw new FailStateException("States.Timeout",
- "Parallel state timed out after " + timeoutSeconds + " seconds");
+ try {
+ for (Future<JsonNode> future : futures) {
+ long remainingNanos = deadlineNanos - System.nanoTime();
+ if (remainingNanos <= 0) {
+ throw new FailStateException("States.Timeout",
+ "Parallel state timed out after " + timeoutSeconds + " seconds");
+ }
+ try {
+ results.add(future.get(remainingNanos, TimeUnit.NANOSECONDS));
+ } catch (java.util.concurrent.TimeoutException e) {
+ throw new FailStateException("States.Timeout",
+ "Parallel state timed out after " + timeoutSeconds + " seconds");
+ }
}
+ } catch (InterruptedException e) {
+ futures.forEach(future -> future.cancel(true));
+ Thread.currentThread().interrupt();
+ throw e;
+ } catch (Exception | Error e) {
+ futures.forEach(future -> future.cancel(true));
+ throw e;
}
if (jsonata) {
@@ -1483,24 +1492,21 @@ public class AslExecutor {
private StateResult executeMapState(String name, JsonNode stateDef, JsonNode input, StateMachine sm,
boolean jsonata, String topLevelQueryLanguage, JsonNode context,
ObjectNode variables) throws Exception {
- // ItemReader, ItemBatcher and ResultWriter are distributed-map-only fields; AWS rejects them on
- // an INLINE map. Guard on all three (not just ItemReader) so a ResultWriter on an inline map is
- // rejected rather than silently exporting to S3.
+ String processorMode = stateDef.path("ItemProcessor").path("ProcessorConfig")
+ .path("Mode").asText("INLINE");
+ boolean distributed = "DISTRIBUTED".equals(processorMode);
if (stateDef.has("ItemReader") || stateDef.has("ItemBatcher") || stateDef.has("ResultWriter")) {
- String mode = stateDef.path("ItemProcessor").path("ProcessorConfig").path("Mode").asText("INLINE");
- if (!"DISTRIBUTED".equals(mode)) {
+ if (!distributed) {
throw new FailStateException("States.Runtime",
"The ItemReader, ItemBatcher and ResultWriter fields are not supported for INLINE maps");
}
}
boolean hasResultWriter = stateDef.has("ResultWriter");
- // ResultWriter Parameters and its distributed Map item selection see the effective input
- // after InputPath. Keep the pre-existing item-selection behavior unchanged for Maps that do
- // not use ResultWriter so this feature does not absorb a separate compatibility correction.
- JsonNode mapInput = jsonata ? input : applyInputPath(stateDef, input);
- JsonNode itemSelectionInput = hasResultWriter ? mapInput : input;
- ResolvedMapItems resolvedItems = resolveMapItems(
- stateDef, itemSelectionInput, jsonata, context, variables);
+
+ // Map input-processing fields, including ItemsPath and MaxConcurrencyPath, resolve against
+ // the effective state input after InputPath has been applied.
+ JsonNode mapInput = applyInputPath(stateDef, input);
+ ResolvedMapItems resolvedItems = resolveMapItems(stateDef, mapInput, jsonata, context, variables);
JsonNode items = resolvedItems.items();
if (!items.isArray()) {
@@ -1517,16 +1523,20 @@ public class AslExecutor {
: stateDef.has("Parameters") ? stateDef.get("Parameters") : null;
ArrayNode results = objectMapper.createArrayNode();
- // When a ResultWriter exports results, AWS records each child execution's input and timing
- // (Transformation NONE); capture them alongside the outputs so the export is faithful.
- ArrayNode childInputs = hasResultWriter ? objectMapper.createArrayNode() : null;
- List<long[]> childTimings = hasResultWriter ? new ArrayList<>() : null;
- int index = 0;
- for (JsonNode item : items) {
+ int itemCount = items.size();
+ JsonNode[] childInputsByIndex = hasResultWriter ? new JsonNode[itemCount] : null;
+ long[][] childTimingsByIndex = hasResultWriter ? new long[itemCount][] : null;
+ int requestedConcurrency = resolveMapMaxConcurrency(
+ stateDef, mapInput, jsonata, context, variables);
+ int effectiveConcurrency = effectiveMapConcurrency(
+ itemCount, requestedConcurrency, distributed);
+
+ java.util.function.IntFunction<Callable<JsonNode>> makeTask = (i) -> () -> {
+ JsonNode item = items.get(i);
ObjectNode iterContext = ((ObjectNode) context).deepCopy();
ObjectNode mapCtx = objectMapper.createObjectNode();
ObjectNode mapItem = objectMapper.createObjectNode();
- mapItem.put("Index", index);
+ mapItem.put("Index", i);
if (resolvedItems.source() == MapItemsSource.ITEM_READER_OBJECT) {
mapItem.put("Key", item.path("Key").asText());
mapItem.set("Value", item.get("Value"));
@@ -1542,23 +1552,35 @@ public class AslExecutor {
iterInput = resolveParameters(itemTransform, mapInput, iterContext);
}
// Each iteration gets an isolated copy of the current variables; assignments inside an
- // iteration are scoped to that iteration and do not leak back to the parent scope.
+ // iteration are scoped to that iteration and do not leak back to the parent scope. An
+ // isolated copy per worker also keeps concurrent iterations from racing on shared state.
long startMs = hasResultWriter ? System.currentTimeMillis() : 0L;
- JsonNode branchOutput =
- executeBranch(startAt, iteratorStates, iterInput, sm, topLevelQueryLanguage, iterContext,
- variables.deepCopy());
if (hasResultWriter) {
- childInputs.add(iterInput);
- childTimings.add(new long[]{startMs, System.currentTimeMillis()});
+ childInputsByIndex[i] = iterInput;
}
- results.add(branchOutput);
- index++;
+ JsonNode branchOutput = executeBranch(startAt, iteratorStates, iterInput, sm,
+ topLevelQueryLanguage, iterContext, variables.deepCopy());
+ if (hasResultWriter) {
+ childTimingsByIndex[i] = new long[]{startMs, System.currentTimeMillis()};
+ }
+ return branchOutput;
+ };
+
+ if (itemCount > 0) {
+ List<JsonNode> itemOutputs = MapIterationScheduler.execute(
+ itemCount, Math.max(1, effectiveConcurrency),
+ i -> () -> callUnderExecutionAccount(sm, makeTask.apply(i)));
+ results.addAll(itemOutputs);
}
- // A distributed Map's ResultWriter formats the per-child results and, when an S3 Resource is
- // given, exports them and yields a reference to the manifest instead of the inline array.
JsonNode mapResult = results;
if (hasResultWriter) {
+ ArrayNode childInputs = objectMapper.createArrayNode();
+ List<long[]> childTimings = new ArrayList<>(itemCount);
+ for (int i = 0; i < itemCount; i++) {
+ childInputs.add(childInputsByIndex[i]);
+ childTimings.add(childTimingsByIndex[i]);
+ }
mapResult = applyResultWriter(name, stateDef, mapInput, results, childInputs, childTimings,
sm, context, jsonata, variables);
}
@@ -1577,6 +1599,42 @@ public class AslExecutor {
return new StateResult(output, stateDef.path("Next").asText(null));
}
+ private int resolveMapMaxConcurrency(JsonNode stateDef, JsonNode mapInput, boolean jsonata,
+ JsonNode context, ObjectNode variables) {
+ JsonNode value;
+ boolean jsonataExpression = false;
+ if (stateDef.has("MaxConcurrencyPath")) {
+ value = resolvePath(stateDef.get("MaxConcurrencyPath").asText(), mapInput);
+ } else if (stateDef.has("MaxConcurrency")) {
+ value = stateDef.get("MaxConcurrency");
+ if (jsonata && value.isTextual() && JsonataEvaluator.isExpression(value.asText())) {
+ jsonataExpression = true;
+ JsonNode statesVar = buildStatesVar(mapInput, null, context);
+ value = jsonataEvaluator.evaluate(value.asText(), statesVar, variables);
+ }
+ } else {
+ return 0;
+ }
+
+ if (!value.isIntegralNumber() || value.bigIntegerValue().signum() < 0) {
+ throw new FailStateException(
+ jsonataExpression ? "States.QueryEvaluationError" : "States.Runtime",
+ "MaxConcurrency must resolve to a non-negative integer");
+ }
+ return value.bigIntegerValue().compareTo(java.math.BigInteger.valueOf(Integer.MAX_VALUE)) > 0
+ ? Integer.MAX_VALUE
+ : value.intValue();
+ }
+
+ static int effectiveMapConcurrency(int itemCount, int requestedConcurrency,
+ boolean distributed) {
+ int serviceLimit = distributed
+ ? DISTRIBUTED_MAP_MAX_CONCURRENCY
+ : INLINE_MAP_MAX_CONCURRENCY;
+ int requestedLimit = requestedConcurrency == 0 ? serviceLimit : requestedConcurrency;
+ return Math.min(itemCount, Math.min(requestedLimit, serviceLimit));
+ }
+
/**
* Emulates a Distributed Map state's {@code ResultWriter}
* (<a href="https://docs.aws.amazon.com/step-functions/latest/dg/input-output-resultwriter.html">AWS docs</a>).
@@ -1598,8 +1656,8 @@ public class AslExecutor {
*/
// Package-private for unit testing of the ResultWriter export/format behaviour.
JsonNode applyResultWriter(String mapStateName, JsonNode stateDef, JsonNode input,
- ArrayNode results, ArrayNode childInputs, List<long[]> childTimings,
- StateMachine sm, JsonNode context, boolean jsonata) throws Exception {
+ ArrayNode results, ArrayNode childInputs, List<long[]> childTimings,
+ StateMachine sm, JsonNode context, boolean jsonata) throws Exception {
return applyResultWriter(mapStateName, stateDef, input, results, childInputs, childTimings,
sm, context, jsonata, objectMapper.createObjectNode());
}
@@ -1737,25 +1795,17 @@ public class AslExecutor {
}
}
- /**
- * Formats a Distributed Map's child results per {@code WriterConfig.Transformation}:
- * <ul>
- * <li>{@code NONE} — an array of per-child execution records (output plus execution metadata),
- * as AWS writes when exporting without a WriterConfig;</li>
- * <li>{@code COMPACT} — the child outputs in their original array structure;</li>
- * <li>{@code FLATTEN} — the child outputs with any array outputs flattened into one array.</li>
- * </ul>
- */
+ /** Formats a Distributed Map's child results per {@code WriterConfig.Transformation}. */
private ArrayNode formatMapResults(String transformation, ArrayNode results, ArrayNode childInputs,
List<long[]> childTimings, String region, String account,
String smName, String mapRunLabel) {
ArrayNode out = objectMapper.createArrayNode();
if ("FLATTEN".equalsIgnoreCase(transformation)) {
- for (JsonNode r : results) {
- if (r.isArray()) {
- r.forEach(out::add);
+ for (JsonNode result : results) {
+ if (result.isArray()) {
+ result.forEach(out::add);
} else {
- out.add(r);
+ out.add(result);
}
}
return out;
@@ -1769,34 +1819,29 @@ public class AslExecutor {
String childSmArn = "arn:aws:states:" + region + ":" + account + ":stateMachine:"
+ smName + "/" + mapRunLabel;
for (int i = 0; i < results.size(); i++) {
- String childId = java.util.UUID.randomUUID().toString();
+ String childId = UUID.randomUUID().toString();
long start = childTimings != null && i < childTimings.size() ? childTimings.get(i)[0] : 0L;
long stop = childTimings != null && i < childTimings.size() ? childTimings.get(i)[1] : 0L;
- ObjectNode rec = out.addObject();
- rec.put("ExecutionArn", "arn:aws:states:" + region + ":" + account + ":execution:"
+ ObjectNode record = out.addObject();
+ record.put("ExecutionArn", "arn:aws:states:" + region + ":" + account + ":execution:"
+ smName + "/" + mapRunLabel + ":" + childId);
- rec.put("Input", stringifyResult(childInputs != null && i < childInputs.size()
+ record.put("Input", stringifyResult(childInputs != null && i < childInputs.size()
? childInputs.get(i) : NullNode.getInstance()));
- rec.putObject("InputDetails").put("Included", true);
- rec.put("Name", childId);
- rec.put("Output", stringifyResult(results.get(i)));
- rec.putObject("OutputDetails").put("Included", true);
- rec.put("RedriveCount", 0);
- rec.put("RedriveStatus", "NOT_REDRIVABLE");
- rec.put("RedriveStatusReason", "Execution is SUCCEEDED and cannot be redriven");
- rec.put("StartDate", java.time.Instant.ofEpochMilli(start).toString());
- rec.put("StateMachineArn", childSmArn);
- rec.put("Status", "SUCCEEDED");
- rec.put("StopDate", java.time.Instant.ofEpochMilli(stop).toString());
+ record.putObject("InputDetails").put("Included", true);
+ record.put("Name", childId);
+ record.put("Output", stringifyResult(results.get(i)));
+ record.putObject("OutputDetails").put("Included", true);
+ record.put("RedriveCount", 0);
+ record.put("RedriveStatus", "NOT_REDRIVABLE");
+ record.put("RedriveStatusReason", "Execution is SUCCEEDED and cannot be redriven");
+ record.put("StartDate", java.time.Instant.ofEpochMilli(start).toString());
+ record.put("StateMachineArn", childSmArn);
+ record.put("Status", "SUCCEEDED");
+ record.put("StopDate", java.time.Instant.ofEpochMilli(stop).toString());
}
return out;
}
- /**
- * AWS records a child execution's Input/Output as a JSON-encoded string. Using {@code toString()}
- * (not {@code asText()}) keeps scalar strings quoted — the string {@code hello} is exported as
- * {@code "hello"} — so the field is always valid JSON regardless of the value's type.
- */
private String stringifyResult(JsonNode node) {
if (node == null || node.isNull() || node.isMissingNode()) {
return "null";
@@ -1804,14 +1849,13 @@ public class AslExecutor {
return node.toString();
}
- /** Serializes a formatted result array as a JSON array or as JSON Lines per {@code OutputType}. */
private byte[] serializeResultFile(JsonNode formatted, String outputType) throws Exception {
if ("JSONL".equalsIgnoreCase(outputType) && formatted.isArray()) {
- StringBuilder sb = new StringBuilder();
+ StringBuilder output = new StringBuilder();
for (JsonNode element : formatted) {
- sb.append(objectMapper.writeValueAsString(element)).append('\n');
+ output.append(objectMapper.writeValueAsString(element)).append('\n');
}
- return sb.toString().getBytes(StandardCharsets.UTF_8);
+ return output.toString().getBytes(StandardCharsets.UTF_8);
}
return objectMapper.writeValueAsBytes(formatted);
}
@@ -1938,6 +1982,9 @@ public class AslExecutor {
String currentState = startAt;
while (currentState != null) {
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedException("Step Functions branch execution was interrupted");
+ }
JsonNode stateDef = states.path(currentState);
if (stateDef.isMissingNode()) {
throw new RuntimeException("State not found: " + currentState);
@@ -2218,24 +2265,65 @@ public class AslExecutor {
return walkPath(splitPathSegments(path), 0, root);
}
- /**
- * Splits a reference path into segments, normalizing bracket notation into dot segments so the
- * AWS bracket forms reduce to the same walk as the dot forms:
- * {@code $.Regions[*].RegionName} and {@code $.Regions.*.RegionName} both yield
- * {@code [Regions, *, RegionName]}, and {@code $[*][*]} yields {@code [*, *]}.
- *
- * <p>Limitation: every literal dot is treated as a segment separator, so a field name that
- * itself contains a dot is mis-split. AWS's bracket-quoted escape hatch ({@code $.a['b.c']})
- * is not supported; this matches the prior behavior and is rare in ASL reference paths.
- */
+ /** Splits dotted, indexed, wildcard, and bracket-quoted AWS reference-path segments. */
private String[] splitPathSegments(String path) {
- String rest = path.substring(1); // drop leading '$'
- rest = rest.replaceAll("\\[(\\*|\\d+)]", ".$1"); // [*] -> .*, [0] -> .0
- rest = rest.replaceAll("\\.{2,}", "."); // collapse ".[0]" -> "..0" -> ".0"
- if (rest.startsWith(".")) {
- rest = rest.substring(1);
+ List<String> segments = new ArrayList<>();
+ int index = 1;
+ while (index < path.length()) {
+ char current = path.charAt(index);
+ if (current == '.') {
+ int start = ++index;
+ while (index < path.length()
+ && path.charAt(index) != '.' && path.charAt(index) != '[') {
+ index++;
+ }
+ if (index > start) {
+ segments.add(path.substring(start, index));
+ }
+ continue;
+ }
+ if (current != '[') {
+ return new String[]{path};
+ }
+ index++;
+ if (index >= path.length()) {
+ return new String[]{path};
+ }
+ char first = path.charAt(index);
+ if (first == '\'' || first == '"') {
+ char quote = first;
+ StringBuilder member = new StringBuilder();
+ index++;
+ boolean closed = false;
+ while (index < path.length()) {
+ char ch = path.charAt(index++);
+ if (ch == '\\' && index < path.length()) {
+ member.append(path.charAt(index++));
+ } else if (ch == quote) {
+ closed = true;
+ break;
+ } else {
+ member.append(ch);
+ }
+ }
+ if (!closed || index >= path.length() || path.charAt(index) != ']') {
+ return new String[]{path};
+ }
+ segments.add(member.toString());
+ index++;
+ continue;
+ }
+ int start = index;
+ while (index < path.length() && path.charAt(index) != ']') {
+ index++;
+ }
+ if (index >= path.length()) {
+ return new String[]{path};
+ }
+ segments.add(path.substring(start, index));
+ index++;
}
- return rest.isEmpty() ? new String[0] : rest.split("\\.");
+ return segments.toArray(String[]::new);
}
/**
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationScheduler.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationScheduler.java
new file mode 100644
index 0000000000..d8cb258798
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationScheduler.java
@@ -0,0 +1,98 @@
+package io.github.hectorvent.floci.services.stepfunctions;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.function.IntFunction;
+
+/** Runs Map iterations with a bounded in-flight window while preserving input order. */
+final class MapIterationScheduler {
+
+ private MapIterationScheduler() {
+ }
+
+ static <T> List<T> execute(int itemCount, int maxConcurrency,
+ IntFunction<Callable<T>> taskFactory) throws Exception {
+ if (itemCount < 0) {
+ throw new IllegalArgumentException("itemCount must not be negative");
+ }
+ if (maxConcurrency < 1) {
+ throw new IllegalArgumentException("maxConcurrency must be at least 1");
+ }
+ if (itemCount == 0) {
+ return List.of();
+ }
+ if (itemCount == 1 || maxConcurrency == 1) {
+ List<T> results = new ArrayList<>(itemCount);
+ for (int i = 0; i < itemCount; i++) {
+ results.add(taskFactory.apply(i).call());
+ }
+ return results;
+ }
+
+ int inFlightLimit = Math.min(itemCount, maxConcurrency);
+ List<T> results = new ArrayList<>(Collections.nCopies(itemCount, null));
+ ExecutorService executor = Executors.newThreadPerTaskExecutor(
+ Thread.ofVirtual().name("sfn-map-", 0).factory());
+ CompletionService<IndexedResult<T>> completions = new ExecutorCompletionService<>(executor);
+ Set<Future<IndexedResult<T>>> inFlight = new HashSet<>();
+ int nextIndex = 0;
+ int completed = 0;
+
+ try {
+ while (nextIndex < inFlightLimit) {
+ inFlight.add(submit(completions, taskFactory, nextIndex++));
+ }
+ while (completed < itemCount) {
+ Future<IndexedResult<T>> future = completions.take();
+ inFlight.remove(future);
+ IndexedResult<T> result = future.get();
+ results.set(result.index(), result.value());
+ completed++;
+ if (nextIndex < itemCount) {
+ inFlight.add(submit(completions, taskFactory, nextIndex++));
+ }
+ }
+ return results;
+ } catch (ExecutionException e) {
+ cancel(inFlight);
+ Throwable cause = e.getCause();
+ if (cause instanceof Exception exception) {
+ throw exception;
+ }
+ if (cause instanceof Error error) {
+ throw error;
+ }
+ throw new RuntimeException(cause);
+ } catch (InterruptedException e) {
+ cancel(inFlight);
+ Thread.currentThread().interrupt();
+ throw e;
+ } finally {
+ cancel(inFlight);
+ executor.shutdownNow();
+ }
+ }
+
+ private static <T> Future<IndexedResult<T>> submit(
+ CompletionService<IndexedResult<T>> completions,
+ IntFunction<Callable<T>> taskFactory, int index) {
+ return completions.submit(() -> new IndexedResult<>(index, taskFactory.apply(index).call()));
+ }
+
+ private static void cancel(Set<? extends Future<?>> futures) {
+ futures.forEach(future -> future.cancel(true));
+ }
+
+ private record IndexedResult<T>(int index, T value) {
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
index 2c32bca3f9..9b0a48d6cb 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
@@ -48,7 +48,8 @@ public class StepFunctionsService implements Resettable {
// Fields that are valid only in JSONPath mode. Validated against real AWS:
// creating a JSONata state machine with any of these fields returns SCHEMA_VALIDATION_FAILED.
private static final Set<String> JSONPATH_ONLY_FIELDS = Set.of(
- "InputPath", "OutputPath", "ResultPath", "ResultSelector", "Parameters", "Result", "ItemsPath");
+ "InputPath", "OutputPath", "ResultPath", "ResultSelector", "Parameters", "Result", "ItemsPath",
+ "MaxConcurrencyPath");
private static final Set<String> ITEM_READER_RESOURCES = Set.of(
"arn:aws:states:::s3:getObject",
"arn:aws:states:::s3:listObjectsV2");
@@ -545,13 +546,14 @@ public class StepFunctionsService implements Resettable {
var fields = states.fields();
while (fields.hasNext()) {
var entry = fields.next();
- validateState(entry.getKey(), entry.getValue(), topLevelJsonata, errors);
+ validateState("/States/" + entry.getKey(), entry.getValue(), topLevelJsonata, errors);
}
}
return errors;
}
- private void validateState(String stateName, JsonNode stateDef, boolean topLevelJsonata, List<String> errors) {
+ private void validateState(String statePath, JsonNode stateDef,
+ boolean topLevelJsonata, List<String> errors) {
String stateQL = stateDef.path("QueryLanguage").asText(null);
boolean stateIsJsonata = stateQL != null ? "JSONata".equals(stateQL) : topLevelJsonata;
@@ -560,41 +562,157 @@ public class StepFunctionsService implements Resettable {
for (String field : JSONPATH_ONLY_FIELDS) {
if (stateDef.has(field)) {
errors.add("The QueryLanguage is set to 'JSONata', but field '" + field
- + "' is only supported for the 'JSONPath' QueryLanguage at /States/" + stateName);
+ + "' is only supported for the 'JSONPath' QueryLanguage at " + statePath);
}
}
}
if ("Map".equals(stateDef.path("Type").asText())) {
+ validateMapConcurrency(statePath, stateDef, stateIsJsonata, errors);
if (stateDef.has("ItemReader")) {
- validateItemReader(stateName, stateDef, errors);
+ validateItemReader(statePath, stateDef, errors);
}
if (stateDef.has("ResultWriter")) {
- validateResultWriter(stateName, stateDef.get("ResultWriter"), stateIsJsonata, errors);
+ validateResultWriter(statePath, stateDef.get("ResultWriter"), stateIsJsonata, errors);
}
+ String processorField = stateDef.has("ItemProcessor") ? "ItemProcessor" : "Iterator";
+ validateNestedStates(stateDef.path(processorField).path("States"),
+ statePath + "/" + processorField + "/States", topLevelJsonata, errors);
+ } else if ("Parallel".equals(stateDef.path("Type").asText())) {
+ JsonNode branches = stateDef.path("Branches");
+ if (branches.isArray()) {
+ for (int i = 0; i < branches.size(); i++) {
+ validateNestedStates(branches.path(i).path("States"),
+ statePath + "/Branches/" + i + "/States", topLevelJsonata, errors);
+ }
+ }
+ }
+ }
+
+ private void validateNestedStates(JsonNode states, String statesPath,
+ boolean inheritedJsonata, List<String> errors) {
+ if (!states.isObject()) {
+ return;
+ }
+ states.fields().forEachRemaining(entry -> validateState(
+ statesPath + "/" + entry.getKey(), entry.getValue(), inheritedJsonata, errors));
+ }
+
+ private void validateMapConcurrency(String statePath, JsonNode stateDef,
+ boolean jsonata, List<String> errors) {
+ if (stateDef.has("MaxConcurrency") && stateDef.has("MaxConcurrencyPath")) {
+ errors.add("A Map state cannot include both field 'MaxConcurrency' and MaxConcurrencyPath"
+ + " at " + statePath);
+ }
+
+ if (stateDef.has("MaxConcurrency")) {
+ JsonNode value = stateDef.get("MaxConcurrency");
+ boolean expression = jsonata && value.isTextual()
+ && JsonataEvaluator.isExpression(value.asText());
+ if (!value.isIntegralNumber() && !expression) {
+ errors.add("The field 'MaxConcurrency' must be a non-negative integer"
+ + (jsonata ? " or a JSONata expression" : "")
+ + " at " + statePath);
+ } else if (value.isIntegralNumber() && value.bigIntegerValue().signum() < 0) {
+ errors.add("The field 'MaxConcurrency' must be a non-negative integer"
+ + " at " + statePath);
+ }
+ }
+
+ if (!jsonata && stateDef.has("MaxConcurrencyPath")) {
+ JsonNode path = stateDef.get("MaxConcurrencyPath");
+ String pathValue = path.isTextual() ? path.asText() : null;
+ if (!isReferencePath(pathValue)) {
+ errors.add("The field 'MaxConcurrencyPath' must be a JSONPath reference path"
+ + " at " + statePath);
+ }
+ }
+ }
+
+ private static boolean isReferencePath(String path) {
+ if (path == null || path.isEmpty() || path.charAt(0) != '$') {
+ return false;
+ }
+ int index = 1;
+ while (index < path.length()) {
+ if (path.charAt(index) == '.') {
+ index++;
+ if (index < path.length() && path.charAt(index) == '[') {
+ continue;
+ }
+ if (index >= path.length()
+ || !(Character.isLetter(path.charAt(index)) || path.charAt(index) == '_')) {
+ return false;
+ }
+ index++;
+ while (index < path.length()
+ && (Character.isLetterOrDigit(path.charAt(index)) || path.charAt(index) == '_')) {
+ index++;
+ }
+ continue;
+ }
+ if (path.charAt(index) != '[') {
+ return false;
+ }
+ index++;
+ if (index >= path.length()) {
+ return false;
+ }
+ char first = path.charAt(index);
+ if (Character.isDigit(first)) {
+ while (index < path.length() && Character.isDigit(path.charAt(index))) {
+ index++;
+ }
+ } else if (first == '\'' || first == '"') {
+ char quote = first;
+ index++;
+ boolean hasCharacter = false;
+ boolean closed = false;
+ while (index < path.length()) {
+ char current = path.charAt(index++);
+ if (current == '\\' && index < path.length()) {
+ index++;
+ hasCharacter = true;
+ } else if (current == quote) {
+ closed = true;
+ break;
+ } else {
+ hasCharacter = true;
+ }
+ }
+ if (!closed || !hasCharacter) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ if (index >= path.length() || path.charAt(index) != ']') {
+ return false;
+ }
+ index++;
}
+ return true;
}
- private void validateItemReader(String stateName, JsonNode stateDef, List<String> errors) {
+ private void validateItemReader(String statePath, JsonNode stateDef, List<String> errors) {
JsonNode itemReader = stateDef.get("ItemReader");
String resource = itemReader.path("Resource").asText(null);
if (resource != null && !ITEM_READER_RESOURCES.contains(resource)) {
errors.add("The field 'Resource' does not match any of the allowed values. Examples: "
+ "[arn:<partition>:states:::s3:getObject, arn:<partition>:states:::s3:listObjectsV2]"
- + " at /States/" + stateName + "/ItemReader/Resource");
+ + " at " + statePath + "/ItemReader/Resource");
}
String inputType = itemReader.path("ReaderConfig").path("InputType").asText(null);
if (inputType != null && !ITEM_READER_INPUT_TYPES.contains(inputType)) {
errors.add("The field 'InputType' should have one of these values: "
+ "[MANIFEST, JSON, CSV, JSONL, PARQUET]"
- + " at /States/" + stateName + "/ItemReader/ReaderConfig/InputType");
+ + " at " + statePath + "/ItemReader/ReaderConfig/InputType");
}
}
- private void validateResultWriter(String stateName, JsonNode writer,
+ private void validateResultWriter(String statePath, JsonNode writer,
boolean jsonata, List<String> errors) {
- String statePath = "/States/" + stateName;
String writerPath = statePath + "/ResultWriter";
if (!writer.isObject() || writer.isEmpty()) {
errors.add("The field 'ResultWriter' must specify WriterConfig or Resource with "
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorPathIntrinsicsTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorPathIntrinsicsTest.java
index 41f8a69c7e..7906b084cb 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorPathIntrinsicsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorPathIntrinsicsTest.java
@@ -94,6 +94,14 @@ class AslExecutorPathIntrinsicsTest {
assertEquals("b", executor.resolvePath("$.items[1]", root).asText());
}
+ @Test
+ void bracketQuotedMembersSupportNamesOutsideDotShorthand() throws Exception {
+ JsonNode root = mapper.readTree("{\"config\":{\"max-limit\":2},\"a.b\":3}");
+
+ assertEquals(2, executor.resolvePath("$['config']['max-limit']", root).asInt());
+ assertEquals(3, executor.resolvePath("$['a.b']", root).asInt());
+ }
+
@Test
void resolvePathNodeDistinguishesExplicitNullFromMissing() throws Exception {
JsonNode root = mapper.readTree("{\"x\":null}");
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationSchedulerTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationSchedulerTest.java
new file mode 100644
index 0000000000..cc2f72691d
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationSchedulerTest.java
@@ -0,0 +1,130 @@
+package io.github.hectorvent.floci.services.stepfunctions;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+class MapIterationSchedulerTest {
+
+ @Test
+ void appliesAwsConcurrencyCeilingsForInlineAndDistributedMaps() {
+ assertEquals(40, AslExecutor.effectiveMapConcurrency(50_000, 0, false));
+ assertEquals(40, AslExecutor.effectiveMapConcurrency(50_000, 500, false));
+ assertEquals(7, AslExecutor.effectiveMapConcurrency(7, 0, false));
+ assertEquals(10_000, AslExecutor.effectiveMapConcurrency(50_000, 0, true));
+ assertEquals(250, AslExecutor.effectiveMapConcurrency(50_000, 250, true));
+ }
+
+ @Test
+ void maxConcurrencyOneStartsEachItemStrictlyInInputOrder() throws Exception {
+ List<Integer> starts = new java.util.concurrent.CopyOnWriteArrayList<>();
+
+ List<Integer> results = MapIterationScheduler.execute(5, 1, index -> () -> {
+ starts.add(index);
+ return index;
+ });
+
+ assertEquals(List.of(0, 1, 2, 3, 4), starts);
+ assertEquals(starts, results);
+ }
+
+ @Test
+ void keepsOnlyMaxConcurrencyIterationsInFlight() throws Exception {
+ int itemCount = 50;
+ int maxConcurrency = 3;
+ AtomicInteger active = new AtomicInteger();
+ AtomicInteger maxObserved = new AtomicInteger();
+ AtomicInteger started = new AtomicInteger();
+ CountDownLatch firstWaveStarted = new CountDownLatch(maxConcurrency);
+ CountDownLatch release = new CountDownLatch(1);
+
+ try (ExecutorService driver = Executors.newSingleThreadExecutor()) {
+ Future<List<Integer>> execution = driver.submit(() -> MapIterationScheduler.execute(
+ itemCount, maxConcurrency, index -> () -> {
+ int nowActive = active.incrementAndGet();
+ maxObserved.accumulateAndGet(nowActive, Math::max);
+ started.incrementAndGet();
+ firstWaveStarted.countDown();
+ try {
+ release.await();
+ return index;
+ } finally {
+ active.decrementAndGet();
+ }
+ }));
+
+ assertTrue(firstWaveStarted.await(2, TimeUnit.SECONDS), "initial worker window did not start");
+ assertEquals(maxConcurrency, started.get(), "queued items must not be submitted early");
+ release.countDown();
+
+ assertEquals(java.util.stream.IntStream.range(0, itemCount).boxed().toList(),
+ execution.get(5, TimeUnit.SECONDS));
+ assertEquals(maxConcurrency, maxObserved.get());
+ }
+ }
+
+ @Test
+ void reportsLaterFailureWithoutWaitingForAnEarlierBlockedItem() throws Exception {
+ CountDownLatch slowStarted = new CountDownLatch(1);
+ CountDownLatch slowInterrupted = new CountDownLatch(1);
+ CountDownLatch neverRelease = new CountDownLatch(1);
+
+ try (ExecutorService driver = Executors.newSingleThreadExecutor()) {
+ Future<List<Integer>> execution = driver.submit(() -> MapIterationScheduler.execute(
+ 2, 2, index -> () -> {
+ if (index == 0) {
+ slowStarted.countDown();
+ try {
+ neverRelease.await();
+ return index;
+ } catch (InterruptedException e) {
+ slowInterrupted.countDown();
+ throw e;
+ }
+ }
+ if (!slowStarted.await(2, TimeUnit.SECONDS)) {
+ fail("slow iteration did not start");
+ }
+ throw new IllegalStateException("later iteration failed");
+ }));
+
+ ExecutionException failure = assertThrows(ExecutionException.class,
+ () -> execution.get(2, TimeUnit.SECONDS));
+ assertEquals(IllegalStateException.class, failure.getCause().getClass());
+ assertEquals("later iteration failed", failure.getCause().getMessage());
+ assertTrue(slowInterrupted.await(2, TimeUnit.SECONDS),
+ "the blocked sibling should be interrupted after failure");
+ }
+ }
+
+ @Test
+ void preservesInputOrderWhenItemsCompleteOutOfOrder() throws Exception {
+ CountDownLatch laterItemsCompleted = new CountDownLatch(2);
+
+ List<Integer> results = org.junit.jupiter.api.Assertions.assertTimeoutPreemptively(
+ Duration.ofSeconds(2),
+ () -> MapIterationScheduler.execute(3, 3, index -> () -> {
+ if (index == 0) {
+ laterItemsCompleted.await();
+ } else {
+ laterItemsCompleted.countDown();
+ }
+ return index;
+ }));
+
+ assertEquals(List.of(0, 1, 2), results);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsMapConcurrencyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsMapConcurrencyIntegrationTest.java
new file mode 100644
index 0000000000..98a82395bd
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsMapConcurrencyIntegrationTest.java
@@ -0,0 +1,212 @@
+package io.github.hectorvent.floci.services.stepfunctions;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * Wire-level coverage for Map concurrency configuration and ordered results. The scheduler's
+ * concurrency, bounded-window, cancellation, and fail-fast behavior is covered deterministically
+ * by {@link MapIterationSchedulerTest} rather than with wall-clock assertions.
+ */
+@QuarkusTest
+class StepFunctionsMapConcurrencyIntegrationTest {
+
+ private static final String SFN_CONTENT_TYPE = "application/x-amz-json-1.0";
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/test-role";
+ private final ObjectMapper mapper = new ObjectMapper();
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ private static String mapDef(Integer maxConcurrency) {
+ String mc = maxConcurrency == null ? "" : "\"MaxConcurrency\":" + maxConcurrency + ",";
+ return mapDefWithConcurrencyField(mc);
+ }
+
+ private static String mapDefWithConcurrencyField(String concurrencyField) {
+ return "{\"StartAt\":\"M\",\"States\":{"
+ + "\"M\":{\"Type\":\"Map\",\"ItemsPath\":\"$.items\"," + concurrencyField
+ + "\"ItemProcessor\":{\"StartAt\":\"P\",\"States\":{"
+ + "\"P\":{\"Type\":\"Pass\",\"End\":true}}},"
+ + "\"End\":true}}}";
+ }
+
+ private static String items(int n) {
+ StringBuilder sb = new StringBuilder("{\"items\":[");
+ for (int i = 0; i < n; i++) {
+ if (i > 0) sb.append(',');
+ sb.append("{\"i\":").append(i).append('}');
+ }
+ return sb.append("]}").toString();
+ }
+
+ private void assertOrderedItems(String output, int n) throws Exception {
+ JsonNode arr = mapper.readTree(output);
+ assertEquals(n, arr.size());
+ for (int i = 0; i < n; i++) {
+ assertEquals(i, arr.get(i).path("i").asInt(), "results must be in input order");
+ }
+ }
+
+ @Test
+ void mapPreservesInputOrderWithDefaultConcurrency() throws Exception {
+ assertOrderedItems(run(mapDef(null), items(50)), 50);
+ }
+
+ @Test
+ void mapAcceptsStaticMaxConcurrency() throws Exception {
+ assertOrderedItems(run(mapDef(1), items(5)), 5);
+ }
+
+ @Test
+ void mapResolvesMaxConcurrencyPathFromItsInput() throws Exception {
+ String input = "{\"ignored\":true,\"payload\":{"
+ + "\"config\":{\"max-limit\":2},\"items\":["
+ + "{\"i\":0},{\"i\":1},{\"i\":2},{\"i\":3}]}}";
+
+ assertOrderedItems(run(mapDefWithConcurrencyField(
+ "\"InputPath\":\"$.payload\","
+ + "\"MaxConcurrencyPath\":\"$.config.['max-limit']\","), input), 4);
+ }
+
+ @Test
+ void mapResolvesItemsPathFromEffectiveInputWithoutResultWriter() throws Exception {
+ String definition = """
+ {
+ "StartAt": "M",
+ "States": {
+ "M": {
+ "Type": "Map",
+ "InputPath": "$.payload",
+ "ItemsPath": "$.items",
+ "ItemProcessor": {
+ "StartAt": "P",
+ "States": {"P": {"Type": "Pass", "End": true}}
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+ String input = "{\"items\":[{\"i\":99}],\"payload\":{\"items\":[{\"i\":0},{\"i\":1}]}}";
+
+ assertOrderedItems(run(definition, input), 2);
+ }
+
+ @Test
+ void jsonataMapEvaluatesMaxConcurrencyExpression() throws Exception {
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "M",
+ "States": {
+ "M": {
+ "Type": "Map",
+ "Items": "{% $states.input.items %}",
+ "MaxConcurrency": "{% $states.input.limit %}",
+ "ItemProcessor": {
+ "StartAt": "P",
+ "States": {"P": {"Type": "Pass", "End": true}}
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ assertOrderedItems(run(definition,
+ "{\"limit\":3,\"items\":[{\"i\":0},{\"i\":1},{\"i\":2},{\"i\":3}]}"), 4);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"true", "-1", "1.5", "$states.input.missing"})
+ void invalidJsonataMaxConcurrencyIsCatchableAsQueryEvaluationError(String expression) throws Exception {
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "M",
+ "States": {
+ "M": {
+ "Type": "Map",
+ "Items": "{% $states.input.items %}",
+ "MaxConcurrency": "{% __EXPRESSION__ %}",
+ "ItemProcessor": {
+ "StartAt": "P",
+ "States": {"P": {"Type": "Pass", "End": true}}
+ },
+ "Catch": [{
+ "ErrorEquals": ["States.QueryEvaluationError"],
+ "Next": "Caught"
+ }],
+ "Next": "Unexpected"
+ },
+ "Caught": {
+ "Type": "Pass",
+ "Output": {"caught": true},
+ "End": true
+ },
+ "Unexpected": {"Type": "Fail", "Error": "UnexpectedSuccess"}
+ }
+ }
+ """.replace("__EXPRESSION__", expression);
+
+ JsonNode output = mapper.readTree(run(definition, "{\"items\":[1,2]}"));
+ assertTrue(output.path("caught").asBoolean());
+ }
+
+ private String run(String definition, String input) throws InterruptedException {
+ String name = "map-concurrency-" + System.nanoTime();
+ Response create = given()
+ .header("X-Amz-Target", "AWSStepFunctions.CreateStateMachine")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("{\"name\":\"" + name + "\",\"definition\":" + quote(definition) + ",\"roleArn\":\"" + ROLE_ARN + "\"}")
+ .when().post("/");
+ create.then().statusCode(200);
+ String smArn = create.jsonPath().getString("stateMachineArn");
+
+ Response start = given()
+ .header("X-Amz-Target", "AWSStepFunctions.StartExecution")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("{\"stateMachineArn\":\"" + smArn + "\",\"input\":" + quote(input) + "}")
+ .when().post("/");
+ start.then().statusCode(200);
+ String execArn = start.jsonPath().getString("executionArn");
+
+ for (int i = 0; i < 100; i++) {
+ Response d = given()
+ .header("X-Amz-Target", "AWSStepFunctions.DescribeExecution")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("{\"executionArn\":\"" + execArn + "\"}")
+ .when().post("/");
+ String status = d.jsonPath().getString("status");
+ if ("SUCCEEDED".equals(status)) {
+ return d.jsonPath().getString("output");
+ }
+ if ("FAILED".equals(status)) {
+ fail("Execution failed: " + d.body().asString());
+ }
+ Thread.sleep(100);
+ }
+ fail("Execution did not complete");
+ return null;
+ }
+
+ private static String quote(String raw) {
+ return "\"" + raw.replace("\\", "\\\\").replace("\"", "\\\"")
+ .replace("\n", "\\n").replace("\r", "\\r").replace("\t", "\\t") + "\"";
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java
index d2212369f9..43cb1fbbcb 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java
@@ -184,6 +184,104 @@ class StepFunctionsValidateStateMachineDefinitionIntegrationTest {
.body("diagnostics[0].location", equalTo("/States/X/InputPath"));
}
+ @Test
+ void mapCannotSpecifyBothMaxConcurrencyFields() {
+ String def = mapDefinition("", "\"MaxConcurrency\":2,"
+ + "\"MaxConcurrencyPath\":\"$.limit\",");
+
+ given().contentType(CT).header("X-Amz-Target", TARGET)
+ .body(definitionRequest(def))
+ .when().post("/")
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].location", equalTo("/States/M/MaxConcurrency"));
+ }
+
+ @Test
+ void mapMaxConcurrencyMustBeANonNegativeInteger() {
+ String def = mapDefinition("", "\"MaxConcurrency\":-1,");
+
+ given().contentType(CT).header("X-Amz-Target", TARGET)
+ .body(definitionRequest(def))
+ .when().post("/")
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].location", equalTo("/States/M/MaxConcurrency"));
+ }
+
+ @Test
+ void jsonataMapRejectsMaxConcurrencyPath() {
+ String def = mapDefinition("\"QueryLanguage\":\"JSONata\",",
+ "\"MaxConcurrencyPath\":\"$.limit\",");
+
+ given().contentType(CT).header("X-Amz-Target", TARGET)
+ .body(definitionRequest(def))
+ .when().post("/")
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].location", equalTo("/States/M/MaxConcurrencyPath"));
+ }
+
+ @Test
+ void jsonataMapAcceptsMaxConcurrencyExpression() {
+ String def = mapDefinition("\"QueryLanguage\":\"JSONata\",",
+ "\"MaxConcurrency\":\"{% $states.input.limit %}\",");
+
+ given().contentType(CT).header("X-Amz-Target", TARGET)
+ .body(definitionRequest(def))
+ .when().post("/")
+ .then().statusCode(200)
+ .body("result", equalTo("OK"))
+ .body("diagnostics", hasSize(0));
+ }
+
+ @Test
+ void nestedMapMaxConcurrencyIsValidatedAtItsStructuredPath() {
+ String def = "{\"StartAt\":\"Outer\",\"States\":{\"Outer\":{\"Type\":\"Map\","
+ + "\"ItemProcessor\":{\"StartAt\":\"Inner\",\"States\":{"
+ + "\"Inner\":{\"Type\":\"Map\",\"MaxConcurrency\":-1,"
+ + "\"ItemProcessor\":{\"StartAt\":\"P\",\"States\":{"
+ + "\"P\":{\"Type\":\"Pass\",\"End\":true}}},\"End\":true}}},"
+ + "\"End\":true}}}";
+
+ given().contentType(CT).header("X-Amz-Target", TARGET)
+ .body(definitionRequest(def))
+ .when().post("/")
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].location", equalTo(
+ "/States/Outer/ItemProcessor/States/Inner/MaxConcurrency"));
+ }
+
+ @Test
+ void maxConcurrencyPathMustSelectASingleNode() {
+ String def = mapDefinition("", "\"MaxConcurrencyPath\":\"$.limits[*]\",");
+
+ given().contentType(CT).header("X-Amz-Target", TARGET)
+ .body(definitionRequest(def))
+ .when().post("/")
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].location", equalTo("/States/M/MaxConcurrencyPath"));
+ }
+
+ @Test
+ void maxConcurrencyLargerThanIntegerRangeIsAcceptedAndRuntimeCapped() {
+ String def = mapDefinition("", "\"MaxConcurrency\":9223372036854775807,");
+
+ given().contentType(CT).header("X-Amz-Target", TARGET)
+ .body(definitionRequest(def))
+ .when().post("/")
+ .then().statusCode(200)
+ .body("result", equalTo("OK"))
+ .body("diagnostics", hasSize(0));
+ }
+
@Test
void unsupportedItemReaderResource_returnsFailWithSchemaError() {
given().contentType(CT).header("X-Amz-Target", TARGET)
@@ -519,4 +617,21 @@ class StepFunctionsValidateStateMachineDefinitionIntegrationTest {
.then().statusCode(400)
.body("__type", containsString("ValidationException"));
}
+
+ private static String mapDefinition(String topLevelFields, String concurrencyFields) {
+ return "{" + topLevelFields
+ + "\"StartAt\":\"M\",\"States\":{\"M\":{\"Type\":\"Map\","
+ + concurrencyFields
+ + "\"ItemProcessor\":{\"StartAt\":\"P\",\"States\":{"
+ + "\"P\":{\"Type\":\"Pass\",\"End\":true}}},\"End\":true}}}";
+ }
+
+ private static String definitionRequest(String definition) {
+ String escaped = definition.replace("\\", "\\\\")
+ .replace("\"", "\\\"")
+ .replace("\n", "\\n")
+ .replace("\r", "\\r")
+ .replace("\t", "\\t");
+ return "{\"definition\":\"" + escaped + "\"}";
+ }
}