Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(stepfunctions): support Map ItemReader for S3-backed JSON datasets (#1588)

shrimptails-f36ef98047e53
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index f3ba57f637..82d2c10b8a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -23,6 +23,8 @@ import io.github.hectorvent.floci.services.lambda.LambdaFunctionStore;
import io.github.hectorvent.floci.services.lambda.model.InvocationType;
import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;
import io.github.hectorvent.floci.services.stepfunctions.model.Execution;
import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;
@@ -65,6 +67,15 @@ import java.util.function.BiConsumer;
@ApplicationScoped
public class AslExecutor {
+ private enum MapItemsSource {
+ DEFAULT,
+ ITEM_READER_ARRAY,
+ ITEM_READER_OBJECT
+ }
+
+ private record ResolvedMapItems(JsonNode items, MapItemsSource source) {
+ }
+
private static final Logger LOG = Logger.getLogger(AslExecutor.class);
private static final int MAX_WAIT_SECONDS = 30;
@@ -1362,19 +1373,15 @@ public class AslExecutor {
private StateResult executeMapState(String name, JsonNode stateDef, JsonNode input, StateMachine sm,
boolean jsonata, String topLevelQueryLanguage, JsonNode context) throws Exception {
- JsonNode items;
- if (jsonata && stateDef.has("Items")) {
- JsonNode itemsNode = stateDef.get("Items");
- if (itemsNode.isTextual() && JsonataEvaluator.isExpression(itemsNode.asText())) {
- JsonNode statesVar = buildStatesVar(input, null, context);
- items = jsonataEvaluator.evaluate(itemsNode.asText(), statesVar);
- } else {
- items = itemsNode;
+ if (stateDef.has("ItemReader")) {
+ String mode = stateDef.path("ItemProcessor").path("ProcessorConfig").path("Mode").asText("INLINE");
+ if (!"DISTRIBUTED".equals(mode)) {
+ throw new FailStateException("States.Runtime",
+ "The ItemReader, ItemBatcher and ResultWriter fields are not supported for INLINE maps");
}
- } else {
- JsonNode itemsPath = stateDef.path("ItemsPath");
- items = itemsPath.isMissingNode() ? input : resolvePath(itemsPath.asText("$"), input);
}
+ ResolvedMapItems resolvedItems = resolveMapItems(stateDef, input, jsonata, context);
+ JsonNode items = resolvedItems.items();
if (!items.isArray()) {
throw new FailStateException("States.Runtime", "Items must reference an array");
@@ -1395,20 +1402,25 @@ public class AslExecutor {
ArrayNode results = objectMapper.createArrayNode();
int index = 0;
for (JsonNode item : items) {
+ ObjectNode iterContext = ((ObjectNode) context).deepCopy();
+ ObjectNode mapCtx = objectMapper.createObjectNode();
+ ObjectNode mapItem = objectMapper.createObjectNode();
+ mapItem.put("Index", index);
+ if (resolvedItems.source() == MapItemsSource.ITEM_READER_OBJECT) {
+ mapItem.put("Key", item.path("Key").asText());
+ mapItem.set("Value", item.get("Value"));
+ } else {
+ mapItem.set("Value", item);
+ }
+ mapCtx.set("Item", mapItem);
+ iterContext.set("Map", mapCtx);
+
JsonNode iterInput = item;
if (itemTransform != null) {
- // Enrich context with Map.Item.Index and Map.Item.Value for $$.Map.* references.
// $ in ItemSelector resolves against the Map state's effective input, not the item.
- ObjectNode iterContext = ((ObjectNode) context).deepCopy();
- ObjectNode mapCtx = objectMapper.createObjectNode();
- ObjectNode mapItem = objectMapper.createObjectNode();
- mapItem.put("Index", index);
- mapItem.set("Value", item);
- mapCtx.set("Item", mapItem);
- iterContext.set("Map", mapCtx);
iterInput = resolveParameters(itemTransform, mapInput, iterContext);
}
- results.add(executeBranch(startAt, iteratorStates, iterInput, sm, topLevelQueryLanguage, context));
+ results.add(executeBranch(startAt, iteratorStates, iterInput, sm, topLevelQueryLanguage, iterContext));
index++;
}
@@ -1426,6 +1438,119 @@ public class AslExecutor {
return new StateResult(output, stateDef.path("Next").asText(null));
}
+ private ResolvedMapItems resolveMapItems(JsonNode stateDef, JsonNode input,
+ boolean jsonata, JsonNode context) throws Exception {
+ if (jsonata && stateDef.has("Items")) {
+ JsonNode itemsNode = stateDef.get("Items");
+ if (itemsNode.isTextual() && JsonataEvaluator.isExpression(itemsNode.asText())) {
+ JsonNode statesVar = buildStatesVar(input, null, context);
+ return new ResolvedMapItems(jsonataEvaluator.evaluate(itemsNode.asText(), statesVar),
+ MapItemsSource.DEFAULT);
+ }
+ return new ResolvedMapItems(itemsNode, MapItemsSource.DEFAULT);
+ }
+
+ if (stateDef.has("ItemReader")) {
+ return resolveItemReaderItems(stateDef.get("ItemReader"), input, context, jsonata);
+ }
+
+ JsonNode itemsPath = stateDef.path("ItemsPath");
+ return new ResolvedMapItems(itemsPath.isMissingNode() ? input : resolvePath(itemsPath.asText("$"), input),
+ MapItemsSource.DEFAULT);
+ }
+
+ private ResolvedMapItems resolveItemReaderItems(JsonNode itemReader, JsonNode input,
+ JsonNode context, boolean jsonata) throws Exception {
+ String resource = itemReader.path("Resource").asText(null);
+ if ("arn:aws:states:::s3:listObjectsV2".equals(resource)) {
+ throw new FailStateException("States.ItemReaderFailed",
+ "ItemReader resource arn:aws:states:::s3:listObjectsV2 is not yet implemented by the emulator");
+ }
+ if (!"arn:aws:states:::s3:getObject".equals(resource)) {
+ throw new FailStateException("States.Runtime", "Unsupported ItemReader resource: " + resource);
+ }
+
+ String inputType = itemReader.path("ReaderConfig").path("InputType").asText(null);
+ if (!"JSON".equals(inputType)) {
+ throw new FailStateException("States.ItemReaderFailed",
+ "ItemReader InputType " + inputType + " is not yet implemented by the emulator");
+ }
+
+ JsonNode resolvedParameters;
+ if (jsonata && itemReader.has("Arguments")) {
+ JsonNode statesVar = buildStatesVar(input, null, context);
+ resolvedParameters = jsonataEvaluator.resolveTemplate(itemReader.get("Arguments"), statesVar);
+ } else {
+ JsonNode parameters = itemReader.path("Parameters");
+ resolvedParameters = resolveParameters(parameters, input, context);
+ }
+ String bucket = resolvedParameters.path("Bucket").asText(null);
+ String key = resolvedParameters.path("Key").asText(null);
+ if (bucket == null || key == null) {
+ throw new FailStateException("States.Runtime", "ItemReader Parameters must include Bucket and Key");
+ }
+
+ try {
+ S3Object object = s3Service.getObject(bucket, key);
+ JsonNode items = objectMapper.readTree(object.getData());
+ items = applyItemsPointer(itemReader, items);
+ if (items.isObject()) {
+ return new ResolvedMapItems(applyMaxItems(itemReader, normalizeObjectItems(items)),
+ MapItemsSource.ITEM_READER_OBJECT);
+ }
+ if (!items.isArray()) {
+ throw new FailStateException("States.ItemReaderFailed",
+ "Attempting to map over non-iterable node.");
+ }
+ return new ResolvedMapItems(applyMaxItems(itemReader, items), MapItemsSource.ITEM_READER_ARRAY);
+ } catch (AwsException e) {
+ throw new FailStateException("States.ItemReaderFailed", e.getMessage());
+ } catch (FailStateException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new FailStateException("States.ItemReaderFailed",
+ e.getMessage() != null ? e.getMessage() : "Failed to parse ItemReader input");
+ }
+ }
+
+ private ArrayNode normalizeObjectItems(JsonNode items) {
+ ArrayNode normalized = objectMapper.createArrayNode();
+ items.fields().forEachRemaining(entry -> {
+ ObjectNode objectItem = objectMapper.createObjectNode();
+ objectItem.put("Key", entry.getKey());
+ objectItem.set("Value", entry.getValue());
+ normalized.add(objectItem);
+ });
+ return normalized;
+ }
+
+ private JsonNode applyItemsPointer(JsonNode itemReader, JsonNode items) {
+ String itemsPointer = itemReader.path("ReaderConfig").path("ItemsPointer").asText(null);
+ if (itemsPointer == null || itemsPointer.isEmpty()) {
+ return items;
+ }
+
+ JsonNode pointedItems = items.at(itemsPointer);
+ if (pointedItems.isMissingNode()) {
+ throw new FailStateException("States.ItemReaderFailed",
+ "The provided ReaderConfig.ItemsPointer does not match any valid path in the JSON structure.");
+ }
+ return pointedItems;
+ }
+
+ private JsonNode applyMaxItems(JsonNode itemReader, JsonNode items) {
+ int maxItems = itemReader.path("ReaderConfig").path("MaxItems").asInt(0);
+ if (maxItems <= 0 || !items.isArray() || items.size() <= maxItems) {
+ return items;
+ }
+
+ ArrayNode limited = objectMapper.createArrayNode();
+ for (int i = 0; i < maxItems; i++) {
+ limited.add(items.get(i));
+ }
+ return limited;
+ }
+
private JsonNode executeBranch(String startAt, JsonNode states, JsonNode input, StateMachine sm,
String topLevelQueryLanguage, JsonNode context) throws Exception {
List<HistoryEvent> ignored = new ArrayList<>();
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
index 5252a7f7a1..b756990b24 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
@@ -49,6 +49,11 @@ public class StepFunctionsService implements Resettable {
// creating a JSONata state machine with any of these fields returns SCHEMA_VALIDATION_FAILED.
private static final Set<String> JSONPATH_ONLY_FIELDS = Set.of(
"InputPath", "OutputPath", "ResultPath", "ResultSelector", "Parameters", "Result", "ItemsPath");
+ private static final Set<String> ITEM_READER_RESOURCES = Set.of(
+ "arn:aws:states:::s3:getObject",
+ "arn:aws:states:::s3:listObjectsV2");
+ private static final Set<String> ITEM_READER_INPUT_TYPES = Set.of(
+ "MANIFEST", "JSON", "CSV", "JSONL", "PARQUET");
@Inject
public StepFunctionsService(StorageFactory storageFactory, RegionResolver regionResolver,
@@ -417,10 +422,10 @@ public class StepFunctionsService implements Resettable {
private static final int MAX_DEFINITION_LENGTH = 1_048_576;
private static final String PARSE_ERROR_MARKER = "INVALID_JSON_DESCRIPTION:";
- // Parse the structured location out of the validator's flat error strings,
- // which currently encode it as "...field 'X' is only supported... at /States/Y".
+ // Parse the structured location out of validator flat error strings,
+ // which currently encode it as "...field 'X' ... at /States/Y".
// AWS's published Diagnostic.location format is "/States/<StateName>/<FieldName>".
- private static final Pattern FIELD_PATTERN = Pattern.compile("field '([^']+)' is only supported");
+ private static final Pattern FIELD_PATTERN = Pattern.compile("field '([^']+)'");
private static final Pattern LOCATION_SUFFIX_PATTERN = Pattern.compile(" at (/States/\\S+)$");
/**
@@ -492,7 +497,11 @@ public class StepFunctionsService implements Resettable {
if (locM.find() && fieldM.find()) {
// Build the structured location and strip the redundant suffix
// from the message, matching AWS's wire format.
- location = locM.group(1) + "/" + fieldM.group(1);
+ location = locM.group(1);
+ String field = fieldM.group(1);
+ if (!location.endsWith("/" + field)) {
+ location = location + "/" + field;
+ }
message = message.substring(0, locM.start()).trim();
}
}
@@ -552,5 +561,26 @@ public class StepFunctionsService implements Resettable {
}
}
}
+
+ if ("Map".equals(stateDef.path("Type").asText()) && stateDef.has("ItemReader")) {
+ validateItemReader(stateName, stateDef, errors);
+ }
+ }
+
+ private void validateItemReader(String stateName, JsonNode stateDef, List<String> errors) {
+ JsonNode itemReader = stateDef.get("ItemReader");
+ String resource = itemReader.path("Resource").asText(null);
+ if (resource != null && !ITEM_READER_RESOURCES.contains(resource)) {
+ errors.add("The field 'Resource' does not match any of the allowed values. Examples: "
+ + "[arn:<partition>:states:::s3:getObject, arn:<partition>:states:::s3:listObjectsV2]"
+ + " at /States/" + stateName + "/ItemReader/Resource");
+ }
+
+ String inputType = itemReader.path("ReaderConfig").path("InputType").asText(null);
+ if (inputType != null && !ITEM_READER_INPUT_TYPES.contains(inputType)) {
+ errors.add("The field 'InputType' should have one of these values: "
+ + "[MANIFEST, JSON, CSV, JSONL, PARQUET]"
+ + " at /States/" + stateName + "/ItemReader/ReaderConfig/InputType");
+ }
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorCatchTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorCatchTest.java
index e8f5efde4e..9f326fb7c3 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorCatchTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorCatchTest.java
@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.services.lambda.LambdaFunctionStore;
import io.github.hectorvent.floci.services.lambda.model.InvocationType;
import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;
import io.github.hectorvent.floci.services.stepfunctions.model.Execution;
import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;
@@ -76,7 +77,7 @@ class AslExecutorCatchTest {
mock(SqsJsonHandler.class),
mock(io.github.hectorvent.floci.services.cloudformation.CloudFormationQueryHandler.class),
mock(io.github.hectorvent.floci.services.ec2.Ec2Service.class),
- mock(io.github.hectorvent.floci.services.s3.S3Service.class),
+ mock(S3Service.class),
mock(io.github.hectorvent.floci.services.ecs.EcsService.class),
mock(io.github.hectorvent.floci.services.ecs.EcsJsonHandler.class),
objectMapper,
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
index d34d817370..35e853974d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
@@ -222,6 +222,948 @@ class StepFunctionsJsonataIntegrationTest {
assertTrue(output.contains("[1,2,3]"));
}
+ @Test
+ void distributedMapWithS3JsonItemReader_readsItemsFromS3Object() throws Exception {
+ createBucket("map-inputs");
+ putObject("map-inputs", "workers.json", "[{\"workerId\":\"w1\"},{\"workerId\":\"w2\"}]");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));
+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_jsonataArgumentsReadsItemsFromS3Object() throws Exception {
+ createBucket("map-inputs-arguments");
+ putObject("map-inputs-arguments", "workers.json", "[{\"workerId\":\"w1\"},{\"workerId\":\"w2\"}]");
+
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Arguments": {
+ "Bucket": "map-inputs-arguments",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-arguments-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));
+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_maxItemsLimitsArrayDataset() throws Exception {
+ createBucket("map-inputs-max-items-array");
+ putObject("map-inputs-max-items-array", "workers.json", """
+ [{"workerId":"w1"},{"workerId":"w2"},{"workerId":"w3"}]
+ """);
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON",
+ "MaxItems": 2
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-max-items-array",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-max-items-array-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));
+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));
+ assertFalse(output.contains("\"workerId\":\"w3\"") || output.contains("\"workerId\": \"w3\""));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_maxItemsLimitsObjectDataset() throws Exception {
+ createBucket("map-inputs-max-items-object");
+ putObject("map-inputs-max-items-object", "workers.json", """
+ {"a":{"workerId":"w1"},"b":{"workerId":"w2"},"c":{"workerId":"w3"}}
+ """);
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON",
+ "MaxItems": 2
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-max-items-object",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-max-items-object-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"Key\":\"a\"") || output.contains("\"Key\": \"a\""));
+ assertTrue(output.contains("\"Key\":\"b\"") || output.contains("\"Key\": \"b\""));
+ assertFalse(output.contains("\"Key\":\"c\"") || output.contains("\"Key\": \"c\""));
+ assertFalse(output.contains("\"workerId\":\"w3\"") || output.contains("\"workerId\": \"w3\""));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_itemsPointerSelectsArrayDataset() throws Exception {
+ createBucket("map-inputs-pointer-array");
+ putObject("map-inputs-pointer-array", "workers.json", """
+ {"records":[{"workerId":"w1"},{"workerId":"w2"}]}
+ """);
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON",
+ "ItemsPointer": "/records"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-pointer-array",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-pointer-array-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));
+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));
+ assertFalse(output.contains("\"records\""));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_itemsPointerSelectsObjectDataset() throws Exception {
+ createBucket("map-inputs-pointer-object");
+ putObject("map-inputs-pointer-object", "workers.json", """
+ {"records":{"a":{"x":1},"b":{"x":2}}}
+ """);
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON",
+ "ItemsPointer": "/records"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-pointer-object",
+ "Key": "workers.json"
+ }
+ },
+ "ItemSelector": {
+ "key.$": "$$.Map.Item.Key",
+ "index.$": "$$.Map.Item.Index",
+ "value.$": "$$.Map.Item.Value"
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-pointer-object-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"key\":\"a\"") || output.contains("\"key\": \"a\""));
+ assertTrue(output.contains("\"key\":\"b\"") || output.contains("\"key\": \"b\""));
+ assertTrue(output.contains("\"index\":0") || output.contains("\"index\": 0"));
+ assertTrue(output.contains("\"index\":1") || output.contains("\"index\": 1"));
+ assertTrue(output.contains("\"value\":{\"x\":1}") || output.contains("\"value\": {\"x\": 1}")
+ || output.contains("\"value\": { \"x\": 1 }"));
+ assertTrue(output.contains("\"value\":{\"x\":2}") || output.contains("\"value\": {\"x\": 2}")
+ || output.contains("\"value\": { \"x\": 2 }"));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_itemsPointerMissingPathFailsWithItemReaderError() throws Exception {
+ createBucket("map-inputs-pointer-missing");
+ putObject("map-inputs-pointer-missing", "workers.json", """
+ {"records":[{"workerId":"w1"}]}
+ """);
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON",
+ "ItemsPointer": "/missing"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-pointer-missing",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-pointer-missing-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ Response failure = waitForExecutionFailure(execArn);
+
+ assertEquals("FAILED", failure.jsonPath().getString("status"));
+ assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));
+ assertEquals("The provided ReaderConfig.ItemsPointer does not match any valid path in the JSON structure.",
+ failure.jsonPath().getString("cause"));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_itemsPointerScalarFailsWithNonIterableCause() throws Exception {
+ createBucket("map-inputs-pointer-scalar");
+ putObject("map-inputs-pointer-scalar", "workers.json", """
+ {"records":123}
+ """);
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON",
+ "ItemsPointer": "/records"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-pointer-scalar",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-pointer-scalar-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ Response failure = waitForExecutionFailure(execArn);
+
+ assertEquals("FAILED", failure.jsonPath().getString("status"));
+ assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));
+ assertEquals("Attempting to map over non-iterable node.", failure.jsonPath().getString("cause"));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_emptyItemsPointerBehavesLikeOmittedPointer() throws Exception {
+ createBucket("map-inputs-pointer-empty");
+ putObject("map-inputs-pointer-empty", "workers.json", """
+ [{"workerId":"w1"},{"workerId":"w2"}]
+ """);
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON",
+ "ItemsPointer": ""
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-pointer-empty",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-pointer-empty-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));
+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));
+ }
+
+ @Test
+ void distributedMapWithItemReaderAndInlineModeFailsAtRuntime() throws Exception {
+ createBucket("map-inputs-inline");
+ putObject("map-inputs-inline", "workers.json", """
+ [{"workerId":"w1"}]
+ """);
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-inline",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-inline-mode-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ Response failure = waitForExecutionFailure(execArn);
+
+ assertEquals("FAILED", failure.jsonPath().getString("status"));
+ assertEquals("States.Runtime", failure.jsonPath().getString("error"));
+ assertEquals("The ItemReader, ItemBatcher and ResultWriter fields are not supported for INLINE maps",
+ failure.jsonPath().getString("cause"));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_exposesMapItemContextInsideProcessor() throws Exception {
+ createBucket("map-inputs-context");
+ putObject("map-inputs-context", "workers.json", "[{\"workerId\":\"w1\"},{\"workerId\":\"w2\"}]");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-context",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "ProjectContext",
+ "States": {
+ "ProjectContext": {
+ "Type": "Pass",
+ "QueryLanguage": "JSONata",
+ "Output": {
+ "index": "{% $states.context.Map.Item.Index %}",
+ "workerId": "{% $states.context.Map.Item.Value.workerId %}"
+ },
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-context-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"index\":0") || output.contains("\"index\": 0"));
+ assertTrue(output.contains("\"index\":1") || output.contains("\"index\": 1"));
+ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));
+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));
+ }
+
+ @Test
+ void distributedMapWithS3JsonArrayEntriesNamedKeyAndValue_keepsWholeElementAsMapItemValue() throws Exception {
+ createBucket("map-inputs-array-key-value");
+ putObject("map-inputs-array-key-value", "workers.json", """
+ [{"Key":"k1","Value":42},{"Key":"k2","Value":84}]
+ """);
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-array-key-value",
+ "Key": "workers.json"
+ }
+ },
+ "ItemSelector": {
+ "value.$": "$$.Map.Item.Value",
+ "index.$": "$$.Map.Item.Index"
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-array-key-value-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"index\":0") || output.contains("\"index\": 0"));
+ assertTrue(output.contains("\"index\":1") || output.contains("\"index\": 1"));
+ assertTrue(output.contains("\"value\":{\"Key\":\"k1\",\"Value\":42}")
+ || output.contains("\"value\": {\"Key\": \"k1\", \"Value\": 42}")
+ || output.contains("\"value\": { \"Key\": \"k1\", \"Value\": 42 }"));
+ assertTrue(output.contains("\"value\":{\"Key\":\"k2\",\"Value\":84}")
+ || output.contains("\"value\": {\"Key\": \"k2\", \"Value\": 84}")
+ || output.contains("\"value\": { \"Key\": \"k2\", \"Value\": 84 }"));
+ assertFalse(output.contains("\"key\":\"k1\"") || output.contains("\"key\": \"k1\""));
+ assertFalse(output.contains("\"key\":\"k2\"") || output.contains("\"key\": \"k2\""));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_invalidJsonFailsWithItemReaderError() throws Exception {
+ createBucket("map-inputs-invalid");
+ putObject("map-inputs-invalid", "workers.json", "not-json");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-invalid",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-invalid-json-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ Response failure = waitForExecutionFailure(execArn);
+
+ assertEquals("FAILED", failure.jsonPath().getString("status"));
+ assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_objectIteratesKeyValuePairs() throws Exception {
+ createBucket("map-inputs-object");
+ putObject("map-inputs-object", "workers.json", """
+ {"a":{"workerId":"w1"},"b":{"workerId":"w2"}}
+ """);
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-object",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-non-array-json-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"Key\":\"a\"") || output.contains("\"Key\": \"a\""));
+ assertTrue(output.contains("\"Key\":\"b\"") || output.contains("\"Key\": \"b\""));
+ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));
+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_objectPassesKeyValueShapeToProcessor() throws Exception {
+ createBucket("map-inputs-object-shape");
+ putObject("map-inputs-object-shape", "workers.json", """
+ {"a":{"x":1},"b":{"x":2}}
+ """);
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-object-shape",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-object-shape-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"Key\":\"a\"") || output.contains("\"Key\": \"a\""));
+ assertTrue(output.contains("\"Value\":{\"x\":1}") || output.contains("\"Value\": {\"x\": 1}")
+ || output.contains("\"Value\": { \"x\": 1 }"));
+ assertTrue(output.contains("\"Key\":\"b\"") || output.contains("\"Key\": \"b\""));
+ assertTrue(output.contains("\"Value\":{\"x\":2}") || output.contains("\"Value\": {\"x\": 2}")
+ || output.contains("\"Value\": { \"x\": 2 }"));
+ }
+
+ @Test
+ void distributedMapWithS3JsonObjectItemReader_itemSelectorExposesKeyIndexAndValue() throws Exception {
+ createBucket("map-inputs-object-selector");
+ putObject("map-inputs-object-selector", "workers.json", """
+ {"a":{"workerId":"w1"},"b":{"workerId":"w2"}}
+ """);
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-object-selector",
+ "Key": "workers.json"
+ }
+ },
+ "ItemSelector": {
+ "key.$": "$$.Map.Item.Key",
+ "index.$": "$$.Map.Item.Index",
+ "workerId.$": "$$.Map.Item.Value.workerId"
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-object-selector-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"key\":\"a\"") || output.contains("\"key\": \"a\""));
+ assertTrue(output.contains("\"key\":\"b\"") || output.contains("\"key\": \"b\""));
+ assertTrue(output.contains("\"index\":0") || output.contains("\"index\": 0"));
+ assertTrue(output.contains("\"index\":1") || output.contains("\"index\": 1"));
+ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));
+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_missingKeyFailsWithItemReaderError() throws Exception {
+ createBucket("map-inputs-missing-key");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-missing-key",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-missing-key-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ Response failure = waitForExecutionFailure(execArn);
+
+ assertEquals("FAILED", failure.jsonPath().getString("status"));
+ assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));
+ }
+
+ @Test
+ void distributedMapWithS3JsonItemReader_missingBucketFailsWithItemReaderError() throws Exception {
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-missing-bucket",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-missing-bucket-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ Response failure = waitForExecutionFailure(execArn);
+
+ assertEquals("FAILED", failure.jsonPath().getString("status"));
+ assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));
+ }
+
@Test
void statesInputVariableAccess() throws Exception {
// Verify $states.input gives access to the state's input
@@ -363,6 +1305,198 @@ class StepFunctionsJsonataIntegrationTest {
.then().statusCode(400);
}
+ @Test
+ void distributedMapWithUnsupportedItemReaderResource_rejectedAtCreateStateMachine() {
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:unknownOperation",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ given()
+ .header("X-Amz-Target", "AWSStepFunctions.CreateStateMachine")
+ .contentType(SFN_CONTENT_TYPE)
+ .body(String.format("""
+ {"name":"map-itemreader-unsupported-resource-test","definition":%s,"roleArn":"%s","type":"STANDARD"}
+ """, quote(definition), ROLE_ARN))
+ .when().post("/")
+ .then().statusCode(400);
+ }
+
+ @Test
+ void distributedMapWithListObjectsV2ItemReader_failsWithNotImplementedItemReaderError() throws Exception {
+ createBucket("map-inputs-list-objects");
+ putObject("map-inputs-list-objects", "workers/a.json", "[]");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:listObjectsV2",
+ "ReaderConfig": {
+ "InputType": "JSON"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-list-objects",
+ "Prefix": "workers/"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-list-objects-v2-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ Response failure = waitForExecutionFailure(execArn);
+
+ assertEquals("FAILED", failure.jsonPath().getString("status"));
+ assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));
+ assertTrue(failure.jsonPath().getString("cause").contains("not yet implemented by the emulator"));
+ }
+
+ @Test
+ void distributedMapWithCsvItemReader_failsWithNotImplementedItemReaderError() throws Exception {
+ createBucket("map-inputs-csv");
+ putObject("map-inputs-csv", "workers.csv", "workerId\nw1\n");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "CSV"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-csv",
+ "Key": "workers.csv"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-csv-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ Response failure = waitForExecutionFailure(execArn);
+
+ assertEquals("FAILED", failure.jsonPath().getString("status"));
+ assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));
+ assertTrue(failure.jsonPath().getString("cause").contains("InputType CSV is not yet implemented by the emulator"));
+ }
+
+ @Test
+ void distributedMapWithUnsupportedItemReaderInputType_rejectedAtCreateStateMachine() {
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "UNSUPPORTED"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs",
+ "Key": "workers.json"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ given()
+ .header("X-Amz-Target", "AWSStepFunctions.CreateStateMachine")
+ .contentType(SFN_CONTENT_TYPE)
+ .body(String.format("""
+ {"name":"map-itemreader-unsupported-inputtype-test","definition":%s,"roleArn":"%s","type":"STANDARD"}
+ """, quote(definition), ROLE_ARN))
+ .when().post("/")
+ .then().statusCode(400);
+ }
+
// ──────────────── Helpers ────────────────
private String createStateMachine(String name, String definition) {
@@ -400,14 +1534,7 @@ class StepFunctionsJsonataIntegrationTest {
private String waitForExecution(String execArn) throws InterruptedException {
for (int i = 0; i < 50; i++) {
- Response resp = given()
- .header("X-Amz-Target", "AWSStepFunctions.DescribeExecution")
- .contentType(SFN_CONTENT_TYPE)
- .body(String.format("""
- { "executionArn": "%s" }
- """, execArn))
- .when()
- .post("/");
+ Response resp = describeExecution(execArn);
String status = resp.jsonPath().getString("status");
if ("SUCCEEDED".equals(status)) {
return resp.jsonPath().getString("output");
@@ -421,6 +1548,50 @@ class StepFunctionsJsonataIntegrationTest {
return null;
}
+ private Response waitForExecutionFailure(String execArn) throws InterruptedException {
+ for (int i = 0; i < 50; i++) {
+ Response resp = describeExecution(execArn);
+ String status = resp.jsonPath().getString("status");
+ if ("FAILED".equals(status) || "ABORTED".equals(status)) {
+ return resp;
+ }
+ if ("SUCCEEDED".equals(status)) {
+ fail("Execution SUCCEEDED: " + resp.body().asString());
+ }
+ Thread.sleep(100);
+ }
+ fail("Execution did not fail within timeout");
+ return null;
+ }
+
+ private Response describeExecution(String execArn) {
+ return given()
+ .header("X-Amz-Target", "AWSStepFunctions.DescribeExecution")
+ .contentType(SFN_CONTENT_TYPE)
+ .body(String.format("""
+ { "executionArn": "%s" }
+ """, execArn))
+ .when()
+ .post("/");
+ }
+
+ private void createBucket(String bucket) {
+ given()
+ .when()
+ .put("/" + bucket)
+ .then()
+ .statusCode(200);
+ }
+
+ private void putObject(String bucket, String key, String body) {
+ given()
+ .body(body)
+ .when()
+ .put("/" + bucket + "/" + key)
+ .then()
+ .statusCode(200);
+ }
+
/**
* JSON-encode a string value (escape and wrap in quotes) for embedding
* inside a JSON body where the field expects a string.
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java
index 33f1c1ba3d..8f9ae51359 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java
@@ -40,6 +40,50 @@ class StepFunctionsValidateStateMachineDefinitionIntegrationTest {
+ "\\\"InputPath\\\":\\\"$.a\\\","
+ "\\\"OutputPath\\\":\\\"$.b\\\","
+ "\\\"ResultPath\\\":\\\"$.c\\\"}}}";
+ private static final String MAP_WITH_UNSUPPORTED_ITEM_READER_RESOURCE =
+ "{\\\"StartAt\\\":\\\"ProcessItems\\\",\\\"States\\\":{\\\"ProcessItems\\\":{"
+ + "\\\"Type\\\":\\\"Map\\\","
+ + "\\\"ItemReader\\\":{"
+ + "\\\"Resource\\\":\\\"arn:aws:states:::s3:unknownOperation\\\","
+ + "\\\"ReaderConfig\\\":{\\\"InputType\\\":\\\"JSON\\\"},"
+ + "\\\"Parameters\\\":{\\\"Bucket\\\":\\\"map-inputs\\\",\\\"Key\\\":\\\"workers.json\\\"}"
+ + "},"
+ + "\\\"ItemProcessor\\\":{"
+ + "\\\"ProcessorConfig\\\":{\\\"Mode\\\":\\\"DISTRIBUTED\\\",\\\"ExecutionType\\\":\\\"STANDARD\\\"},"
+ + "\\\"StartAt\\\":\\\"PassItem\\\","
+ + "\\\"States\\\":{\\\"PassItem\\\":{\\\"Type\\\":\\\"Pass\\\",\\\"End\\\":true}}"
+ + "},"
+ + "\\\"End\\\":true"
+ + "}}}";
+ private static final String MAP_WITH_UNSUPPORTED_ITEM_READER_INPUT_TYPE =
+ "{\\\"StartAt\\\":\\\"ProcessItems\\\",\\\"States\\\":{\\\"ProcessItems\\\":{"
+ + "\\\"Type\\\":\\\"Map\\\","
+ + "\\\"ItemReader\\\":{"
+ + "\\\"Resource\\\":\\\"arn:aws:states:::s3:getObject\\\","
+ + "\\\"ReaderConfig\\\":{\\\"InputType\\\":\\\"UNSUPPORTED\\\"},"
+ + "\\\"Parameters\\\":{\\\"Bucket\\\":\\\"map-inputs\\\",\\\"Key\\\":\\\"workers.json\\\"}"
+ + "},"
+ + "\\\"ItemProcessor\\\":{"
+ + "\\\"ProcessorConfig\\\":{\\\"Mode\\\":\\\"DISTRIBUTED\\\",\\\"ExecutionType\\\":\\\"STANDARD\\\"},"
+ + "\\\"StartAt\\\":\\\"PassItem\\\","
+ + "\\\"States\\\":{\\\"PassItem\\\":{\\\"Type\\\":\\\"Pass\\\",\\\"End\\\":true}}"
+ + "},"
+ + "\\\"End\\\":true"
+ + "}}}";
+ private static final String MAP_WITH_ITEM_READER_WITHOUT_DISTRIBUTED_MODE =
+ "{\\\"StartAt\\\":\\\"ProcessItems\\\",\\\"States\\\":{\\\"ProcessItems\\\":{"
+ + "\\\"Type\\\":\\\"Map\\\","
+ + "\\\"ItemReader\\\":{"
+ + "\\\"Resource\\\":\\\"arn:aws:states:::s3:getObject\\\","
+ + "\\\"ReaderConfig\\\":{\\\"InputType\\\":\\\"JSON\\\"},"
+ + "\\\"Parameters\\\":{\\\"Bucket\\\":\\\"map-inputs\\\",\\\"Key\\\":\\\"workers.json\\\"}"
+ + "},"
+ + "\\\"ItemProcessor\\\":{"
+ + "\\\"StartAt\\\":\\\"PassItem\\\","
+ + "\\\"States\\\":{\\\"PassItem\\\":{\\\"Type\\\":\\\"Pass\\\",\\\"End\\\":true}}"
+ + "},"
+ + "\\\"End\\\":true"
+ + "}}}";
@BeforeAll
static void configure() {
@@ -88,6 +132,43 @@ class StepFunctionsValidateStateMachineDefinitionIntegrationTest {
.body("diagnostics[0].location", equalTo("/States/X/InputPath"));
}
+ @Test
+ void unsupportedItemReaderResource_returnsFailWithSchemaError() {
+ given().contentType(CT).header("X-Amz-Target", TARGET)
+ .body("{\"definition\":\"" + MAP_WITH_UNSUPPORTED_ITEM_READER_RESOURCE + "\"}")
+ .when().post("/")
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].severity", equalTo("ERROR"))
+ .body("diagnostics[0].code", equalTo("SCHEMA_VALIDATION_FAILED"))
+ .body("diagnostics[0].location", equalTo("/States/ProcessItems/ItemReader/Resource"));
+ }
+
+ @Test
+ void unsupportedItemReaderInputType_returnsFailWithSchemaError() {
+ given().contentType(CT).header("X-Amz-Target", TARGET)
+ .body("{\"definition\":\"" + MAP_WITH_UNSUPPORTED_ITEM_READER_INPUT_TYPE + "\"}")
+ .when().post("/")
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].severity", equalTo("ERROR"))
+ .body("diagnostics[0].code", equalTo("SCHEMA_VALIDATION_FAILED"))
+ .body("diagnostics[0].location", equalTo("/States/ProcessItems/ItemReader/ReaderConfig/InputType"));
+ }
+
+ @Test
+ void itemReaderWithoutDistributedMode_isAcceptedAtDefinitionTime() {
+ given().contentType(CT).header("X-Amz-Target", TARGET)
+ .body("{\"definition\":\"" + MAP_WITH_ITEM_READER_WITHOUT_DISTRIBUTED_MODE + "\"}")
+ .when().post("/")
+ .then().statusCode(200)
+ .body("result", equalTo("OK"))
+ .body("diagnostics", hasSize(0))
+ .body("truncated", is(false));
+ }
+
@Test
void emptyDefinition_returns400() {
given().contentType(CT).header("X-Amz-Target", TARGET)