Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(sfn): Map ItemReader with s3:listObjectsV2 (#3460)

TATSUNO “Taz” Yasuhiro5db516d9d522
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.md
index 5782ff95a1..e4f2d9ae27 100644
--- a/docs/services/step-functions.md
+++ b/docs/services/step-functions.md
@@ -125,6 +125,15 @@ Results remain in input order even when iterations finish out of order. If an it
the Map state fails promptly, cancels its active sibling iterations, and does not start queued
iterations.
+## Distributed Map ItemReader
+
+`ItemReader` supports `arn:aws:states:::s3:getObject` with `InputType: JSON` and
+`arn:aws:states:::s3:listObjectsV2`. The listing reads every page under `Prefix`, and each item
+carries the AWS fields `Etag`, `Key`, `LastModified` (epoch seconds), `Size` and `StorageClass`.
+An empty prefix gives zero iterations and the Map succeeds. `ReaderConfig.MaxItems` applies to
+both readers; `MaxItemsPath` is not supported. The `CSV`, `JSONL`, `PARQUET` and `MANIFEST` input
+types are accepted by `CreateStateMachine` and fail the execution with `States.ItemReaderFailed`.
+
## Distributed Map ItemBatcher
`ItemBatcher` hands each child execution a batch of items instead of a single item. The child input
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index 0ed2467e7f..a89d08d965 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -2407,15 +2407,22 @@ public class AslExecutor {
long startMs = hasResultWriter ? System.currentTimeMillis() : 0L;
JsonNode branchOutput;
+ // AWS evaluates ItemSelector before it records MapIterationStarted, so a failing
+ // expression fails the Map state without any event for that iteration.
+ boolean iterationStarted = false;
try {
- if (!distributed) {
- iterationChain.publish("MapIterationStarted", Map.of("name", name, "index", i));
- }
JsonNode iterInput = item;
if (!batchedChild && itemTransform != null) {
// $ in ItemSelector resolves against the Map state's effective input, not the item.
- iterInput = resolveParameters(itemTransform, mapInput, iterContext);
+ iterInput = jsonata
+ ? jsonataEvaluator.resolveTemplate(itemTransform, "ItemSelector",
+ buildStatesVar(mapInput, null, iterContext), variables)
+ : resolveParameters(itemTransform, mapInput, iterContext);
}
+ if (!distributed) {
+ iterationChain.publish("MapIterationStarted", Map.of("name", name, "index", i));
+ }
+ iterationStarted = true;
if (hasResultWriter) {
childInputsByIndex[i] = iterInput;
}
@@ -2429,7 +2436,7 @@ public class AslExecutor {
} catch (FailStateException e) {
int failedSoFar = failedItems.addAndGet(itemsInChild);
failedExecutions.incrementAndGet();
- if (!distributed && !e.isRuntimeError()) {
+ if (iterationStarted && !distributed && !e.isRuntimeError()) {
iterationChain.publishAside("MapIterationFailed", Map.of("name", name, "index", i));
}
if (!tolerated.declared()) {
@@ -3063,8 +3070,7 @@ public class AslExecutor {
ObjectNode variables) throws Exception {
String resource = itemReader.path("Resource").asText(null);
if ("arn:aws:states:::s3:listObjectsV2".equals(resource)) {
- throw new FailStateException("States.ItemReaderFailed",
- "ItemReader resource arn:aws:states:::s3:listObjectsV2 is not yet implemented by the emulator");
+ return resolveListObjectsItems(itemReader, input, context, jsonata, variables);
}
if (!"arn:aws:states:::s3:getObject".equals(resource)) {
throw new FailStateException("States.Runtime", "Unsupported ItemReader resource: " + resource);
@@ -3076,15 +3082,7 @@ public class AslExecutor {
"ItemReader InputType " + inputType + " is not yet implemented by the emulator");
}
- JsonNode resolvedParameters;
- if (jsonata && itemReader.has("Arguments")) {
- JsonNode statesVar = buildStatesVar(input, null, context);
- resolvedParameters = jsonataEvaluator.resolveTemplate(
- itemReader.get("Arguments"), "ItemReader/Arguments", statesVar, variables);
- } else {
- JsonNode parameters = itemReader.path("Parameters");
- resolvedParameters = resolveParameters(parameters, input, context);
- }
+ JsonNode resolvedParameters = resolveItemReaderParameters(itemReader, input, context, jsonata, variables);
String bucket = resolvedParameters.path("Bucket").asText(null);
String key = resolvedParameters.path("Key").asText(null);
if (bucket == null || key == null) {
@@ -3114,6 +3112,56 @@ public class AslExecutor {
}
}
+ private JsonNode resolveItemReaderParameters(JsonNode itemReader, JsonNode input, JsonNode context,
+ boolean jsonata, ObjectNode variables) throws Exception {
+ if (jsonata && itemReader.has("Arguments")) {
+ JsonNode statesVar = buildStatesVar(input, null, context);
+ return jsonataEvaluator.resolveTemplate(
+ itemReader.get("Arguments"), "ItemReader/Arguments", statesVar, variables);
+ }
+ return resolveParameters(itemReader.path("Parameters"), input, context);
+ }
+
+ private ResolvedMapItems resolveListObjectsItems(JsonNode itemReader, JsonNode input, JsonNode context,
+ boolean jsonata, ObjectNode variables) throws Exception {
+ JsonNode parameters = resolveItemReaderParameters(itemReader, input, context, jsonata, variables);
+ String bucket = parameters.path("Bucket").asText(null);
+ if (bucket == null) {
+ throw new FailStateException("States.Runtime", "ItemReader Parameters must include Bucket");
+ }
+ String prefix = parameters.path("Prefix").asText(null);
+
+ ArrayNode items = objectMapper.createArrayNode();
+ int maxItems = maxItems(itemReader);
+ try {
+ // MaxItems keeps the first keys in order, so the listing itself is capped.
+ for (S3Object object : s3Service.listObjects(bucket, prefix, null,
+ maxItems > 0 ? maxItems : Integer.MAX_VALUE)) {
+ items.add(listObjectsItem(object, jsonata));
+ }
+ } catch (AwsException e) {
+ throw new FailStateException("States.ItemReaderFailed", e.getMessage());
+ }
+ return new ResolvedMapItems(items, MapItemsSource.ITEM_READER_ARRAY);
+ }
+
+ // AWS renders LastModified as epoch seconds: a double in JSONPath state machines and an
+ // integer in JSONata ones.
+ private ObjectNode listObjectsItem(S3Object object, boolean jsonata) {
+ ObjectNode item = objectMapper.createObjectNode();
+ item.put("Etag", object.getETag());
+ item.put("Key", object.getKey());
+ long lastModified = object.getLastModified().getEpochSecond();
+ if (jsonata) {
+ item.put("LastModified", lastModified);
+ } else {
+ item.put("LastModified", (double) lastModified);
+ }
+ item.put("Size", object.getSize());
+ item.put("StorageClass", object.getStorageClass());
+ return item;
+ }
+
private ArrayNode normalizeObjectItems(JsonNode items) {
ArrayNode normalized = objectMapper.createArrayNode();
items.fields().forEachRemaining(entry -> {
@@ -3139,8 +3187,12 @@ public class AslExecutor {
return pointedItems;
}
+ private int maxItems(JsonNode itemReader) {
+ return itemReader.path("ReaderConfig").path("MaxItems").asInt(0);
+ }
+
private JsonNode applyMaxItems(JsonNode itemReader, JsonNode items) {
- int maxItems = itemReader.path("ReaderConfig").path("MaxItems").asInt(0);
+ int maxItems = maxItems(itemReader);
if (maxItems <= 0 || !items.isArray() || items.size() <= maxItems) {
return items;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
index 0f9ba77e83..cf6d685ddf 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
@@ -2,18 +2,26 @@ package io.github.hectorvent.floci.services.stepfunctions;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
import io.restassured.response.Response;
+import jakarta.inject.Inject;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import java.util.HashMap;
+import java.util.List;
+
import static io.restassured.RestAssured.given;
import static org.junit.jupiter.api.Assertions.*;
@QuarkusTest
class StepFunctionsJsonataIntegrationTest {
+ @Inject
+ S3Service s3Service;
+
private static final String SFN_CONTENT_TYPE = "application/x-amz-json-1.0";
private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/test-role";
@@ -2260,6 +2268,91 @@ class StepFunctionsJsonataIntegrationTest {
+ "returned nothing (undefined).", failure.jsonPath().getString("cause"));
}
+ private static final String JSONATA_MAP_ITEM_SELECTOR_DEFINITION = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "M",
+ "States": {
+ "M": {
+ "Type": "Map",
+ "Items": "{% $states.input.numbers %}",
+ "ItemSelector": {
+ "n": "{% $states.context.Map.Item.Value %}",
+ "i": "{% $states.context.Map.Item.Index %}",
+ "label": "{% $states.input.label %}"
+ },
+ "ItemProcessor": {
+ "StartAt": "P",
+ "States": {"P": {"Type": "Pass", "End": true}}
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ /**
+ * Checked against real Step Functions (us-east-1, 2026-09-12): inside ItemSelector,
+ * $states.input is the Map state's input and $states.context.Map.Item carries Value and Index.
+ */
+ @Test
+ void mapItemSelectorEvaluatesJsonataAgainstMapInputAndItemContext() throws Exception {
+ String smArn = createStateMachine("jsonata-map-item-selector-test", JSONATA_MAP_ITEM_SELECTOR_DEFINITION);
+ String output = waitForExecution(startExecution(smArn, "{\"numbers\":[10,20],\"label\":\"x\"}"));
+
+ assertEquals("[{\"n\":10,\"i\":0,\"label\":\"x\"},{\"n\":20,\"i\":1,\"label\":\"x\"}]", output);
+ }
+
+ @Test
+ void mapItemSelectorReturningNothingFailsTheStateNamingTheField() throws Exception {
+ // Real AWS names 'ItemSelector/<field>'.
+ String smArn = createStateMachine("jsonata-map-item-selector-returned-nothing-test",
+ JSONATA_MAP_ITEM_SELECTOR_DEFINITION);
+ String execArn = startExecution(smArn, "{\"numbers\":[10]}");
+ Response failure = waitForExecutionFailure(execArn);
+
+ assertEquals("States.QueryEvaluationError", failure.jsonPath().getString("error"));
+ assertEquals("An error occurred while executing the state 'M' (entered at the event id #2). "
+ + "The JSONata expression '$states.input.label' specified for the field 'ItemSelector/label' "
+ + "returned nothing (undefined).", failure.jsonPath().getString("cause"));
+ // AWS evaluates ItemSelector before it records the iteration, so no MapIteration* event.
+ assertEquals(List.of("ExecutionStarted", "MapStateEntered", "MapStateStarted",
+ "EvaluationFailed", "MapStateFailed", "ExecutionFailed"), historyEventTypes(execArn));
+ }
+
+ @Test
+ void mapItemSelectorMissingJsonPathFailsBeforeTheIterationIsRecorded() throws Exception {
+ // Real AWS: States.Runtime straight after MapStateStarted, no MapIteration* event.
+ String definition = """
+ {
+ "StartAt": "M",
+ "States": {
+ "M": {
+ "Type": "Map",
+ "ItemsPath": "$.items",
+ "ItemSelector": {"v.$": "$$.Map.Item.Value.x"},
+ "ItemProcessor": {
+ "StartAt": "P",
+ "States": {"P": {"Type": "Pass", "End": true}}
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("jsonpath-map-item-selector-missing-path-test", definition);
+ String execArn = startExecution(smArn, "{\"items\":[{}]}");
+ Response failure = waitForExecutionFailure(execArn);
+
+ assertEquals("States.Runtime", failure.jsonPath().getString("error"));
+ assertTrue(failure.jsonPath().getString("cause").contains(
+ "specified for the field 'v.$' could not be found in the input"),
+ failure.jsonPath().getString("cause"));
+ assertEquals(List.of("ExecutionStarted", "MapStateEntered", "MapStateStarted", "ExecutionFailed"),
+ historyEventTypes(execArn));
+ }
+
@Test
void mapMaxConcurrencyReturningNothingFailsTheStateNamingTheField() throws Exception {
// Real AWS names 'MaxConcurrency'. Before this guard the undefined value reached the
@@ -2809,10 +2902,67 @@ class StepFunctionsJsonataIntegrationTest {
.then().statusCode(400);
}
+ /**
+ * The listObjectsV2 ItemReader expectations below were checked against real Step Functions
+ * (us-east-1, 2026-09-12): the item fields, the quoted Etag, LastModified as epoch seconds
+ * rendered as a double in JSONPath state machines and as an integer in JSONata ones, keys
+ * returned as stored, every page read, and an empty prefix succeeding with zero iterations.
+ */
@Test
- void distributedMapWithListObjectsV2ItemReader_failsWithNotImplementedItemReaderError() throws Exception {
+ void distributedMapWithListObjectsV2ItemReader_iteratesObjectsUnderPrefix() throws Exception {
createBucket("map-inputs-list-objects");
putObject("map-inputs-list-objects", "workers/a.json", "[]");
+ putObject("map-inputs-list-objects", "workers/b+c.json", "{}");
+ putObject("map-inputs-list-objects", "workers/c.json", "[1]");
+ putObject("map-inputs-list-objects", "other/z.json", "[]");
+
+ String definition = listObjectsDefinition("""
+ "Parameters": {
+ "Bucket": "map-inputs-list-objects",
+ "Prefix.$": "$.prefix"
+ }
+ """);
+
+ String smArn = createStateMachine("map-itemreader-s3-list-objects-v2-test", definition);
+ String execArn = startExecution(smArn, "{\"prefix\":\"workers/\"}");
+ String output = waitForExecution(execArn);
+ JsonNode items = objectMapper.readTree(output);
+
+ assertEquals(3, items.size(), output);
+ JsonNode first = items.get(0);
+ assertEquals("\"d751713988987e9331980363e24189ce\"", first.path("Etag").asText());
+ assertEquals("workers/a.json", first.path("Key").asText());
+ assertTrue(first.path("LastModified").isDouble(), output);
+ assertEquals(2, first.path("Size").asInt());
+ assertEquals("STANDARD", first.path("StorageClass").asText());
+ assertEquals("workers/b+c.json", items.get(1).path("Key").asText());
+ assertEquals("workers/c.json", items.get(2).path("Key").asText());
+ }
+
+ @Test
+ void distributedMapWithListObjectsV2ItemReader_emptyPrefixSucceedsWithNoIterations() throws Exception {
+ createBucket("map-inputs-list-objects-empty");
+ putObject("map-inputs-list-objects-empty", "other/z.json", "[]");
+
+ String definition = listObjectsDefinition("""
+ "Parameters": {
+ "Bucket": "map-inputs-list-objects-empty",
+ "Prefix": "workers/"
+ }
+ """);
+
+ String smArn = createStateMachine("map-itemreader-s3-list-objects-v2-empty-test", definition);
+ String execArn = startExecution(smArn, "{}");
+
+ assertEquals("[]", waitForExecution(execArn));
+ }
+
+ @Test
+ void distributedMapWithListObjectsV2ItemReader_honorsMaxItemsAndItemSelector() throws Exception {
+ createBucket("map-inputs-list-objects-max");
+ putObject("map-inputs-list-objects-max", "workers/a.json", "[]");
+ putObject("map-inputs-list-objects-max", "workers/b.json", "[]");
+ putObject("map-inputs-list-objects-max", "workers/c.json", "[]");
String definition = """
{
@@ -2823,13 +2973,17 @@ class StepFunctionsJsonataIntegrationTest {
"ItemReader": {
"Resource": "arn:aws:states:::s3:listObjectsV2",
"ReaderConfig": {
- "InputType": "JSON"
+ "MaxItems": 2
},
"Parameters": {
- "Bucket": "map-inputs-list-objects",
+ "Bucket": "map-inputs-list-objects-max",
"Prefix": "workers/"
}
},
+ "ItemSelector": {
+ "key.$": "$$.Map.Item.Value.Key",
+ "size.$": "$$.Map.Item.Value.Size"
+ },
"ItemProcessor": {
"ProcessorConfig": {
"Mode": "DISTRIBUTED",
@@ -2849,13 +3003,127 @@ class StepFunctionsJsonataIntegrationTest {
}
""";
- String smArn = createStateMachine("map-itemreader-s3-list-objects-v2-test", definition);
+ String smArn = createStateMachine("map-itemreader-s3-list-objects-v2-max-test", definition);
String execArn = startExecution(smArn, "{}");
- Response failure = waitForExecutionFailure(execArn);
+ JsonNode items = objectMapper.readTree(waitForExecution(execArn));
- assertEquals("FAILED", failure.jsonPath().getString("status"));
- assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));
- assertTrue(failure.jsonPath().getString("cause").contains("not yet implemented by the emulator"));
+ assertEquals(2, items.size());
+ assertEquals("workers/a.json", items.get(0).path("key").asText());
+ assertEquals(2, items.get(0).path("size").asInt());
+ assertEquals("workers/b.json", items.get(1).path("key").asText());
+ }
+
+ @Test
+ void distributedMapWithListObjectsV2ItemReader_readsMoreThanOneThousandObjects() throws Exception {
+ // In-process puts; 1001 REST puts are slow.
+ s3Service.createBucket("map-inputs-list-objects-pages", "us-east-1");
+ for (int i = 1; i <= 1001; i++) {
+ s3Service.putObject("map-inputs-list-objects-pages", String.format("workers/%04d.json", i),
+ "[]".getBytes(), "application/json", new HashMap<>());
+ }
+
+ String definition = listObjectsDefinition("""
+ "Parameters": {
+ "Bucket": "map-inputs-list-objects-pages",
+ "Prefix": "workers/"
+ }
+ """);
+
+ String smArn = createStateMachine("map-itemreader-s3-list-objects-v2-pages-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ JsonNode items = objectMapper.readTree(waitForExecution(execArn, 300));
+
+ assertEquals(1001, items.size());
+ assertEquals("workers/0001.json", items.get(0).path("Key").asText());
+ assertEquals("workers/1001.json", items.get(1000).path("Key").asText());
+ }
+
+ @Test
+ void distributedMapWithListObjectsV2ItemReader_jsonataArgumentsAndItemSelectorBuildChildInputs() throws Exception {
+ createBucket("map-inputs-list-objects-jsonata");
+ putObject("map-inputs-list-objects-jsonata", "workers/a.json", "[]");
+ putObject("map-inputs-list-objects-jsonata", "workers/b.json", "[]");
+
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:listObjectsV2",
+ "Arguments": {
+ "Bucket": "{% $states.input.bucket %}",
+ "Prefix": "{% $states.input.prefix %}"
+ }
+ },
+ "ItemSelector": {
+ "bucket": "{% $states.input.bucket %}",
+ "key": "{% $states.context.Map.Item.Value.Key %}",
+ "modified": "{% $states.context.Map.Item.Value.LastModified %}"
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-list-objects-v2-jsonata-test", definition);
+ String execArn = startExecution(smArn,
+ "{\"bucket\":\"map-inputs-list-objects-jsonata\",\"prefix\":\"workers/\"}");
+ String output = waitForExecution(execArn);
+ JsonNode items = objectMapper.readTree(output);
+
+ assertEquals(2, items.size(), output);
+ assertEquals("map-inputs-list-objects-jsonata", items.get(0).path("bucket").asText());
+ assertEquals("workers/a.json", items.get(0).path("key").asText());
+ assertTrue(items.get(0).path("modified").isIntegralNumber(), output);
+ assertEquals("workers/b.json", items.get(1).path("key").asText());
+ }
+
+ private static String listObjectsDefinition(String itemReaderFields) {
+ return String.format("""
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:listObjectsV2",
+ %s
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """, itemReaderFields);
}
@Test
@@ -2990,7 +3258,11 @@ class StepFunctionsJsonataIntegrationTest {
}
private String waitForExecution(String execArn) throws InterruptedException {
- for (int i = 0; i < 50; i++) {
+ return waitForExecution(execArn, 50);
+ }
+
+ private String waitForExecution(String execArn, int attempts) throws InterruptedException {
+ for (int i = 0; i < attempts; i++) {
Response resp = describeExecution(execArn);
String status = resp.jsonPath().getString("status");
if ("SUCCEEDED".equals(status)) {
@@ -3021,6 +3293,18 @@ class StepFunctionsJsonataIntegrationTest {
return null;
}
+ private List<String> historyEventTypes(String execArn) {
+ return given()
+ .header("X-Amz-Target", "AWSStepFunctions.GetExecutionHistory")
+ .contentType(SFN_CONTENT_TYPE)
+ .body(String.format("""
+ { "executionArn": "%s" }
+ """, execArn))
+ .when()
+ .post("/")
+ .jsonPath().getList("events.type", String.class);
+ }
+
private Response describeExecution(String execArn) {
return given()
.header("X-Amz-Target", "AWSStepFunctions.DescribeExecution")