Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(sfn): support ItemReader InputType JSONL (#3602)

Ovidiu (Ovi)ef341e1c01da
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.md
index e4f2d9ae27..37064893c1 100644
--- a/docs/services/step-functions.md
+++ b/docs/services/step-functions.md
@@ -127,12 +127,23 @@ iterations.
## Distributed Map ItemReader
-`ItemReader` supports `arn:aws:states:::s3:getObject` with `InputType: JSON` and
-`arn:aws:states:::s3:listObjectsV2`. The listing reads every page under `Prefix`, and each item
-carries the AWS fields `Etag`, `Key`, `LastModified` (epoch seconds), `Size` and `StorageClass`.
-An empty prefix gives zero iterations and the Map succeeds. `ReaderConfig.MaxItems` applies to
-both readers; `MaxItemsPath` is not supported. The `CSV`, `JSONL`, `PARQUET` and `MANIFEST` input
-types are accepted by `CreateStateMachine` and fail the execution with `States.ItemReaderFailed`.
+`ItemReader` reads a dataset from S3. The resource decides how the dataset is found, and
+`ReaderConfig.InputType` decides how it is read.
+
+`arn:aws:states:::s3:getObject` reads a single object:
+
+- `JSON` is either an array, or an object whose entries become `Key` and `Value` items.
+ `ReaderConfig.ItemsPointer` selects a node inside it.
+- `JSONL` is one item per line. Blank lines are skipped, and `ItemsPointer` does not apply,
+ matching AWS.
+- `CSV`, `PARQUET` and `MANIFEST` are accepted by `CreateStateMachine` and fail the execution
+ with `States.ItemReaderFailed`.
+
+`arn:aws:states:::s3:listObjectsV2` reads every page under `Prefix`. Each item carries the AWS
+fields `Etag`, `Key`, `LastModified` (epoch seconds), `Size` and `StorageClass`. An empty prefix
+gives zero iterations and the Map succeeds.
+
+`ReaderConfig.MaxItems` applies to every reader; `MaxItemsPath` is not supported.
## Distributed Map ItemBatcher
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index a89d08d965..e4fed0fc7b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -3077,7 +3077,7 @@ public class AslExecutor {
}
String inputType = itemReader.path("ReaderConfig").path("InputType").asText(null);
- if (!"JSON".equals(inputType)) {
+ if (!"JSON".equals(inputType) && !"JSONL".equals(inputType)) {
throw new FailStateException("States.ItemReaderFailed",
"ItemReader InputType " + inputType + " is not yet implemented by the emulator");
}
@@ -3091,6 +3091,10 @@ public class AslExecutor {
try {
S3Object object = s3Service.getObject(bucket, key);
+ if ("JSONL".equals(inputType)) {
+ return new ResolvedMapItems(applyMaxItems(itemReader, readJsonLines(object.getData())),
+ MapItemsSource.ITEM_READER_ARRAY);
+ }
JsonNode items = objectMapper.readTree(object.getData());
items = applyItemsPointer(itemReader, items);
if (items.isObject()) {
@@ -3112,6 +3116,20 @@ public class AslExecutor {
}
}
+ /**
+ * One item per non-empty line. ReaderConfig.ItemsPointer is JSON only on AWS, so a JSONL
+ * dataset is always the whole file.
+ */
+ private ArrayNode readJsonLines(byte[] data) throws IOException {
+ ArrayNode items = objectMapper.createArrayNode();
+ for (String line : new String(data, StandardCharsets.UTF_8).split("\\R")) {
+ if (!line.isBlank()) {
+ items.add(objectMapper.readTree(line));
+ }
+ }
+ return items;
+ }
+
private JsonNode resolveItemReaderParameters(JsonNode itemReader, JsonNode input, JsonNode context,
boolean jsonata, ObjectNode variables) throws Exception {
if (jsonata && itemReader.has("Arguments")) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
index cf6d685ddf..f8d652c823 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
@@ -950,6 +950,154 @@ class StepFunctionsJsonataIntegrationTest {
assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));
}
+ @Test
+ void distributedMapWithS3JsonlItemReader_readsOneItemPerLine() throws Exception {
+ createBucket("map-inputs-jsonl");
+ putObject("map-inputs-jsonl", "workers.jsonl",
+ "{\"workerId\":\"w1\"}\n{\"workerId\":\"w2\"}\n\n");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSONL"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-jsonl",
+ "Key": "workers.jsonl"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-jsonl-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));
+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));
+ }
+
+ @Test
+ void distributedMapWithS3JsonlItemReader_maxItemsLimitsDataset() throws Exception {
+ createBucket("map-inputs-jsonl-max-items");
+ putObject("map-inputs-jsonl-max-items", "workers.jsonl",
+ "{\"workerId\":\"w1\"}\n{\"workerId\":\"w2\"}\n{\"workerId\":\"w3\"}\n");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSONL",
+ "MaxItems": 2
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-jsonl-max-items",
+ "Key": "workers.jsonl"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-jsonl-max-items-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));
+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));
+ assertFalse(output.contains("\"workerId\":\"w3\"") || output.contains("\"workerId\": \"w3\""));
+ }
+
+ @Test
+ void distributedMapWithS3JsonlItemReader_malformedLineFailsWithItemReaderError() throws Exception {
+ createBucket("map-inputs-jsonl-invalid");
+ putObject("map-inputs-jsonl-invalid", "workers.jsonl", "{\"workerId\":\"w1\"}\nnot-json\n");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "JSONL"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-jsonl-invalid",
+ "Key": "workers.jsonl"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-jsonl-invalid-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ Response failure = waitForExecutionFailure(execArn);
+
+ assertEquals("FAILED", failure.jsonPath().getString("status"));
+ assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));
+ }
+
@Test
void distributedMapWithItemBatcher_groupsItemsIntoBatchesOfMaxItemsPerBatch() throws Exception {
String definition = """