Light, fluffy, and always free - The AWS Local Emulator alternative
feat(sfn): support ItemReader InputType JSONL (#3602)
Ovidiu (Ovi)ef341e1c01da
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.mdindex e4f2d9ae27..37064893c1 100644--- a/docs/services/step-functions.md+++ b/docs/services/step-functions.md@@ -127,12 +127,23 @@ iterations.## Distributed Map ItemReader-`ItemReader` supports `arn:aws:states:::s3:getObject` with `InputType: JSON` and-`arn:aws:states:::s3:listObjectsV2`. The listing reads every page under `Prefix`, and each item-carries the AWS fields `Etag`, `Key`, `LastModified` (epoch seconds), `Size` and `StorageClass`.-An empty prefix gives zero iterations and the Map succeeds. `ReaderConfig.MaxItems` applies to-both readers; `MaxItemsPath` is not supported. The `CSV`, `JSONL`, `PARQUET` and `MANIFEST` input-types are accepted by `CreateStateMachine` and fail the execution with `States.ItemReaderFailed`.+`ItemReader` reads a dataset from S3. The resource decides how the dataset is found, and+`ReaderConfig.InputType` decides how it is read.++`arn:aws:states:::s3:getObject` reads a single object:++- `JSON` is either an array, or an object whose entries become `Key` and `Value` items.+ `ReaderConfig.ItemsPointer` selects a node inside it.+- `JSONL` is one item per line. Blank lines are skipped, and `ItemsPointer` does not apply,+ matching AWS.+- `CSV`, `PARQUET` and `MANIFEST` are accepted by `CreateStateMachine` and fail the execution+ with `States.ItemReaderFailed`.++`arn:aws:states:::s3:listObjectsV2` reads every page under `Prefix`. Each item carries the AWS+fields `Etag`, `Key`, `LastModified` (epoch seconds), `Size` and `StorageClass`. An empty prefix+gives zero iterations and the Map succeeds.++`ReaderConfig.MaxItems` applies to every reader; `MaxItemsPath` is not supported.## Distributed Map ItemBatcherdiff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.javaindex a89d08d965..e4fed0fc7b 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java@@ -3077,7 +3077,7 @@ public class AslExecutor {}String inputType = itemReader.path("ReaderConfig").path("InputType").asText(null);- if (!"JSON".equals(inputType)) {+ if (!"JSON".equals(inputType) && !"JSONL".equals(inputType)) {throw new FailStateException("States.ItemReaderFailed","ItemReader InputType " + inputType + " is not yet implemented by the emulator");}@@ -3091,6 +3091,10 @@ public class AslExecutor {try {S3Object object = s3Service.getObject(bucket, key);+ if ("JSONL".equals(inputType)) {+ return new ResolvedMapItems(applyMaxItems(itemReader, readJsonLines(object.getData())),+ MapItemsSource.ITEM_READER_ARRAY);+ }JsonNode items = objectMapper.readTree(object.getData());items = applyItemsPointer(itemReader, items);if (items.isObject()) {@@ -3112,6 +3116,20 @@ public class AslExecutor {}}+ /**+ * One item per non-empty line. ReaderConfig.ItemsPointer is JSON only on AWS, so a JSONL+ * dataset is always the whole file.+ */+ private ArrayNode readJsonLines(byte[] data) throws IOException {+ ArrayNode items = objectMapper.createArrayNode();+ for (String line : new String(data, StandardCharsets.UTF_8).split("\\R")) {+ if (!line.isBlank()) {+ items.add(objectMapper.readTree(line));+ }+ }+ return items;+ }+private JsonNode resolveItemReaderParameters(JsonNode itemReader, JsonNode input, JsonNode context,boolean jsonata, ObjectNode variables) throws Exception {if (jsonata && itemReader.has("Arguments")) {diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.javaindex cf6d685ddf..f8d652c823 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java@@ -950,6 +950,154 @@ class StepFunctionsJsonataIntegrationTest {assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));}+ @Test+ void distributedMapWithS3JsonlItemReader_readsOneItemPerLine() throws Exception {+ createBucket("map-inputs-jsonl");+ putObject("map-inputs-jsonl", "workers.jsonl",+ "{\"workerId\":\"w1\"}\n{\"workerId\":\"w2\"}\n\n");++ String definition = """+ {+ "StartAt": "ProcessWorkers",+ "States": {+ "ProcessWorkers": {+ "Type": "Map",+ "ItemReader": {+ "Resource": "arn:aws:states:::s3:getObject",+ "ReaderConfig": {+ "InputType": "JSONL"+ },+ "Parameters": {+ "Bucket": "map-inputs-jsonl",+ "Key": "workers.jsonl"+ }+ },+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "PassItem",+ "States": {+ "PassItem": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-itemreader-s3-jsonl-test", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);++ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));+ }++ @Test+ void distributedMapWithS3JsonlItemReader_maxItemsLimitsDataset() throws Exception {+ createBucket("map-inputs-jsonl-max-items");+ putObject("map-inputs-jsonl-max-items", "workers.jsonl",+ "{\"workerId\":\"w1\"}\n{\"workerId\":\"w2\"}\n{\"workerId\":\"w3\"}\n");++ String definition = """+ {+ "StartAt": "ProcessWorkers",+ "States": {+ "ProcessWorkers": {+ "Type": "Map",+ "ItemReader": {+ "Resource": "arn:aws:states:::s3:getObject",+ "ReaderConfig": {+ "InputType": "JSONL",+ "MaxItems": 2+ },+ "Parameters": {+ "Bucket": "map-inputs-jsonl-max-items",+ "Key": "workers.jsonl"+ }+ },+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "PassItem",+ "States": {+ "PassItem": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-itemreader-s3-jsonl-max-items-test", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);++ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));+ assertFalse(output.contains("\"workerId\":\"w3\"") || output.contains("\"workerId\": \"w3\""));+ }++ @Test+ void distributedMapWithS3JsonlItemReader_malformedLineFailsWithItemReaderError() throws Exception {+ createBucket("map-inputs-jsonl-invalid");+ putObject("map-inputs-jsonl-invalid", "workers.jsonl", "{\"workerId\":\"w1\"}\nnot-json\n");++ String definition = """+ {+ "StartAt": "ProcessWorkers",+ "States": {+ "ProcessWorkers": {+ "Type": "Map",+ "ItemReader": {+ "Resource": "arn:aws:states:::s3:getObject",+ "ReaderConfig": {+ "InputType": "JSONL"+ },+ "Parameters": {+ "Bucket": "map-inputs-jsonl-invalid",+ "Key": "workers.jsonl"+ }+ },+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "PassItem",+ "States": {+ "PassItem": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-itemreader-s3-jsonl-invalid-test", definition);+ String execArn = startExecution(smArn, "{}");+ Response failure = waitForExecutionFailure(execArn);++ assertEquals("FAILED", failure.jsonPath().getString("status"));+ assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));+ }+@Testvoid distributedMapWithItemBatcher_groupsItemsIntoBatchesOfMaxItemsPerBatch() throws Exception {String definition = """