Light, fluffy, and always free - The AWS Local Emulator alternative
feat(sfn): support ItemReader InputType CSV (#3603)
Ovidiu (Ovi)d3486867444c
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.mdindex 37064893c1..789f308fc7 100644--- a/docs/services/step-functions.md+++ b/docs/services/step-functions.md@@ -136,8 +136,15 @@ iterations.`ReaderConfig.ItemsPointer` selects a node inside it.- `JSONL` is one item per line. Blank lines are skipped, and `ItemsPointer` does not apply,matching AWS.-- `CSV`, `PARQUET` and `MANIFEST` are accepted by `CreateStateMachine` and fail the execution- with `States.ItemReaderFailed`.+- `CSV` takes its field names from the first row, or from `ReaderConfig.CSVHeaders` when+ `CSVHeaderLocation` is `GIVEN`. Every value is a string: a row shorter than the headers pads+ with empty strings, and a longer one drops the surplus. `ReaderConfig.CSVDelimiter` selects+ `COMMA`, `PIPE`, `SEMICOLON`, `SPACE` or `TAB`, and a quoted field may contain the delimiter+ or a line break without ending the record. A doubled quote stands for a single quote, in an+ unquoted field as well as a quoted one, and a backslash escapes another backslash, a quote or+ the delimiter. A backslash before anything else is dropped, as AWS documents.+- `PARQUET` and `MANIFEST` are accepted by `CreateStateMachine` and fail the execution with+ `States.ItemReaderFailed`.`arn:aws:states:::s3:listObjectsV2` reads every page under `Prefix`. Each item carries the AWSfields `Etag`, `Key`, `LastModified` (epoch seconds), `Size` and `StorageClass`. An empty prefixdiff --git a/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.java b/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.javaindex ba993e26eb..e82edfbfc9 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.java@@ -7,45 +7,116 @@ public class CsvParser {private CsvParser() {}+ /**+ * Parses one comma separated line. A field that opens with a quote may hold commas, and a+ * doubled quote inside it stands for a single quote. Backslashes stay literal, which is what+ * S3 Select and Athena expect of their input.+ */public static List<String> parseLine(String line) {+ return scan(line, ',', false).get(0).fields();+ }++ public static List<List<String>> parseAll(String content) {+ return parseAll(content, ',');+ }++ /**+ * Parses a whole text delimited document the way Step Functions reads an ItemReader dataset.+ * On top of the quoting above, a quoted field may hold line breaks, a doubled quote in an+ * unquoted field stands for a single quote, and a backslash escapes another backslash, a+ * quote or the delimiter. A backslash before anything else is dropped, as AWS documents. A+ * line carrying no fields at all is skipped, as a trailing newline is normal.+ */+ public static List<List<String>> parseAll(String content, char delimiter) {+ List<List<String>> rows = new ArrayList<>();+ for (Record record : scan(content, delimiter, true)) {+ if (!record.blank()) {+ rows.add(record.fields());+ }+ }+ return rows;+ }++ private record Record(List<String> fields, boolean quoted) {+ boolean blank() {+ return !quoted && fields.size() == 1 && fields.get(0).isBlank();+ }+ }++ /**+ * The single quote tracking loop behind both entry points, so the escaping rules live in one+ * place. Only a quote opening a field starts a quoted run: a quote further in is literal, which+ * is what lets an unquoted field carry a doubled quote. Always returns at least one record.+ */+ private static List<Record> scan(String content, char delimiter, boolean backslashEscapes) {+ List<Record> records = new ArrayList<>();List<String> fields = new ArrayList<>();- StringBuilder current = new StringBuilder();+ StringBuilder field = new StringBuilder();boolean inQuotes = false;- for (int i = 0; i < line.length(); i++) {- char c = line.charAt(i);- if (inQuotes) {- if (c == '"') {- if (i + 1 < line.length() && line.charAt(i + 1) == '"') {- current.append('"');+ boolean quoted = false;+ boolean fieldStart = true;++ for (int i = 0; i < content.length(); i++) {+ char c = content.charAt(i);+ char next = i + 1 < content.length() ? content.charAt(i + 1) : '\0';++ if (backslashEscapes && c == '\\' && i + 1 < content.length()) {+ if (next == '\\' || next == '"' || next == delimiter) {+ field.append(next);+ i++;+ }+ fieldStart = false;+ continue;+ }++ if (c == '"') {+ if (inQuotes) {+ if (next == '"') {+ field.append('"');i++;} else {inQuotes = false;}- } else {- current.append(c);- }- } else {- if (c == '"') {+ } else if (fieldStart) {inQuotes = true;- } else if (c == ',') {- fields.add(current.toString());- current.setLength(0);+ quoted = true;} else {- current.append(c);+ field.append('"');+ if (next == '"') {+ i++;+ }}+ fieldStart = false;+ continue;}- }- fields.add(current.toString());- return fields;- }- public static List<List<String>> parseAll(String content) {- List<List<String>> rows = new ArrayList<>();- for (String line : content.split("\\r?\\n")) {- if (!line.trim().isEmpty()) {- rows.add(parseLine(line));+ if (inQuotes) {+ field.append(c);+ continue;+ }++ if (c == delimiter) {+ fields.add(field.toString());+ field.setLength(0);+ fieldStart = true;+ } else if (c == '\n' || c == '\r') {+ if (c == '\r' && next == '\n') {+ i++;+ }+ fields.add(field.toString());+ field.setLength(0);+ records.add(new Record(fields, quoted));+ fields = new ArrayList<>();+ quoted = false;+ fieldStart = true;+ } else {+ field.append(c);+ fieldStart = false;}}- return rows;++ fields.add(field.toString());+ records.add(new Record(fields, quoted));+ return records;}}diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.javaindex e4fed0fc7b..dc18c6daaa 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.AwsErrorResponse;+import io.github.hectorvent.floci.core.common.CsvParser;import io.github.hectorvent.floci.core.common.CustomResourceLiveness;import io.github.hectorvent.floci.core.common.RequestContext;import io.github.hectorvent.floci.core.common.XmlParser;@@ -3077,7 +3078,7 @@ public class AslExecutor {}String inputType = itemReader.path("ReaderConfig").path("InputType").asText(null);- if (!"JSON".equals(inputType) && !"JSONL".equals(inputType)) {+ if (!"JSON".equals(inputType) && !"JSONL".equals(inputType) && !"CSV".equals(inputType)) {throw new FailStateException("States.ItemReaderFailed","ItemReader InputType " + inputType + " is not yet implemented by the emulator");}@@ -3095,6 +3096,10 @@ public class AslExecutor {return new ResolvedMapItems(applyMaxItems(itemReader, readJsonLines(object.getData())),MapItemsSource.ITEM_READER_ARRAY);}+ if ("CSV".equals(inputType)) {+ return new ResolvedMapItems(applyMaxItems(itemReader, readCsvRows(itemReader, object.getData())),+ MapItemsSource.ITEM_READER_ARRAY);+ }JsonNode items = objectMapper.readTree(object.getData());items = applyItemsPointer(itemReader, items);if (items.isObject()) {@@ -3130,6 +3135,56 @@ public class AslExecutor {return items;}+ /**+ * Each data row becomes an object keyed by the headers. A row shorter than the headers pads+ * with empty strings and a longer one drops the surplus, as on AWS. Every value is a string.+ */+ private ArrayNode readCsvRows(JsonNode itemReader, byte[] data) {+ JsonNode readerConfig = itemReader.path("ReaderConfig");+ String headerLocation = readerConfig.path("CSVHeaderLocation").asText("FIRST_ROW");+ List<List<String>> rows = CsvParser.parseAll(new String(data, StandardCharsets.UTF_8),+ csvDelimiter(readerConfig.path("CSVDelimiter").asText("COMMA")));++ List<String> headers;+ int firstDataRow;+ if ("GIVEN".equals(headerLocation)) {+ headers = new ArrayList<>();+ for (JsonNode header : readerConfig.path("CSVHeaders")) {+ headers.add(header.asText());+ }+ firstDataRow = 0;+ } else if ("FIRST_ROW".equals(headerLocation)) {+ headers = rows.isEmpty() ? List.of() : rows.get(0);+ firstDataRow = 1;+ } else {+ throw new FailStateException("States.ItemReaderFailed",+ "ItemReader CSVHeaderLocation " + headerLocation + " is not supported");+ }++ ArrayNode items = objectMapper.createArrayNode();+ for (int row = firstDataRow; row < rows.size(); row++) {+ List<String> values = rows.get(row);+ ObjectNode item = objectMapper.createObjectNode();+ for (int column = 0; column < headers.size(); column++) {+ item.put(headers.get(column), column < values.size() ? values.get(column) : "");+ }+ items.add(item);+ }+ return items;+ }++ private char csvDelimiter(String delimiter) {+ return switch (delimiter) {+ case "COMMA" -> ',';+ case "PIPE" -> '|';+ case "SEMICOLON" -> ';';+ case "SPACE" -> ' ';+ case "TAB" -> '\t';+ default -> throw new FailStateException("States.ItemReaderFailed",+ "ItemReader CSVDelimiter " + delimiter + " is not supported");+ };+ }+private JsonNode resolveItemReaderParameters(JsonNode itemReader, JsonNode input, JsonNode context,boolean jsonata, ObjectNode variables) throws Exception {if (jsonata && itemReader.has("Arguments")) {diff --git a/src/test/java/io/github/hectorvent/floci/core/common/CsvParserTest.java b/src/test/java/io/github/hectorvent/floci/core/common/CsvParserTest.javanew file mode 100644index 0000000000..2dc6aba953--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/core/common/CsvParserTest.java@@ -0,0 +1,99 @@+package io.github.hectorvent.floci.core.common;++import org.junit.jupiter.api.Test;++import java.util.List;++import static org.junit.jupiter.api.Assertions.assertEquals;++class CsvParserTest {++ /**+ * The file from the Step Functions "how Step Functions parses input CSV files" reference,+ * whose third field carries a doubled quote and a line break and whose fourth field is an+ * unquoted object relying on doubled quotes to survive States.StringToJson.+ */+ @Test+ void parsesTheAwsReferenceRecord() {+ String csv = "abc,123,\"This string contains commas, a double quotation marks (\"\"), "+ + "and a newline (\n)\",{\"\"MyKey\"\":\"\"MyValue\"\"},\"[1,2,3]\"";++ List<List<String>> rows = CsvParser.parseAll(csv, ',');++ assertEquals(1, rows.size(), "the quoted newline must not split the record");+ assertEquals(List.of(+ "abc",+ "123",+ "This string contains commas, a double quotation marks (\"), and a newline (\n)",+ "{\"MyKey\":\"MyValue\"}",+ "[1,2,3]"), rows.get(0));+ }++ @Test+ void doubledQuotesSurviveInAnUnquotedField() {+ assertEquals(List.of("{\"MyKey\":\"MyValue\"}"),+ CsvParser.parseAll("{\"\"MyKey\"\":\"\"MyValue\"\"}", ',').get(0));+ }++ @Test+ void doubledQuotesSurviveInAQuotedField() {+ assertEquals(List.of("a\"b"), CsvParser.parseAll("\"a\"\"b\"", ',').get(0));+ }++ /** A field that opens with a quote still follows CSV rules, so "" stays an empty field. */+ @Test+ void aQuotedEmptyFieldStaysEmpty() {+ assertEquals(List.of("", "x"), CsvParser.parseAll("\"\",x", ',').get(0));+ }++ @Test+ void backslashEscapesABackslash() {+ String csv = "path,size\nC:\\\\Program Files\\\\MyApp.exe,6534512\n";++ List<List<String>> rows = CsvParser.parseAll(csv, ',');++ assertEquals(List.of("C:\\Program Files\\MyApp.exe", "6534512"), rows.get(1));+ }++ @Test+ void backslashEscapesAQuoteAndTheDelimiter() {+ assertEquals(List.of("a\"b"), CsvParser.parseAll("a\\\"b", ',').get(0));+ assertEquals(List.of("a,b"), CsvParser.parseAll("a\\,b", ',').get(0));+ assertEquals(List.of("a|b"), CsvParser.parseAll("a\\|b", '|').get(0));+ }++ /** AWS drops a backslash that precedes anything else and keeps the character. */+ @Test+ void backslashBeforeAnyOtherCharacterIsDropped() {+ assertEquals(List.of("azb"), CsvParser.parseAll("a\\zb", ',').get(0));+ }++ @Test+ void honoursANonCommaDelimiter() {+ assertEquals(List.of("a", "b,c"), CsvParser.parseAll("a|\"b,c\"", '|').get(0));+ }++ @Test+ void skipsABlankTrailingLine() {+ assertEquals(1, CsvParser.parseAll("a,b\n", ',').size());+ }++ /**+ * S3 Select and Athena read line by line and expect a backslash to stay put, so the backslash+ * rules above must not reach parseLine.+ */+ @Test+ void parseLineKeepsBackslashesLiteral() {+ assertEquals(List.of("C:\\Users\\x"), CsvParser.parseLine("C:\\Users\\x"));+ }++ @Test+ void parseLineKeepsQuotedCommasTogether() {+ assertEquals(List.of("a", "b,c"), CsvParser.parseLine("a,\"b,c\""));+ }++ @Test+ void parseLineReturnsOneEmptyFieldForAnEmptyLine() {+ assertEquals(List.of(""), CsvParser.parseLine(""));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.javaindex f8d652c823..2ca5c75689 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java@@ -1098,6 +1098,382 @@ class StepFunctionsJsonataIntegrationTest {assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));}+ @Test+ void distributedMapWithS3CsvItemReader_firstRowSuppliesHeaders() throws Exception {+ createBucket("map-inputs-csv");+ putObject("map-inputs-csv", "workers.csv", "workerId,team\nw1,\"alpha,beta\"\nw2,gamma\n");++ String definition = """+ {+ "StartAt": "ProcessWorkers",+ "States": {+ "ProcessWorkers": {+ "Type": "Map",+ "ItemReader": {+ "Resource": "arn:aws:states:::s3:getObject",+ "ReaderConfig": {+ "InputType": "CSV"+ },+ "Parameters": {+ "Bucket": "map-inputs-csv",+ "Key": "workers.csv"+ }+ },+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "PassItem",+ "States": {+ "PassItem": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-itemreader-s3-csv-test", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);++ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));+ assertTrue(output.contains("\"team\":\"alpha,beta\"") || output.contains("\"team\": \"alpha,beta\""));+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));+ }++ @Test+ void distributedMapWithS3CsvItemReader_givenHeadersReadEveryRow() throws Exception {+ createBucket("map-inputs-csv-given");+ putObject("map-inputs-csv-given", "workers.csv", "w1,alpha\nw2,gamma\n");++ String definition = """+ {+ "StartAt": "ProcessWorkers",+ "States": {+ "ProcessWorkers": {+ "Type": "Map",+ "ItemReader": {+ "Resource": "arn:aws:states:::s3:getObject",+ "ReaderConfig": {+ "InputType": "CSV",+ "CSVHeaderLocation": "GIVEN",+ "CSVHeaders": ["workerId", "team"]+ },+ "Parameters": {+ "Bucket": "map-inputs-csv-given",+ "Key": "workers.csv"+ }+ },+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "PassItem",+ "States": {+ "PassItem": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-itemreader-s3-csv-given-test", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);++ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));+ }++ @Test+ void distributedMapWithS3CsvItemReader_shortRowPadsWithEmptyValues() throws Exception {+ createBucket("map-inputs-csv-ragged");+ putObject("map-inputs-csv-ragged", "workers.csv", "workerId,team\nw1\n");++ String definition = """+ {+ "StartAt": "ProcessWorkers",+ "States": {+ "ProcessWorkers": {+ "Type": "Map",+ "ItemReader": {+ "Resource": "arn:aws:states:::s3:getObject",+ "ReaderConfig": {+ "InputType": "CSV"+ },+ "Parameters": {+ "Bucket": "map-inputs-csv-ragged",+ "Key": "workers.csv"+ }+ },+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "PassItem",+ "States": {+ "PassItem": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-itemreader-s3-csv-ragged-test", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);++ assertTrue(output.contains("\"team\":\"\"") || output.contains("\"team\": \"\""));+ }++ @Test+ void distributedMapWithS3CsvItemReader_honoursCsvDelimiter() throws Exception {+ createBucket("map-inputs-csv-pipe");+ putObject("map-inputs-csv-pipe", "workers.csv", "workerId|team\nw1|alpha\nw2|gamma\n");++ String definition = """+ {+ "StartAt": "ProcessWorkers",+ "States": {+ "ProcessWorkers": {+ "Type": "Map",+ "ItemReader": {+ "Resource": "arn:aws:states:::s3:getObject",+ "ReaderConfig": {+ "InputType": "CSV",+ "CSVDelimiter": "PIPE"+ },+ "Parameters": {+ "Bucket": "map-inputs-csv-pipe",+ "Key": "workers.csv"+ }+ },+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "PassItem",+ "States": {+ "PassItem": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-itemreader-s3-csv-pipe-test", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);++ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));+ assertTrue(output.contains("\"team\":\"alpha\"") || output.contains("\"team\": \"alpha\""));+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));+ }++ @Test+ void distributedMapWithS3CsvItemReader_keepsNewlinesInsideAQuotedField() throws Exception {+ createBucket("map-inputs-csv-multiline");+ putObject("map-inputs-csv-multiline", "workers.csv",+ "workerId,note\nw1,\"first line\nsecond line\"\nw2,plain\n");++ String definition = """+ {+ "StartAt": "ProcessWorkers",+ "States": {+ "ProcessWorkers": {+ "Type": "Map",+ "ItemReader": {+ "Resource": "arn:aws:states:::s3:getObject",+ "ReaderConfig": {+ "InputType": "CSV"+ },+ "Parameters": {+ "Bucket": "map-inputs-csv-multiline",+ "Key": "workers.csv"+ }+ },+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "PassItem",+ "States": {+ "PassItem": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-itemreader-s3-csv-multiline-test", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);++ JsonNode items = new ObjectMapper().readTree(output);+ assertEquals(2, items.size(), "the quoted line break must not split the record: " + output);+ assertEquals("first line\nsecond line", items.get(0).path("note").asText());+ assertEquals("plain", items.get(1).path("note").asText());+ }++ /**+ * The file and the state machine from the AWS "how Step Functions parses input CSV files"+ * reference. The unquoted MyObject field only survives States.StringToJson if the doubled+ * quotes around its keys are preserved, so this pins the documented output.+ */+ @Test+ void distributedMapWithS3CsvItemReader_parsesTheAwsReferenceFile() throws Exception {+ createBucket("map-inputs-csv-reference");+ putObject("map-inputs-csv-reference", "myCSVInput.csv",+ "abc,123,\"This string contains commas, a double quotation marks (\"\"), "+ + "and a newline (\n)\",{\"\"MyKey\"\":\"\"MyValue\"\"},\"[1,2,3]\"");++ String definition = """+ {+ "StartAt": "Map",+ "States": {+ "Map": {+ "Type": "Map",+ "ItemReader": {+ "Resource": "arn:aws:states:::s3:getObject",+ "ReaderConfig": {+ "InputType": "CSV",+ "CSVHeaderLocation": "GIVEN",+ "CSVHeaders": ["MyLetters", "MyNumbers", "MyString",+ "MyObject", "MyArray"]+ },+ "Parameters": {+ "Bucket": "map-inputs-csv-reference",+ "Key": "myCSVInput.csv"+ }+ },+ "ItemSelector": {+ "MyLetters.$": "$$.Map.Item.Value.MyLetters",+ "MyNumbers.$": "States.StringToJson($$.Map.Item.Value.MyNumbers)",+ "MyString.$": "$$.Map.Item.Value.MyString",+ "MyObject.$": "States.StringToJson($$.Map.Item.Value.MyObject)",+ "MyArray.$": "States.StringToJson($$.Map.Item.Value.MyArray)"+ },+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "PassItem",+ "States": {+ "PassItem": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-itemreader-s3-csv-reference-test", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);++ JsonNode items = new ObjectMapper().readTree(output);+ assertEquals(1, items.size(), "the quoted newline must not split the record: " + output);+ JsonNode item = items.get(0);+ assertEquals("abc", item.path("MyLetters").asText());+ assertEquals(123, item.path("MyNumbers").asInt());+ assertEquals("This string contains commas, a double quotation marks (\"), "+ + "and a newline (\n)", item.path("MyString").asText());+ assertEquals("MyValue", item.path("MyObject").path("MyKey").asText(),+ "doubled quotes in an unquoted field must survive: " + output);+ assertEquals(3, item.path("MyArray").size());+ assertEquals(1, item.path("MyArray").get(0).asInt());+ }++ @Test+ void distributedMapWithS3CsvItemReader_honoursBackslashEscapes() throws Exception {+ createBucket("map-inputs-csv-backslash");+ putObject("map-inputs-csv-backslash", "paths.csv",+ "path,note\nC:\\\\Program Files\\\\MyApp.exe,a\\\"quoted\\\" word\n"+ + "second\\,field,dropped\\zslash\n");++ String definition = """+ {+ "StartAt": "ProcessPaths",+ "States": {+ "ProcessPaths": {+ "Type": "Map",+ "ItemReader": {+ "Resource": "arn:aws:states:::s3:getObject",+ "ReaderConfig": {+ "InputType": "CSV"+ },+ "Parameters": {+ "Bucket": "map-inputs-csv-backslash",+ "Key": "paths.csv"+ }+ },+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "PassItem",+ "States": {+ "PassItem": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-itemreader-s3-csv-backslash-test", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);++ JsonNode items = new ObjectMapper().readTree(output);+ assertEquals(2, items.size(), output);+ assertEquals("C:\\Program Files\\MyApp.exe", items.get(0).path("path").asText());+ assertEquals("a\"quoted\" word", items.get(0).path("note").asText());+ assertEquals("second,field", items.get(1).path("path").asText(),+ "an escaped delimiter must not split the field: " + output);+ assertEquals("dropped" + "zslash", items.get(1).path("note").asText());+ }+@Testvoid distributedMapWithItemBatcher_groupsItemsIntoBatchesOfMaxItemsPerBatch() throws Exception {String definition = """@@ -3274,55 +3650,6 @@ class StepFunctionsJsonataIntegrationTest {""", itemReaderFields);}- @Test- void distributedMapWithCsvItemReader_failsWithNotImplementedItemReaderError() throws Exception {- createBucket("map-inputs-csv");- putObject("map-inputs-csv", "workers.csv", "workerId\nw1\n");-- String definition = """- {- "StartAt": "ProcessWorkers",- "States": {- "ProcessWorkers": {- "Type": "Map",- "ItemReader": {- "Resource": "arn:aws:states:::s3:getObject",- "ReaderConfig": {- "InputType": "CSV"- },- "Parameters": {- "Bucket": "map-inputs-csv",- "Key": "workers.csv"- }- },- "ItemProcessor": {- "ProcessorConfig": {- "Mode": "DISTRIBUTED",- "ExecutionType": "STANDARD"- },- "StartAt": "PassItem",- "States": {- "PassItem": {- "Type": "Pass",- "End": true- }- }- },- "End": true- }- }- }- """;-- String smArn = createStateMachine("map-itemreader-s3-csv-test", definition);- String execArn = startExecution(smArn, "{}");- Response failure = waitForExecutionFailure(execArn);-- assertEquals("FAILED", failure.jsonPath().getString("status"));- assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));- assertTrue(failure.jsonPath().getString("cause").contains("InputType CSV is not yet implemented by the emulator"));- }-@Testvoid distributedMapWithUnsupportedItemReaderInputType_rejectedAtCreateStateMachine() {String definition = """