Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(sfn): support ItemReader InputType CSV (#3603)

Ovidiu (Ovi)d3486867444c
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.md
index 37064893c1..789f308fc7 100644
--- a/docs/services/step-functions.md
+++ b/docs/services/step-functions.md
@@ -136,8 +136,15 @@ iterations.
`ReaderConfig.ItemsPointer` selects a node inside it.
- `JSONL` is one item per line. Blank lines are skipped, and `ItemsPointer` does not apply,
matching AWS.
-- `CSV`, `PARQUET` and `MANIFEST` are accepted by `CreateStateMachine` and fail the execution
- with `States.ItemReaderFailed`.
+- `CSV` takes its field names from the first row, or from `ReaderConfig.CSVHeaders` when
+ `CSVHeaderLocation` is `GIVEN`. Every value is a string: a row shorter than the headers pads
+ with empty strings, and a longer one drops the surplus. `ReaderConfig.CSVDelimiter` selects
+ `COMMA`, `PIPE`, `SEMICOLON`, `SPACE` or `TAB`, and a quoted field may contain the delimiter
+ or a line break without ending the record. A doubled quote stands for a single quote, in an
+ unquoted field as well as a quoted one, and a backslash escapes another backslash, a quote or
+ the delimiter. A backslash before anything else is dropped, as AWS documents.
+- `PARQUET` and `MANIFEST` are accepted by `CreateStateMachine` and fail the execution with
+ `States.ItemReaderFailed`.
`arn:aws:states:::s3:listObjectsV2` reads every page under `Prefix`. Each item carries the AWS
fields `Etag`, `Key`, `LastModified` (epoch seconds), `Size` and `StorageClass`. An empty prefix
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.java b/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.java
index ba993e26eb..e82edfbfc9 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.java
@@ -7,45 +7,116 @@ public class CsvParser {
private CsvParser() {}
+ /**
+ * Parses one comma separated line. A field that opens with a quote may hold commas, and a
+ * doubled quote inside it stands for a single quote. Backslashes stay literal, which is what
+ * S3 Select and Athena expect of their input.
+ */
public static List<String> parseLine(String line) {
+ return scan(line, ',', false).get(0).fields();
+ }
+
+ public static List<List<String>> parseAll(String content) {
+ return parseAll(content, ',');
+ }
+
+ /**
+ * Parses a whole text delimited document the way Step Functions reads an ItemReader dataset.
+ * On top of the quoting above, a quoted field may hold line breaks, a doubled quote in an
+ * unquoted field stands for a single quote, and a backslash escapes another backslash, a
+ * quote or the delimiter. A backslash before anything else is dropped, as AWS documents. A
+ * line carrying no fields at all is skipped, as a trailing newline is normal.
+ */
+ public static List<List<String>> parseAll(String content, char delimiter) {
+ List<List<String>> rows = new ArrayList<>();
+ for (Record record : scan(content, delimiter, true)) {
+ if (!record.blank()) {
+ rows.add(record.fields());
+ }
+ }
+ return rows;
+ }
+
+ private record Record(List<String> fields, boolean quoted) {
+ boolean blank() {
+ return !quoted && fields.size() == 1 && fields.get(0).isBlank();
+ }
+ }
+
+ /**
+ * The single quote tracking loop behind both entry points, so the escaping rules live in one
+ * place. Only a quote opening a field starts a quoted run: a quote further in is literal, which
+ * is what lets an unquoted field carry a doubled quote. Always returns at least one record.
+ */
+ private static List<Record> scan(String content, char delimiter, boolean backslashEscapes) {
+ List<Record> records = new ArrayList<>();
List<String> fields = new ArrayList<>();
- StringBuilder current = new StringBuilder();
+ StringBuilder field = new StringBuilder();
boolean inQuotes = false;
- for (int i = 0; i < line.length(); i++) {
- char c = line.charAt(i);
- if (inQuotes) {
- if (c == '"') {
- if (i + 1 < line.length() && line.charAt(i + 1) == '"') {
- current.append('"');
+ boolean quoted = false;
+ boolean fieldStart = true;
+
+ for (int i = 0; i < content.length(); i++) {
+ char c = content.charAt(i);
+ char next = i + 1 < content.length() ? content.charAt(i + 1) : '\0';
+
+ if (backslashEscapes && c == '\\' && i + 1 < content.length()) {
+ if (next == '\\' || next == '"' || next == delimiter) {
+ field.append(next);
+ i++;
+ }
+ fieldStart = false;
+ continue;
+ }
+
+ if (c == '"') {
+ if (inQuotes) {
+ if (next == '"') {
+ field.append('"');
i++;
} else {
inQuotes = false;
}
- } else {
- current.append(c);
- }
- } else {
- if (c == '"') {
+ } else if (fieldStart) {
inQuotes = true;
- } else if (c == ',') {
- fields.add(current.toString());
- current.setLength(0);
+ quoted = true;
} else {
- current.append(c);
+ field.append('"');
+ if (next == '"') {
+ i++;
+ }
}
+ fieldStart = false;
+ continue;
}
- }
- fields.add(current.toString());
- return fields;
- }
- public static List<List<String>> parseAll(String content) {
- List<List<String>> rows = new ArrayList<>();
- for (String line : content.split("\\r?\\n")) {
- if (!line.trim().isEmpty()) {
- rows.add(parseLine(line));
+ if (inQuotes) {
+ field.append(c);
+ continue;
+ }
+
+ if (c == delimiter) {
+ fields.add(field.toString());
+ field.setLength(0);
+ fieldStart = true;
+ } else if (c == '\n' || c == '\r') {
+ if (c == '\r' && next == '\n') {
+ i++;
+ }
+ fields.add(field.toString());
+ field.setLength(0);
+ records.add(new Record(fields, quoted));
+ fields = new ArrayList<>();
+ quoted = false;
+ fieldStart = true;
+ } else {
+ field.append(c);
+ fieldStart = false;
}
}
- return rows;
+
+ fields.add(field.toString());
+ records.add(new Record(fields, quoted));
+ return records;
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index e4fed0fc7b..dc18c6daaa 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.AwsErrorResponse;
+import io.github.hectorvent.floci.core.common.CsvParser;
import io.github.hectorvent.floci.core.common.CustomResourceLiveness;
import io.github.hectorvent.floci.core.common.RequestContext;
import io.github.hectorvent.floci.core.common.XmlParser;
@@ -3077,7 +3078,7 @@ public class AslExecutor {
}
String inputType = itemReader.path("ReaderConfig").path("InputType").asText(null);
- if (!"JSON".equals(inputType) && !"JSONL".equals(inputType)) {
+ if (!"JSON".equals(inputType) && !"JSONL".equals(inputType) && !"CSV".equals(inputType)) {
throw new FailStateException("States.ItemReaderFailed",
"ItemReader InputType " + inputType + " is not yet implemented by the emulator");
}
@@ -3095,6 +3096,10 @@ public class AslExecutor {
return new ResolvedMapItems(applyMaxItems(itemReader, readJsonLines(object.getData())),
MapItemsSource.ITEM_READER_ARRAY);
}
+ if ("CSV".equals(inputType)) {
+ return new ResolvedMapItems(applyMaxItems(itemReader, readCsvRows(itemReader, object.getData())),
+ MapItemsSource.ITEM_READER_ARRAY);
+ }
JsonNode items = objectMapper.readTree(object.getData());
items = applyItemsPointer(itemReader, items);
if (items.isObject()) {
@@ -3130,6 +3135,56 @@ public class AslExecutor {
return items;
}
+ /**
+ * Each data row becomes an object keyed by the headers. A row shorter than the headers pads
+ * with empty strings and a longer one drops the surplus, as on AWS. Every value is a string.
+ */
+ private ArrayNode readCsvRows(JsonNode itemReader, byte[] data) {
+ JsonNode readerConfig = itemReader.path("ReaderConfig");
+ String headerLocation = readerConfig.path("CSVHeaderLocation").asText("FIRST_ROW");
+ List<List<String>> rows = CsvParser.parseAll(new String(data, StandardCharsets.UTF_8),
+ csvDelimiter(readerConfig.path("CSVDelimiter").asText("COMMA")));
+
+ List<String> headers;
+ int firstDataRow;
+ if ("GIVEN".equals(headerLocation)) {
+ headers = new ArrayList<>();
+ for (JsonNode header : readerConfig.path("CSVHeaders")) {
+ headers.add(header.asText());
+ }
+ firstDataRow = 0;
+ } else if ("FIRST_ROW".equals(headerLocation)) {
+ headers = rows.isEmpty() ? List.of() : rows.get(0);
+ firstDataRow = 1;
+ } else {
+ throw new FailStateException("States.ItemReaderFailed",
+ "ItemReader CSVHeaderLocation " + headerLocation + " is not supported");
+ }
+
+ ArrayNode items = objectMapper.createArrayNode();
+ for (int row = firstDataRow; row < rows.size(); row++) {
+ List<String> values = rows.get(row);
+ ObjectNode item = objectMapper.createObjectNode();
+ for (int column = 0; column < headers.size(); column++) {
+ item.put(headers.get(column), column < values.size() ? values.get(column) : "");
+ }
+ items.add(item);
+ }
+ return items;
+ }
+
+ private char csvDelimiter(String delimiter) {
+ return switch (delimiter) {
+ case "COMMA" -> ',';
+ case "PIPE" -> '|';
+ case "SEMICOLON" -> ';';
+ case "SPACE" -> ' ';
+ case "TAB" -> '\t';
+ default -> throw new FailStateException("States.ItemReaderFailed",
+ "ItemReader CSVDelimiter " + delimiter + " is not supported");
+ };
+ }
+
private JsonNode resolveItemReaderParameters(JsonNode itemReader, JsonNode input, JsonNode context,
boolean jsonata, ObjectNode variables) throws Exception {
if (jsonata && itemReader.has("Arguments")) {
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/CsvParserTest.java b/src/test/java/io/github/hectorvent/floci/core/common/CsvParserTest.java
new file mode 100644
index 0000000000..2dc6aba953
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/core/common/CsvParserTest.java
@@ -0,0 +1,99 @@
+package io.github.hectorvent.floci.core.common;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class CsvParserTest {
+
+ /**
+ * The file from the Step Functions "how Step Functions parses input CSV files" reference,
+ * whose third field carries a doubled quote and a line break and whose fourth field is an
+ * unquoted object relying on doubled quotes to survive States.StringToJson.
+ */
+ @Test
+ void parsesTheAwsReferenceRecord() {
+ String csv = "abc,123,\"This string contains commas, a double quotation marks (\"\"), "
+ + "and a newline (\n)\",{\"\"MyKey\"\":\"\"MyValue\"\"},\"[1,2,3]\"";
+
+ List<List<String>> rows = CsvParser.parseAll(csv, ',');
+
+ assertEquals(1, rows.size(), "the quoted newline must not split the record");
+ assertEquals(List.of(
+ "abc",
+ "123",
+ "This string contains commas, a double quotation marks (\"), and a newline (\n)",
+ "{\"MyKey\":\"MyValue\"}",
+ "[1,2,3]"), rows.get(0));
+ }
+
+ @Test
+ void doubledQuotesSurviveInAnUnquotedField() {
+ assertEquals(List.of("{\"MyKey\":\"MyValue\"}"),
+ CsvParser.parseAll("{\"\"MyKey\"\":\"\"MyValue\"\"}", ',').get(0));
+ }
+
+ @Test
+ void doubledQuotesSurviveInAQuotedField() {
+ assertEquals(List.of("a\"b"), CsvParser.parseAll("\"a\"\"b\"", ',').get(0));
+ }
+
+ /** A field that opens with a quote still follows CSV rules, so "" stays an empty field. */
+ @Test
+ void aQuotedEmptyFieldStaysEmpty() {
+ assertEquals(List.of("", "x"), CsvParser.parseAll("\"\",x", ',').get(0));
+ }
+
+ @Test
+ void backslashEscapesABackslash() {
+ String csv = "path,size\nC:\\\\Program Files\\\\MyApp.exe,6534512\n";
+
+ List<List<String>> rows = CsvParser.parseAll(csv, ',');
+
+ assertEquals(List.of("C:\\Program Files\\MyApp.exe", "6534512"), rows.get(1));
+ }
+
+ @Test
+ void backslashEscapesAQuoteAndTheDelimiter() {
+ assertEquals(List.of("a\"b"), CsvParser.parseAll("a\\\"b", ',').get(0));
+ assertEquals(List.of("a,b"), CsvParser.parseAll("a\\,b", ',').get(0));
+ assertEquals(List.of("a|b"), CsvParser.parseAll("a\\|b", '|').get(0));
+ }
+
+ /** AWS drops a backslash that precedes anything else and keeps the character. */
+ @Test
+ void backslashBeforeAnyOtherCharacterIsDropped() {
+ assertEquals(List.of("azb"), CsvParser.parseAll("a\\zb", ',').get(0));
+ }
+
+ @Test
+ void honoursANonCommaDelimiter() {
+ assertEquals(List.of("a", "b,c"), CsvParser.parseAll("a|\"b,c\"", '|').get(0));
+ }
+
+ @Test
+ void skipsABlankTrailingLine() {
+ assertEquals(1, CsvParser.parseAll("a,b\n", ',').size());
+ }
+
+ /**
+ * S3 Select and Athena read line by line and expect a backslash to stay put, so the backslash
+ * rules above must not reach parseLine.
+ */
+ @Test
+ void parseLineKeepsBackslashesLiteral() {
+ assertEquals(List.of("C:\\Users\\x"), CsvParser.parseLine("C:\\Users\\x"));
+ }
+
+ @Test
+ void parseLineKeepsQuotedCommasTogether() {
+ assertEquals(List.of("a", "b,c"), CsvParser.parseLine("a,\"b,c\""));
+ }
+
+ @Test
+ void parseLineReturnsOneEmptyFieldForAnEmptyLine() {
+ assertEquals(List.of(""), CsvParser.parseLine(""));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
index f8d652c823..2ca5c75689 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
@@ -1098,6 +1098,382 @@ class StepFunctionsJsonataIntegrationTest {
assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));
}
+ @Test
+ void distributedMapWithS3CsvItemReader_firstRowSuppliesHeaders() throws Exception {
+ createBucket("map-inputs-csv");
+ putObject("map-inputs-csv", "workers.csv", "workerId,team\nw1,\"alpha,beta\"\nw2,gamma\n");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "CSV"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-csv",
+ "Key": "workers.csv"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-csv-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));
+ assertTrue(output.contains("\"team\":\"alpha,beta\"") || output.contains("\"team\": \"alpha,beta\""));
+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));
+ }
+
+ @Test
+ void distributedMapWithS3CsvItemReader_givenHeadersReadEveryRow() throws Exception {
+ createBucket("map-inputs-csv-given");
+ putObject("map-inputs-csv-given", "workers.csv", "w1,alpha\nw2,gamma\n");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "CSV",
+ "CSVHeaderLocation": "GIVEN",
+ "CSVHeaders": ["workerId", "team"]
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-csv-given",
+ "Key": "workers.csv"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-csv-given-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));
+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));
+ }
+
+ @Test
+ void distributedMapWithS3CsvItemReader_shortRowPadsWithEmptyValues() throws Exception {
+ createBucket("map-inputs-csv-ragged");
+ putObject("map-inputs-csv-ragged", "workers.csv", "workerId,team\nw1\n");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "CSV"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-csv-ragged",
+ "Key": "workers.csv"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-csv-ragged-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"team\":\"\"") || output.contains("\"team\": \"\""));
+ }
+
+ @Test
+ void distributedMapWithS3CsvItemReader_honoursCsvDelimiter() throws Exception {
+ createBucket("map-inputs-csv-pipe");
+ putObject("map-inputs-csv-pipe", "workers.csv", "workerId|team\nw1|alpha\nw2|gamma\n");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "CSV",
+ "CSVDelimiter": "PIPE"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-csv-pipe",
+ "Key": "workers.csv"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-csv-pipe-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(output.contains("\"workerId\":\"w1\"") || output.contains("\"workerId\": \"w1\""));
+ assertTrue(output.contains("\"team\":\"alpha\"") || output.contains("\"team\": \"alpha\""));
+ assertTrue(output.contains("\"workerId\":\"w2\"") || output.contains("\"workerId\": \"w2\""));
+ }
+
+ @Test
+ void distributedMapWithS3CsvItemReader_keepsNewlinesInsideAQuotedField() throws Exception {
+ createBucket("map-inputs-csv-multiline");
+ putObject("map-inputs-csv-multiline", "workers.csv",
+ "workerId,note\nw1,\"first line\nsecond line\"\nw2,plain\n");
+
+ String definition = """
+ {
+ "StartAt": "ProcessWorkers",
+ "States": {
+ "ProcessWorkers": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "CSV"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-csv-multiline",
+ "Key": "workers.csv"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-csv-multiline-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ JsonNode items = new ObjectMapper().readTree(output);
+ assertEquals(2, items.size(), "the quoted line break must not split the record: " + output);
+ assertEquals("first line\nsecond line", items.get(0).path("note").asText());
+ assertEquals("plain", items.get(1).path("note").asText());
+ }
+
+ /**
+ * The file and the state machine from the AWS "how Step Functions parses input CSV files"
+ * reference. The unquoted MyObject field only survives States.StringToJson if the doubled
+ * quotes around its keys are preserved, so this pins the documented output.
+ */
+ @Test
+ void distributedMapWithS3CsvItemReader_parsesTheAwsReferenceFile() throws Exception {
+ createBucket("map-inputs-csv-reference");
+ putObject("map-inputs-csv-reference", "myCSVInput.csv",
+ "abc,123,\"This string contains commas, a double quotation marks (\"\"), "
+ + "and a newline (\n)\",{\"\"MyKey\"\":\"\"MyValue\"\"},\"[1,2,3]\"");
+
+ String definition = """
+ {
+ "StartAt": "Map",
+ "States": {
+ "Map": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "CSV",
+ "CSVHeaderLocation": "GIVEN",
+ "CSVHeaders": ["MyLetters", "MyNumbers", "MyString",
+ "MyObject", "MyArray"]
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-csv-reference",
+ "Key": "myCSVInput.csv"
+ }
+ },
+ "ItemSelector": {
+ "MyLetters.$": "$$.Map.Item.Value.MyLetters",
+ "MyNumbers.$": "States.StringToJson($$.Map.Item.Value.MyNumbers)",
+ "MyString.$": "$$.Map.Item.Value.MyString",
+ "MyObject.$": "States.StringToJson($$.Map.Item.Value.MyObject)",
+ "MyArray.$": "States.StringToJson($$.Map.Item.Value.MyArray)"
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-csv-reference-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ JsonNode items = new ObjectMapper().readTree(output);
+ assertEquals(1, items.size(), "the quoted newline must not split the record: " + output);
+ JsonNode item = items.get(0);
+ assertEquals("abc", item.path("MyLetters").asText());
+ assertEquals(123, item.path("MyNumbers").asInt());
+ assertEquals("This string contains commas, a double quotation marks (\"), "
+ + "and a newline (\n)", item.path("MyString").asText());
+ assertEquals("MyValue", item.path("MyObject").path("MyKey").asText(),
+ "doubled quotes in an unquoted field must survive: " + output);
+ assertEquals(3, item.path("MyArray").size());
+ assertEquals(1, item.path("MyArray").get(0).asInt());
+ }
+
+ @Test
+ void distributedMapWithS3CsvItemReader_honoursBackslashEscapes() throws Exception {
+ createBucket("map-inputs-csv-backslash");
+ putObject("map-inputs-csv-backslash", "paths.csv",
+ "path,note\nC:\\\\Program Files\\\\MyApp.exe,a\\\"quoted\\\" word\n"
+ + "second\\,field,dropped\\zslash\n");
+
+ String definition = """
+ {
+ "StartAt": "ProcessPaths",
+ "States": {
+ "ProcessPaths": {
+ "Type": "Map",
+ "ItemReader": {
+ "Resource": "arn:aws:states:::s3:getObject",
+ "ReaderConfig": {
+ "InputType": "CSV"
+ },
+ "Parameters": {
+ "Bucket": "map-inputs-csv-backslash",
+ "Key": "paths.csv"
+ }
+ },
+ "ItemProcessor": {
+ "ProcessorConfig": {
+ "Mode": "DISTRIBUTED",
+ "ExecutionType": "STANDARD"
+ },
+ "StartAt": "PassItem",
+ "States": {
+ "PassItem": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("map-itemreader-s3-csv-backslash-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ JsonNode items = new ObjectMapper().readTree(output);
+ assertEquals(2, items.size(), output);
+ assertEquals("C:\\Program Files\\MyApp.exe", items.get(0).path("path").asText());
+ assertEquals("a\"quoted\" word", items.get(0).path("note").asText());
+ assertEquals("second,field", items.get(1).path("path").asText(),
+ "an escaped delimiter must not split the field: " + output);
+ assertEquals("dropped" + "zslash", items.get(1).path("note").asText());
+ }
+
@Test
void distributedMapWithItemBatcher_groupsItemsIntoBatchesOfMaxItemsPerBatch() throws Exception {
String definition = """
@@ -3274,55 +3650,6 @@ class StepFunctionsJsonataIntegrationTest {
""", itemReaderFields);
}
- @Test
- void distributedMapWithCsvItemReader_failsWithNotImplementedItemReaderError() throws Exception {
- createBucket("map-inputs-csv");
- putObject("map-inputs-csv", "workers.csv", "workerId\nw1\n");
-
- String definition = """
- {
- "StartAt": "ProcessWorkers",
- "States": {
- "ProcessWorkers": {
- "Type": "Map",
- "ItemReader": {
- "Resource": "arn:aws:states:::s3:getObject",
- "ReaderConfig": {
- "InputType": "CSV"
- },
- "Parameters": {
- "Bucket": "map-inputs-csv",
- "Key": "workers.csv"
- }
- },
- "ItemProcessor": {
- "ProcessorConfig": {
- "Mode": "DISTRIBUTED",
- "ExecutionType": "STANDARD"
- },
- "StartAt": "PassItem",
- "States": {
- "PassItem": {
- "Type": "Pass",
- "End": true
- }
- }
- },
- "End": true
- }
- }
- }
- """;
-
- String smArn = createStateMachine("map-itemreader-s3-csv-test", definition);
- String execArn = startExecution(smArn, "{}");
- Response failure = waitForExecutionFailure(execArn);
-
- assertEquals("FAILED", failure.jsonPath().getString("status"));
- assertEquals("States.ItemReaderFailed", failure.jsonPath().getString("error"));
- assertTrue(failure.jsonPath().getString("cause").contains("InputType CSV is not yet implemented by the emulator"));
- }
-
@Test
void distributedMapWithUnsupportedItemReaderInputType_rejectedAtCreateStateMachine() {
String definition = """