Light, fluffy, and always free - The AWS Local Emulator alternative
feat(sfn): honour ToleratedFailureCount and ToleratedFailurePercentage (#3605)
Ovidiu (Ovi)1319ed19bc8f
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.mdindex 9519a82bba..5782ff95a1 100644--- a/docs/services/step-functions.md+++ b/docs/services/step-functions.md@@ -110,8 +110,9 @@ scheduling time, not when a worker actually picks up the task. `TaskSubmitted`,AWS emits for `.sync` and `.waitForTaskToken` integrations, is not emitted yet. When abranch fails, AWS records `*StateAborted` and `MapIterationAborted` events for the states itssibling branches were in; Floci cancels the siblings without recording them. A Distributed-`Map` whose item fails reports the item's own error rather than AWS's-`States.ExceedToleratedFailureThreshold`, and emits `MapRunFailed` with that error.+`Map` that declares no tolerance reports a failed item's own error rather than AWS's+`States.ExceedToleratedFailureThreshold`, and emits `MapRunFailed` with that error. A `Map` that+declares one reports `States.ExceedToleratedFailureThreshold`, as AWS does.## Map concurrency@@ -142,6 +143,24 @@ building a batch AWS would reject: reduce the item with `ItemSelector` first.than per item. `DescribeMapRun` reports items under `itemCounts` and batches under`executionCounts`.+## Tolerated failures++`ToleratedFailureCount` and `ToleratedFailurePercentage` let a Distributed `Map` absorb failed items+instead of failing on the first one. Both accept a `...Path` field, or an expression in a JSONata+state machine, and the percentage is taken over the item count. Declaring both applies the stricter+of the two.++An absorbed failure contributes no result, so the `Map` output carries one entry per successful+child execution. A `ResultWriter` still exports it: successful children go to `SUCCEEDED_0.json` and+absorbed failures to `FAILED_0.json`, each listed under the matching key of the manifest's+`ResultFiles`. A failed record carries `Error` and `Cause` in place of an output. Once the budget is+spent, the state fails with `States.ExceedToleratedFailureThreshold` and the run emits+`MapRunFailed`.++`DescribeMapRun` reports the declared values under `toleratedFailureCount` and+`toleratedFailurePercentage`. A `Map` that declares neither keeps the earlier behaviour: the first+failed item fails the state, carrying that item's own error.+## Retry policies`Task`, `Parallel`, and `Map` states honor their `Retry` field. `ErrorEquals` matchingdiff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.javaindex 1e071c5ed9..0ed2467e7f 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java@@ -2370,18 +2370,27 @@ public class AslExecutor {int effectiveConcurrency = effectiveMapConcurrency(childCount, requestedConcurrency, distributed);+ // A declared tolerance lets a Distributed Map absorb failed items instead of failing on the+ // first one. Absent, the state keeps the earlier behaviour and fails with the item's error.+ ToleratedFailures tolerated = resolveToleratedFailures(stateDef, itemCount, mapInput, jsonata,+ context, variables);+chain.publish("MapStateStarted", Map.of("length", itemCount));MapRunIdentity mapRun = null;MapRun mapRunRecord = null;if (distributed) {mapRun = newMapRunIdentity(stateDef, sm, context);mapRunRecord = newMapRun(mapRun, context, itemCount, childCount, requestedConcurrency);+ mapRunRecord.setToleratedFailureCount(tolerated.declaredCount());+ mapRunRecord.setToleratedFailurePercentage(tolerated.declaredPercentage());chain.publish("MapRunStarted", Map.of("mapRunArn", mapRun.arn()));}var succeededItems = new AtomicInteger();var failedItems = new AtomicInteger();AtomicInteger succeededExecutions = new AtomicInteger();AtomicInteger failedExecutions = new AtomicInteger();+ List<Integer> succeededChildren = new ArrayList<>(childCount);+ FailedChild[] failedByIndex = hasResultWriter ? new FailedChild[childCount] : null;var iterationChains = new ArrayList<HistoryChain>(childCount);for (var i = 0; i < childCount; i++) {iterationChains.add(distributed ? HistoryChain.ofChildExecution() : chain.fork());@@ -2418,12 +2427,25 @@ public class AslExecutor {branchOutput = executeBranch(startAt, iteratorStates, iterInput, iterationChain, sm,topLevelQueryLanguage, iterContext, variables.deepCopy());} catch (FailStateException e) {- failedItems.addAndGet(itemsInChild);+ int failedSoFar = failedItems.addAndGet(itemsInChild);failedExecutions.incrementAndGet();if (!distributed && !e.isRuntimeError()) {iterationChain.publishAside("MapIterationFailed", Map.of("name", name, "index", i));}- throw new IterationFailure(i, e);+ if (!tolerated.declared()) {+ throw new IterationFailure(i, e);+ }+ if (failedSoFar > tolerated.threshold()) {+ throw new IterationFailure(i, new FailStateException(+ "States.ExceedToleratedFailureThreshold",+ "The map run failed because a tolerated failure threshold was exceeded. "+ + failedSoFar + " of " + itemCount + " items failed."));+ }+ if (hasResultWriter) {+ failedByIndex[i] = new FailedChild(childInputsByIndex[i],+ new long[]{startMs, System.currentTimeMillis()}, e.error, e.cause);+ }+ return null;}succeededItems.addAndGet(itemsInChild);succeededExecutions.incrementAndGet();@@ -2460,20 +2482,33 @@ public class AslExecutor {} finally {iterationChains.forEach(HistoryChain::abandon);}- results.addAll(itemOutputs);+ for (int i = 0; i < itemOutputs.size(); i++) {+ if (itemOutputs.get(i) != null) {+ succeededChildren.add(i);+ results.add(itemOutputs.get(i));+ }+ }}JsonNode mapResult = results;if (hasResultWriter) {ArrayNode childInputs = objectMapper.createArrayNode();- List<long[]> childTimings = new ArrayList<>(childCount);- for (int i = 0; i < childCount; i++) {+ List<long[]> childTimings = new ArrayList<>(succeededChildren.size());+ for (int i : succeededChildren) {childInputs.add(childInputsByIndex[i]);childTimings.add(childTimingsByIndex[i]);}+ // Tolerated failures stay observable: AWS exports them to FAILED_n.json even when the run+ // itself stays within its budget.+ List<FailedChild> failedChildren = new ArrayList<>();+ for (int i = 0; i < childCount; i++) {+ if (failedByIndex[i] != null) {+ failedChildren.add(failedByIndex[i]);+ }+ }try {mapResult = applyResultWriter(name, stateDef, mapInput, results, childInputs, childTimings,- sm, context, jsonata, variables, mapRun);+ failedChildren, sm, context, jsonata, variables, mapRun);} catch (FailStateException e) {// A ResultWriter failure fails the Map run on AWS.publishMapRunFailedEvent(chain, e);@@ -2681,6 +2716,56 @@ public class AslExecutor {return batch;}+ /**+ * The failed-item budget a Distributed Map declared. {@code declaredCount} and+ * {@code declaredPercentage} are what DescribeMapRun reports; {@code threshold} is the number of+ * failed items the run absorbs, which is the stricter of the two whenever both are declared.+ */+ private record ToleratedFailures(boolean declared, int declaredCount, double declaredPercentage,+ int threshold) {+ }++ private ToleratedFailures resolveToleratedFailures(JsonNode stateDef, int itemCount, JsonNode mapInput,+ boolean jsonata, JsonNode context, ObjectNode variables) {+ boolean hasCount = stateDef.has("ToleratedFailureCount") || stateDef.has("ToleratedFailureCountPath");+ boolean hasPercentage = stateDef.has("ToleratedFailurePercentage")+ || stateDef.has("ToleratedFailurePercentagePath");+ if (!hasCount && !hasPercentage) {+ return new ToleratedFailures(false, 0, 0.0, 0);+ }++ int count = hasCount+ ? resolveToleranceField(stateDef, "ToleratedFailureCount", Integer.MAX_VALUE, mapInput,+ jsonata, context, variables)+ : 0;+ int percentage = hasPercentage+ ? resolveToleranceField(stateDef, "ToleratedFailurePercentage", 100, mapInput, jsonata,+ context, variables)+ : 0;++ int fromPercentage = (int) ((long) itemCount * percentage / 100);+ int threshold;+ if (hasCount && hasPercentage) {+ threshold = Math.min(count, fromPercentage);+ } else if (hasCount) {+ threshold = count;+ } else {+ threshold = fromPercentage;+ }+ return new ToleratedFailures(true, count, percentage, threshold);+ }++ /** A tolerance is a percentage or a count, so it shares the Map integer resolver and adds a ceiling. */+ private int resolveToleranceField(JsonNode stateDef, String field, int maximum, JsonNode mapInput,+ boolean jsonata, JsonNode context, ObjectNode variables) {+ int value = resolveMapIntegerField(stateDef, field, 0, mapInput, jsonata, context, variables);+ if (value > maximum) {+ throw new FailStateException("States.Runtime",+ field + " must resolve to an integer between 0 and " + maximum, field);+ }+ return value;+ }+/*** Emulates a Distributed Map state's {@code ResultWriter}@@ -2698,15 +2783,16 @@ public class AslExecutor {** <p>By construction every child branch here has already succeeded (a failed branch throws and* fails the Map before this point, since inline Maps here do not implement tolerated-failure),- * so {@code ResultFiles.FAILED} / {@code PENDING} are empty and all results go to a single- * {@code SUCCEEDED_0.json}.+ * so it passes no failed children: {@code ResultFiles.FAILED} / {@code PENDING} are empty and all+ * results go to a single {@code SUCCEEDED_0.json}.*/// Package-private for unit testing of the ResultWriter export/format behaviour.JsonNode applyResultWriter(String mapStateName, JsonNode stateDef, JsonNode input,ArrayNode results, ArrayNode childInputs, List<long[]> childTimings,StateMachine sm, JsonNode context, boolean jsonata) throws Exception {return applyResultWriter(mapStateName, stateDef, input, results, childInputs, childTimings,- sm, context, jsonata, objectMapper.createObjectNode(), newMapRunIdentity(stateDef, sm, context));+ List.of(), sm, context, jsonata, objectMapper.createObjectNode(),+ newMapRunIdentity(stateDef, sm, context));}private record MapRunIdentity(String label, String id, String arn) {@@ -2727,8 +2813,9 @@ public class AslExecutor {private JsonNode applyResultWriter(String mapStateName, JsonNode stateDef, JsonNode input,ArrayNode results, ArrayNode childInputs, List<long[]> childTimings,- StateMachine sm, JsonNode context, boolean jsonata,- ObjectNode variables, MapRunIdentity mapRun) throws Exception {+ List<FailedChild> failedChildren, StateMachine sm, JsonNode context,+ boolean jsonata, ObjectNode variables, MapRunIdentity mapRun)+ throws Exception {JsonNode writer = stateDef.get("ResultWriter");JsonNode writerConfig = writer.path("WriterConfig");boolean export = writer.hasNonNull("Resource");@@ -2818,7 +2905,17 @@ public class AslExecutor {manifest.put("DestinationBucket", bucket);manifest.put("MapRunArn", mapRunArn);ObjectNode resultFiles = manifest.putObject("ResultFiles");- resultFiles.putArray("FAILED");+ ArrayNode failedFiles = resultFiles.putArray("FAILED");+ if (!failedChildren.isEmpty()) {+ String failedKey = base + "FAILED_0.json";+ byte[] failedBytes = serializeResultFile(+ formatFailedChildren(failedChildren, region, account, smName, mapRun.label()),+ outputType);+ s3Service.putObject(bucket, failedKey, failedBytes, "application/json", new HashMap<>());+ ObjectNode failedEntry = failedFiles.addObject();+ failedEntry.put("Key", failedKey);+ failedEntry.put("Size", failedBytes.length);+ }resultFiles.putArray("PENDING");ObjectNode succeededEntry = resultFiles.putArray("SUCCEEDED").addObject();succeededEntry.put("Key", succeededKey);@@ -2888,6 +2985,38 @@ public class AslExecutor {return out;}+ /** A child execution that failed within the Map's tolerated budget. */+ private record FailedChild(JsonNode input, long[] timing, String error, String cause) {+ }++ /** The FAILED_n.json records, which carry the child's error rather than an output. */+ private ArrayNode formatFailedChildren(List<FailedChild> failedChildren, String region, String account,+ String smName, String mapRunLabel) {+ ArrayNode out = objectMapper.createArrayNode();+ String childSmArn = "arn:aws:states:" + region + ":" + account + ":stateMachine:"+ + smName + "/" + mapRunLabel;+ for (FailedChild child : failedChildren) {+ String childId = UUID.randomUUID().toString();+ ObjectNode record = out.addObject();+ record.put("ExecutionArn", "arn:aws:states:" + region + ":" + account + ":execution:"+ + smName + "/" + mapRunLabel + ":" + childId);+ record.put("Input", stringifyResult(child.input()));+ record.putObject("InputDetails").put("Included", true);+ record.put("Name", childId);+ record.put("Error", child.error());+ record.put("Cause", child.cause());+ record.putObject("OutputDetails").put("Included", false);+ record.put("RedriveCount", 0);+ record.put("RedriveStatus", "REDRIVABLE");+ record.put("RedriveStatusReason", "Execution is FAILED and can be redriven");+ record.put("StartDate", Instant.ofEpochMilli(child.timing()[0]).toString());+ record.put("StateMachineArn", childSmArn);+ record.put("Status", "FAILED");+ record.put("StopDate", Instant.ofEpochMilli(child.timing()[1]).toString());+ }+ return out;+ }+private String stringifyResult(JsonNode node) {if (node == null || node.isNull() || node.isMissingNode()) {return "null";diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.javaindex 04ae29f710..a5697d03dd 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java@@ -393,7 +393,8 @@ public class StepFunctionsJsonHandler {* {@code arn:aws:states:::aws-sdk:sfn:describeMapRun} Task integration renders the same node in* PascalCase, so this is the one place the response is described.*- * <p>Both tolerances and {@code redriveCount} are zero because Floci implements neither, and+ * <p>The tolerances are the ones the Map state declared, and default to zero.+ * {@code redriveCount} is zero because Floci does not implement redrive, and* {@code redriveDate} is absent until a run is redriven, which no run here ever is.*/static ObjectNode describeMapRunResponse(ObjectMapper objectMapper, MapRun mapRun) {@@ -404,8 +405,8 @@ public class StepFunctionsJsonHandler {response.put("startDate", mapRun.getStartDate());response.put("stopDate", mapRun.getStopDate());response.put("maxConcurrency", mapRun.getMaxConcurrency());- response.put("toleratedFailurePercentage", 0.0);- response.put("toleratedFailureCount", 0);+ response.put("toleratedFailurePercentage", mapRun.getToleratedFailurePercentage());+ response.put("toleratedFailureCount", mapRun.getToleratedFailureCount());putMapRunCounts(response.putObject("itemCounts"), mapRun.getItemCount(),mapRun.getSucceededCount(), mapRun.getFailedCount());// An ItemBatcher run has one execution per batch, so the two blocks differ there.diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/MapRun.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/MapRun.javaindex 1e5244c161..651e07c60e 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/MapRun.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/MapRun.java@@ -27,6 +27,9 @@ public class MapRun {private int succeededExecutionCount;private int failedExecutionCount;+ private int toleratedFailureCount;+ private double toleratedFailurePercentage;+private int maxConcurrency;public String getMapRunArn() { return mapRunArn; }@@ -51,6 +54,11 @@ public class MapRun {public int getFailedCount() { return failedCount; }public void setFailedCount(int failedCount) { this.failedCount = failedCount; }+ public int getToleratedFailureCount() { return toleratedFailureCount; }+ public void setToleratedFailureCount(int count) { this.toleratedFailureCount = count; }+ public double getToleratedFailurePercentage() { return toleratedFailurePercentage; }+ public void setToleratedFailurePercentage(double percentage) { this.toleratedFailurePercentage = percentage; }+public int getExecutionCount() { return executionCount; }public void setExecutionCount(int executionCount) { this.executionCount = executionCount; }public int getSucceededExecutionCount() { return status == null ? executionCount : succeededExecutionCount; }diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsDescribeMapRunIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsDescribeMapRunIntegrationTest.javaindex 4029c98c9d..105743be83 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsDescribeMapRunIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsDescribeMapRunIntegrationTest.java@@ -334,6 +334,43 @@ class StepFunctionsDescribeMapRunIntegrationTest {assertEquals(3, response.jsonPath().getInt("executionCounts.succeeded"));}+ /** The reported tolerances are the ones the Map state declared, not a measured run. */+ @Test+ @Order(13)+ void describeMapRunReportsTheDeclaredTolerances() throws Exception {+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "End": true,+ "Items": [{"n": 1}, {"n": 2}],+ "ToleratedFailureCount": 2,+ "ToleratedFailurePercentage": 50,+ "ItemProcessor": {+ "ProcessorConfig": {"Mode": "DISTRIBUTED", "ExecutionType": "STANDARD"},+ "StartAt": "Keep",+ "States": {"Keep": {"Type": "Pass", "End": true}}+ }+ }+ }+ }+ """;++ String smArn = createStateMachine("describe-map-run-tolerances", definition);+ String execArn = startExecution(smArn, "{}");+ Response describe = waitForTerminalState(execArn);+ assertEquals("SUCCEEDED", describe.jsonPath().getString("status"),+ "cause: " + describe.jsonPath().getString("cause"));++ Response response = describeMapRun(mapRunArnFromHistory(execArn));+ response.then().statusCode(200);+ assertEquals(2, response.jsonPath().getInt("toleratedFailureCount"));+ assertEquals(50.0, response.jsonPath().getDouble("toleratedFailurePercentage"));+ }+private static String distributedMap(String items, String bucket, Integer maxConcurrency) {return """{diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.javaindex 60eeaae2e6..0f9ba77e83 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java@@ -1227,6 +1227,296 @@ class StepFunctionsJsonataIntegrationTest {assertEquals("States.DataLimitExceeded", failure.jsonPath().getString("error"));}+ @Test+ void distributedMapToleratedFailureCount_absorbsFailuresUpToTheThreshold() throws Exception {+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "Items": [{"n": 1}, {"n": -1}, {"n": 3}],+ "ToleratedFailureCount": 1,+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "Check",+ "States": {+ "Check": {+ "Type": "Choice",+ "Choices": [+ {+ "Condition": "{% $states.input.n < 0 %}",+ "Next": "Boom"+ }+ ],+ "Default": "Keep"+ },+ "Boom": {+ "Type": "Fail",+ "Error": "ItemFailed",+ "Cause": "the item asked to fail"+ },+ "Keep": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-tolerated-count-within", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);+ JsonNode results = new ObjectMapper().readTree(output);++ assertEquals(2, results.size(), "the failed item leaves no result: " + output);+ }++ @Test+ void distributedMapToleratedFailureCount_failsWithExceedToleratedFailureThreshold() throws Exception {+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "Items": [{"n": 1}, {"n": -1}, {"n": -2}],+ "ToleratedFailureCount": 1,+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "Check",+ "States": {+ "Check": {+ "Type": "Choice",+ "Choices": [+ {+ "Condition": "{% $states.input.n < 0 %}",+ "Next": "Boom"+ }+ ],+ "Default": "Keep"+ },+ "Boom": {+ "Type": "Fail",+ "Error": "ItemFailed",+ "Cause": "the item asked to fail"+ },+ "Keep": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-tolerated-count-exceeded", definition);+ String execArn = startExecution(smArn, "{}");+ Response failure = waitForExecutionFailure(execArn);++ assertEquals("FAILED", failure.jsonPath().getString("status"));+ assertEquals("States.ExceedToleratedFailureThreshold", failure.jsonPath().getString("error"));+ }++ @Test+ void distributedMapToleratedFailurePercentage_absorbsFailuresUpToTheThreshold() throws Exception {+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "Items": [{"n": 1}, {"n": 2}, {"n": 3}, {"n": -1}],+ "ToleratedFailurePercentage": 25,+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "Check",+ "States": {+ "Check": {+ "Type": "Choice",+ "Choices": [+ {+ "Condition": "{% $states.input.n < 0 %}",+ "Next": "Boom"+ }+ ],+ "Default": "Keep"+ },+ "Boom": {+ "Type": "Fail",+ "Error": "ItemFailed",+ "Cause": "the item asked to fail"+ },+ "Keep": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-tolerated-percentage-within", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);+ JsonNode results = new ObjectMapper().readTree(output);++ assertEquals(3, results.size(), "one of four items may fail at 25 percent: " + output);+ }++ @Test+ void distributedMapWithoutAToleratedFailure_stillFailsWithTheItemsOwnError() throws Exception {+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "Items": [{"n": 1}, {"n": -1}],++ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "Check",+ "States": {+ "Check": {+ "Type": "Choice",+ "Choices": [+ {+ "Condition": "{% $states.input.n < 0 %}",+ "Next": "Boom"+ }+ ],+ "Default": "Keep"+ },+ "Boom": {+ "Type": "Fail",+ "Error": "ItemFailed",+ "Cause": "the item asked to fail"+ },+ "Keep": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-tolerated-absent", definition);+ String execArn = startExecution(smArn, "{}");+ Response failure = waitForExecutionFailure(execArn);++ assertEquals("FAILED", failure.jsonPath().getString("status"));+ assertEquals("ItemFailed", failure.jsonPath().getString("error"));+ }++ @Test+ void distributedMapToleratedFailure_exportsFailedChildrenAlongsideSucceededOnes() throws Exception {+ createBucket("map-tolerated-export");++ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "Items": [{"n": 1}, {"n": -1}, {"n": 3}],+ "ToleratedFailureCount": 1,+ "ItemProcessor": {+ "ProcessorConfig": {+ "Mode": "DISTRIBUTED",+ "ExecutionType": "STANDARD"+ },+ "StartAt": "Check",+ "States": {+ "Check": {+ "Type": "Choice",+ "Choices": [+ {+ "Condition": "{% $states.input.n < 0 %}",+ "Next": "Boom"+ }+ ],+ "Default": "Keep"+ },+ "Boom": {+ "Type": "Fail",+ "Error": "ItemFailed",+ "Cause": "the item asked to fail"+ },+ "Keep": {+ "Type": "Pass",+ "End": true+ }+ }+ },+ "ResultWriter": {+ "Resource": "arn:aws:states:::s3:putObject",+ "Arguments": {+ "Bucket": "map-tolerated-export",+ "Prefix": "out"+ }+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("map-tolerated-export", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);++ ObjectMapper json = new ObjectMapper();+ JsonNode details = json.readTree(output).path("ResultWriterDetails");+ JsonNode manifest = json.readTree(getObject(details.path("Bucket").asText(),+ details.path("Key").asText()));++ assertEquals(1, manifest.path("ResultFiles").path("SUCCEEDED").size());+ assertEquals(1, manifest.path("ResultFiles").path("FAILED").size(),+ "the tolerated failure must still be exported: " + manifest);++ JsonNode succeeded = json.readTree(getObject("map-tolerated-export",+ manifest.path("ResultFiles").path("SUCCEEDED").get(0).path("Key").asText()));+ assertEquals(2, succeeded.size());++ JsonNode failed = json.readTree(getObject("map-tolerated-export",+ manifest.path("ResultFiles").path("FAILED").get(0).path("Key").asText()));+ assertEquals(1, failed.size());+ assertEquals("FAILED", failed.get(0).path("Status").asText());+ assertEquals("ItemFailed", failed.get(0).path("Error").asText());+ }+@Testvoid distributedMapWithS3JsonItemReader_objectIteratesKeyValuePairs() throws Exception {createBucket("map-inputs-object");@@ -2750,6 +3040,16 @@ class StepFunctionsJsonataIntegrationTest {.statusCode(200);}+ private String getObject(String bucket, String key) {+ return given()+ .when()+ .get("/" + bucket + "/" + key)+ .then()+ .statusCode(200)+ .extract()+ .asString();+ }+private void putObject(String bucket, String key, String body) {given().body(body)