Light, fluffy, and always free - The AWS Local Emulator alternative
fix(stepfunctions): fail a Parallel the moment any branch fails, whichever is listed first (#4713)
mitsukado-shinagawa6ca104042b25
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.mdindex 01cbfc8361..19c71319dc 100644--- a/docs/services/step-functions.md+++ b/docs/services/step-functions.md@@ -92,7 +92,9 @@ the parent execution's history, as on AWS. A `Parallel` records `ParallelStateSthistory. A `Task` whose failure ends its branch also records `TaskStateAborted`. When a failureends a `Parallel`, each other branch that is still inside a `Task` or a `Wait` records`TaskStateAborted` or `WaitStateAborted`, chained to the failing branch's last event and recorded-before `ParallelStateFailed`.+before `ParallelStateFailed`. The `Parallel` fails the moment any branch fails, whichever branch+is listed first, and cuts the others then; the branch outputs still come back in declaration+order.Branches and iterations run concurrently, so the order in which their events interleave differsfrom run to run. Each branch chains its own events through `previousEventId`, and that chain isdiff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.javaindex 542b55b961..cf745b619a 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java@@ -108,8 +108,10 @@ import java.util.UUID;import java.util.concurrent.Callable;import java.util.concurrent.CompletableFuture;import java.util.concurrent.CompletionException;+import java.util.concurrent.CompletionService;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutionException;+import java.util.concurrent.ExecutorCompletionService;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;@@ -2623,6 +2625,11 @@ public class AslExecutor {JsonNode branches = stateDef.path("Branches");chain.publish("ParallelStateStarted", null);var branchChains = new ArrayList<HistoryChain>();+ // The branches are joined in the order they complete, not the order they are declared, so+ // a failure in any branch ends the Parallel while the others are still running, as on AWS+ // (measured: the Parallel fails within about 0.1 s of the failing branch, whichever branch+ // is listed first). The outputs still land in declaration order.+ CompletionService<JsonNode> completions = new ExecutorCompletionService<>(executor);List<Future<JsonNode>> futures = new ArrayList<>();for (JsonNode branch : branches) {@@ -2644,7 +2651,7 @@ public class AslExecutor {// A branch state that declares no QueryLanguage defaults to the state machine's, not to// this Parallel's: the ASL specification calls the two independent, so what travels// into the branch is topLevelQueryLanguage. https://states-language.net/spec.html- futures.add(executor.submit(() -> callUnderExecutionAccount(sm,+ futures.add(completions.submit(() -> callUnderExecutionAccount(sm,() -> executeBranch(startAt, branchStates, capturedInput, branchChain, sm,topLevelQueryLanguage, branchContext, branchVariables))));}@@ -2657,20 +2664,22 @@ public class AslExecutor {// own TimeoutSeconds, and the state machine's budget for the whole execution.long joinDeadlineNanos = Math.min(stateDeadlineNanos, executionDeadlineNanos);- ArrayNode results = objectMapper.createArrayNode();- var joined = 0;+ JsonNode[] outputs = new JsonNode[futures.size()];+ // The branch whose future was last taken from the completion queue: the failed one when+ // the join ends in ExecutionException.+ int current = -1;try {- for (Future<JsonNode> future : futures) {+ for (int joined = 0; joined < futures.size(); joined++) {long remainingNanos = joinDeadlineNanos - System.nanoTime();if (remainingNanos <= 0) {throw parallelJoinExpired(stateDeadlineNanos, timeoutSeconds);}- try {- results.add(future.get(remainingNanos, TimeUnit.NANOSECONDS));- } catch (java.util.concurrent.TimeoutException e) {+ Future<JsonNode> done = completions.poll(remainingNanos, TimeUnit.NANOSECONDS);+ if (done == null) {throw parallelJoinExpired(stateDeadlineNanos, timeoutSeconds);}- joined++;+ current = futures.indexOf(done);+ outputs[current] = done.get();}} catch (InterruptedException e) {abandon(branchChains, futures);@@ -2678,7 +2687,7 @@ public class AslExecutor {throw e;} catch (ExecutionException e) {List<String> abortedEventTypes = cutAfterFailure(branchChains, futures);- chain.continueFrom(branchChains.get(joined).lastEventId());+ chain.continueFrom(branchChains.get(current).lastEventId());if (e.getCause() instanceof FailStateException failure && !failure.isRuntimeError()) {for (String abortedEventType : abortedEventTypes) {chain.publishBeside(abortedEventType, null);@@ -2705,6 +2714,10 @@ public class AslExecutor {chain.continueAfter(branchChains);chain.publishAside("ParallelStateSucceeded", null);+ ArrayNode results = objectMapper.createArrayNode();+ for (JsonNode output : outputs) {+ results.add(output);+ }if (jsonata) {JsonNode output = applyJsonataOutput(stateDef, input, results, context, variables);return new StateResult(output, stateDef.path("Next").asText(null));diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorParallelCutTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorParallelCutTest.javaindex ca50b340f5..fde877ddac 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorParallelCutTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorParallelCutTest.java@@ -22,6 +22,7 @@ import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;import jakarta.enterprise.inject.Instance;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;import java.time.Clock;import java.util.ArrayList;@@ -50,7 +51,9 @@ import static org.mockito.Mockito.when;* cause below, and records one {@code TaskStateAborted} (or {@code WaitStateAborted}) per cut* branch, each chained to the failing branch's last event and recorded before* {@code ParallelStateFailed}, which is chained to that same event. When the execution's budget- * cuts the Parallel instead, the child is aborted as well and no such event is recorded. An+ * cuts the Parallel instead, the child is aborted as well and no such event is recorded. The+ * failure is seen the moment it happens, whichever branch is listed first: the branches are+ * joined as they complete, and the outputs still come back in declaration order. An* iteration of an inline Map that fails cuts the other iterations the same way, and the job one of* them waits on is aborted too (measured: within about 0.15 s, with the same cause, whether or not* a Catch on the Map keeps the execution going).@@ -94,6 +97,11 @@ class AslExecutorParallelCutTest {"Boom":{"Type":"Fail","Error":"Boom","Cause":"sibling failed"},"Nest":{"Type":"Task","Resource":"arn:aws:states:::states:startExecution.sync:2","Parameters":{"StateMachineArn":"%s"},"End":true}}}}""".formatted(CHILD_SM_ARN);+ private static final String SLOW_RESULT_BRANCH = """+ {"StartAt":"Slow","States":{"Slow":{"Type":"Wait","Seconds":1,"Next":"Out"},+ "Out":{"Type":"Pass","Result":"slow","End":true}}}""";+ private static final String FAST_RESULT_BRANCH = """+ {"StartAt":"Out","States":{"Out":{"Type":"Pass","Result":"fast","End":true}}}""";private final ObjectMapper objectMapper = new ObjectMapper();private StepFunctionsService sfnService;@@ -128,6 +136,47 @@ class AslExecutorParallelCutTest {verify(sfnService, timeout(5_000)).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());}+ /** Fails against a declaration-order join after the default 300 s Task timeout; the limit ends it sooner. */+ @Test+ @Timeout(20)+ void aFailureInALaterBranchCutsAnEarlierBranchStillWaitingOnItsJob() {+ Execution execution = run(parallel(NESTED_SYNC_BRANCH, FAILING_BRANCH), 1, 0);++ assertEquals("FAILED", execution.getStatus());+ assertEquals("Boom", execution.getError());+ long failure = eventOfType("FailStateEntered").getId();+ HistoryEvent aborted = eventOfType("TaskStateAborted");+ assertEquals(failure, aborted.getPreviousEventId().longValue());+ assertEquals(failure, eventOfType("ParallelStateFailed").getPreviousEventId().longValue());+ assertTrue(aborted.getId() < eventOfType("ParallelStateFailed").getId());+ verify(sfnService, timeout(5_000)).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());+ }++ /** The earlier branch's Wait really sleeps 20 s; a declaration-order join sees the failure only then. */+ @Test+ @Timeout(10)+ void aFailureInALaterBranchIsSeenWhileAnEarlierWaitStillRuns() {+ Execution execution = run(parallel(LONG_WAIT_BRANCH, FAILING_BRANCH), 1, 0);++ assertEquals("FAILED", execution.getStatus());+ assertEquals("Boom", execution.getError());+ long failure = eventOfType("FailStateEntered").getId();+ assertEquals(failure, eventOfType("WaitStateAborted").getPreviousEventId().longValue());+ // The failing branch's own Pause exits; the cut Long never does.+ List<String> exitedWaits = eventsOfType("WaitStateExited").stream()+ .map(event -> String.valueOf(event.getDetails().get("name"))).toList();+ assertEquals(List.of("Pause"), exitedWaits, types().toString());+ }++ @Test+ void outputsStayInDeclarationOrderWhenAnEarlierBranchFinishesLast() throws Exception {+ AslExecutor.Sleeper sleeper = nanos -> TimeUnit.MILLISECONDS.sleep(200);+ Execution execution = run(parallel(SLOW_RESULT_BRANCH, FAST_RESULT_BRANCH), sleeper, 0);++ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());+ assertEquals(objectMapper.readTree("[\"slow\",\"fast\"]"), objectMapper.readTree(execution.getOutput()));+ }+@Testvoid everyCutBranchRecordsItsOwnTaskStateAbortedAgainstTheSameEvent() {Execution execution = run(parallel(FAILING_BRANCH, NESTED_SYNC_BRANCH, NESTED_SYNC_BRANCH), 2, 0);diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsParallelBranchFailureIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsParallelBranchFailureIntegrationTest.javanew file mode 100644index 0000000000..5a4528368e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsParallelBranchFailureIntegrationTest.java@@ -0,0 +1,142 @@+package io.github.hectorvent.floci.services.stepfunctions;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.util.List;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assertions.fail;++/**+ * A {@code Parallel} fails the moment any of its branches fails, whichever branch is listed first,+ * the way AWS does (measured in ap-northeast-1 on 2026-09-29: the execution fails within about+ * 0.1 s of the failing branch while a sibling listed before it is still running). The branch that+ * was still in its {@code Wait} records {@code WaitStateAborted} and never exits.+ */+@QuarkusTest+class StepFunctionsParallelBranchFailureIntegrationTest {++ private static final String SFN_CONTENT_TYPE = "application/x-amz-json-1.0";+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/test-role";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void aFailureInTheSecondBranchEndsTheParallelWhileTheFirstStillWaits() throws Exception {+ String smArn = createStateMachine("parallel-later-branch-fails", """+ {+ "StartAt": "P",+ "States": {+ "P": {+ "Type": "Parallel",+ "End": true,+ "Branches": [+ {"StartAt": "Long", "States": {"Long": {"Type": "Wait", "Seconds": 15, "End": true}}},+ {"StartAt": "Pause", "States": {+ "Pause": {"Type": "Wait", "Seconds": 1, "Next": "Boom"},+ "Boom": {"Type": "Fail", "Error": "Boom", "Cause": "sibling failed"}}}+ ]+ }+ }+ }+ """);+ String execArn = startExecution(smArn, "{}");++ // Well inside the first branch's 15 s Wait: a declaration-order join is still RUNNING here.+ Response failed = waitForTerminalExecution(execArn, 8_000);++ assertEquals("FAILED", failed.jsonPath().getString("status"), failed.body().asString());+ assertEquals("Boom", failed.jsonPath().getString("error"));+ assertEquals("sibling failed", failed.jsonPath().getString("cause"));+ Response history = getExecutionHistory(execArn);+ List<String> types = history.jsonPath().getList("events.type");+ assertTrue(types.contains("WaitStateAborted"), types.toString());+ assertTrue(types.indexOf("WaitStateAborted") < types.indexOf("ParallelStateFailed"), types.toString());+ // The failing branch's own Pause exits; the cut Long never does.+ List<String> exitedWaits = history.jsonPath()+ .getList("events.findAll { it.type == 'WaitStateExited' }.stateExitedEventDetails.name");+ assertEquals(List.of("Pause"), exitedWaits, types.toString());+ }++ private static String createStateMachine(String name, String definition) {+ Response resp = given()+ .header("X-Amz-Target", "AWSStepFunctions.CreateStateMachine")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"name": "%s-%d", "definition": %s, "roleArn": "%s"}+ """.formatted(name, System.currentTimeMillis(), quote(definition), ROLE_ARN))+ .when()+ .post("/");+ resp.then().statusCode(200);+ return resp.jsonPath().getString("stateMachineArn");+ }++ private static String startExecution(String smArn, String input) {+ Response resp = given()+ .header("X-Amz-Target", "AWSStepFunctions.StartExecution")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"stateMachineArn": "%s", "input": %s}+ """.formatted(smArn, quote(input)))+ .when()+ .post("/");+ resp.then().statusCode(200);+ return resp.jsonPath().getString("executionArn");+ }++ private static Response describeExecution(String execArn) {+ return given()+ .header("X-Amz-Target", "AWSStepFunctions.DescribeExecution")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"executionArn": "%s"}+ """.formatted(execArn))+ .when()+ .post("/");+ }++ private static Response getExecutionHistory(String execArn) {+ Response resp = given()+ .header("X-Amz-Target", "AWSStepFunctions.GetExecutionHistory")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"executionArn": "%s"}+ """.formatted(execArn))+ .when()+ .post("/");+ resp.then().statusCode(200);+ return resp;+ }++ private static Response waitForTerminalExecution(String execArn, long limitMillis) throws InterruptedException {+ long deadline = System.currentTimeMillis() + limitMillis;+ while (System.currentTimeMillis() < deadline) {+ Response resp = describeExecution(execArn);+ if (!"RUNNING".equals(resp.jsonPath().getString("status"))) {+ return resp;+ }+ Thread.sleep(100);+ }+ fail("Execution " + execArn + " was still RUNNING after " + limitMillis + " ms");+ return null;+ }++ private static String quote(String raw) {+ return "\"" + raw+ .replace("\\", "\\\\")+ .replace("\"", "\\\"")+ .replace("\n", "\\n")+ .replace("\r", "\\r")+ .replace("\t", "\\t")+ + "\"";+ }+}