Light, fluffy, and always free - The AWS Local Emulator alternative
fix(stepfunctions): end the execution when a state throws an Error (#2693)
Facundo Paredesacf88794ca4e
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.javaindex 4589329543..2e0ab71ebf 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java@@ -217,7 +217,16 @@ public class AslExecutor {MockedTestCase mockedTestCase,BiConsumer<Execution, List<HistoryEvent>> onUpdate) {registerMocks(exec, mockedTestCase);- executor.submit(() -> runUnderExecutionAccount(sm, () -> doExecute(sm, exec, history, onUpdate)));+ executor.submit(() -> {+ try {+ runUnderExecutionAccount(sm, () -> doExecute(sm, exec, history, onUpdate));+ } catch (RuntimeException | Error e) {+ // submit() parks whatever the task throws in a Future nobody reads, so without this+ // the worker dies silently. doExecute has already published the terminal status by+ // now; this is the only place the stack trace of what killed it reaches the log.+ LOG.errorv(e, "ASL execution worker failed for {0}", exec.getExecutionArn());+ }+ });}/**@@ -313,8 +322,10 @@ public class AslExecutor {private void doExecute(StateMachine sm, Execution exec, List<HistoryEvent> history,BiConsumer<Execution, List<HistoryEvent>> onUpdate) {+ // Declared outside the try so the terminal handlers below can keep numbering the history+ // where the execution left it.+ AtomicLong eventId = new AtomicLong(history.size());try {- AtomicLong eventId = new AtomicLong(history.size());JsonNode definition = objectMapper.readTree(sm.getDefinition());JsonNode states = definition.path("States");String startAt = definition.path("StartAt").asText();@@ -364,17 +375,6 @@ public class AslExecutor {failExecution(exec, history, eventId, e);onUpdate.accept(exec, history);return;- } catch (Exception e) {- String runtimeError = "States.Runtime";- String runtimeCause = e.getMessage() != null ? e.getMessage() : "Unknown error";- exec.setError(runtimeError);- exec.setCause(runtimeCause);- exec.setStopDate(System.currentTimeMillis() / 1000.0);- exec.setStatus("FAILED");- addEvent(history, eventId, "ExecutionFailed", null,- Map.of("error", runtimeError, "cause", runtimeCause));- onUpdate.accept(exec, history);- return;}}@@ -393,11 +393,19 @@ public class AslExecutor {LOG.warnv("ASL execution failed for {0}: {1}", exec.getExecutionArn(), e.getMessage());// This path previously set only the status, leaving error and cause null forever on an// execution DescribeExecution reports as FAILED.- exec.setError("States.Runtime");- exec.setCause(e.getMessage() != null ? e.getMessage() : "Unknown error");- exec.setStopDate(System.currentTimeMillis() / 1000.0);- exec.setStatus("FAILED");+ failExecution(exec, history, eventId, "States.Runtime",+ e.getMessage() != null ? e.getMessage() : "Unknown error");onUpdate.accept(exec, history);+ } catch (Error e) {+ // An Error is not a state failure: it says the runtime itself is broken, and no retry+ // of the state machine can get past it. Publishing the same terminal FAILED an+ // exception produces is what keeps DescribeExecution from reporting RUNNING forever,+ // and the rethrow keeps the Error itself from being swallowed here. The cause carries+ // toString() rather than getMessage(), because an Error's message is often null and+ // the type name is the whole diagnostic.+ failExecution(exec, history, eventId, "States.Runtime", e.toString());+ onUpdate.accept(exec, history);+ throw e;} finally {activeMocks.remove(exec.getExecutionArn());}@@ -1635,13 +1643,18 @@ public class AslExecutor {throw e;} catch (ExecutionException e) {futures.forEach(future -> future.cancel(true));- // Unwrap so a branch's FailStateException reaches the Parallel state's own- // Retry and Catch handling instead of surfacing as States.Runtime. An Error- // cause stays wrapped so the execution-level Exception handlers still- // publish a terminal FAILED update instead of leaving the execution RUNNING.+ // Unwrap so a branch's FailStateException reaches the Parallel state's own Retry and+ // Catch handling instead of surfacing as States.Runtime, and so an Error reaches the+ // execution-level handler as itself rather than as an ExecutionException wrapper. The+ // reported cause is the same either way, since ExecutionException.getMessage() is the+ // cause's toString(), but only the unwrapped Error is rethrown and logged with its+ // stack trace.if (e.getCause() instanceof Exception exception) {throw exception;}+ if (e.getCause() instanceof Error error) {+ throw error;+ }throw e;} catch (Exception | Error e) {futures.forEach(future -> future.cancel(true));@@ -3110,14 +3123,23 @@ public class AslExecutor {}private void failExecution(Execution exec, List<HistoryEvent> history, AtomicLong eventId, FailStateException e) {- String failError = e.error != null ? e.error : "States.Runtime";- String failCause = e.cause != null ? e.cause : "";- exec.setError(failError);- exec.setCause(failCause);+ failExecution(exec, history, eventId, e.error != null ? e.error : "States.Runtime",+ e.cause != null ? e.cause : "");+ }++ /**+ * The single terminal-failure write: every way an execution can fail leaves the same+ * {@code error}, {@code cause} and {@code ExecutionFailed} event behind, so a client cannot+ * tell a Fail state from a state that threw from a runtime Error by what it reads back.+ */+ private void failExecution(Execution exec, List<HistoryEvent> history, AtomicLong eventId,+ String error, String cause) {+ exec.setError(error);+ exec.setCause(cause);exec.setStopDate(System.currentTimeMillis() / 1000.0);exec.setStatus("FAILED");addEvent(history, eventId, "ExecutionFailed", null,- Map.of("error", failError, "cause", failCause));+ Map.of("error", error, "cause", cause));}private StateResult handleCatch(JsonNode stateDef, JsonNode input, FailStateException failure,diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorErrorTerminationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorErrorTerminationTest.javanew file mode 100644index 0000000000..60c055cc83--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorErrorTerminationTest.java@@ -0,0 +1,256 @@+package io.github.hectorvent.floci.services.stepfunctions;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbJsonHandler;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;+import io.github.hectorvent.floci.services.lambda.LambdaExecutorService;+import io.github.hectorvent.floci.services.lambda.LambdaFunctionStore;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;+import io.github.hectorvent.floci.services.stepfunctions.model.Execution;+import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;+import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;+import io.quarkus.test.junit.QuarkusTest;+import io.vertx.mutiny.core.Vertx;+import jakarta.enterprise.inject.Instance;+import jakarta.inject.Inject;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;++import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.mock;++/**+ * Guards that an {@link Error} escaping a state ends the execution instead of leaving it RUNNING+ * forever, the failure reported in issue #2666.+ *+ * <p>The trigger described there is native-image only: the JSONata {@code ??} and {@code ?:}+ * operators clone their parser through Java serialization, which a GraalVM image answers with an+ * {@code UnsupportedFeatureError}. The defect underneath it is not native-specific, so these tests+ * inject the same shape on the JVM: a {@link JsonataEvaluator} whose template resolution throws a+ * {@link LinkageError}, which is what evaluating {@code Output} does inside the image.+ *+ * <p>Every case is bounded by a timeout, because the symptom being guarded is an execution that+ * never terminates.+ */+@QuarkusTest+class AslExecutorErrorTerminationTest {++ private static final String REGION = "us-east-2";+ private static final String ACCOUNT = "000000000000";+ private static final String ERROR_MESSAGE = "parser clone is unavailable in this image";+ private static final String EXPECTED_CAUSE = "java.lang.LinkageError: " + ERROR_MESSAGE;++ private final ObjectMapper objectMapper = new ObjectMapper();+ private AslExecutor executor;++ @Inject+ Vertx vertx;++ /** Throws where the native image throws: resolving a state's JSONata {@code Output}. */+ private static final class CrashingJsonataEvaluator extends JsonataEvaluator {+ CrashingJsonataEvaluator(ObjectMapper objectMapper) {+ super(objectMapper);+ }++ @Override+ JsonNode resolveTemplate(JsonNode template, JsonNode statesVar, JsonNode variables) {+ throw new LinkageError(ERROR_MESSAGE);+ }+ }++ @BeforeEach+ void setUp() {+ executor = new AslExecutor(+ mock(LambdaExecutorService.class),+ mock(LambdaFunctionStore.class),+ mock(DynamoDbService.class),+ mock(DynamoDbJsonHandler.class),+ mock(SqsJsonHandler.class),+ mock(io.github.hectorvent.floci.services.cloudformation.CloudFormationQueryHandler.class),+ mock(io.github.hectorvent.floci.services.ec2.Ec2Service.class),+ mock(S3Service.class),+ mock(io.github.hectorvent.floci.services.ecs.EcsService.class),+ mock(io.github.hectorvent.floci.services.ecs.EcsJsonHandler.class),+ objectMapper,+ new CrashingJsonataEvaluator(objectMapper),+ mock(Instance.class), mock(EmulatorConfig.class), vertx);+ }++ private static final String PASS_WITH_JSONATA_OUTPUT = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Crash",+ "States": {+ "Crash": {+ "Type": "Pass",+ "Output": {"v": "{% 42 %}"},+ "End": true+ }+ }+ }+ """;++ @Test+ @Timeout(60)+ void errorInsideAStateFailsTheExecutionWithTheErrorAsCause() {+ List<HistoryEvent> history = new ArrayList<>();+ Execution execution = runSync(PASS_WITH_JSONATA_OUTPUT, history);++ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.Runtime", execution.getError());+ assertEquals(EXPECTED_CAUSE, execution.getCause());+ assertNotNull(execution.getStopDate());+ }++ @Test+ @Timeout(60)+ void errorInsideAStateRecordsAnExecutionFailedEvent() {+ List<HistoryEvent> history = new ArrayList<>();+ runSync(PASS_WITH_JSONATA_OUTPUT, history);++ HistoryEvent failed = history.stream()+ .filter(event -> "ExecutionFailed".equals(event.getType()))+ .findFirst()+ .orElse(null);+ assertNotNull(failed, "history has no ExecutionFailed event: " + typesOf(history));+ assertEquals("States.Runtime", failed.getDetails().get("error"));+ assertEquals(EXPECTED_CAUSE, failed.getDetails().get("cause"));+ }++ @Test+ @Timeout(60)+ void errorInsideAStatePublishesTheTerminalUpdateToTheListener() throws Exception {+ StateMachine stateMachine = stateMachine(PASS_WITH_JSONATA_OUTPUT);+ Execution execution = startedExecution(stateMachine);+ List<HistoryEvent> history = new ArrayList<>();+ CountDownLatch published = new CountDownLatch(1);++ executor.executeAsync(stateMachine, execution, history,+ (updated, events) -> published.countDown());++ assertTrue(published.await(30, TimeUnit.SECONDS),+ "no terminal update was published; execution is still " + execution.getStatus());+ assertEquals("FAILED", execution.getStatus());+ assertEquals(EXPECTED_CAUSE, execution.getCause());+ }++ @Test+ @Timeout(60)+ void errorInsideAParallelBranchFailsTheExecutionWithTheSameCause() {+ Execution execution = runSync("""+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Parallel",+ "Branches": [{+ "StartAt": "Crash",+ "States": {+ "Crash": {"Type": "Pass", "Output": {"v": "{% 42 %}"}, "End": true}+ }+ }],+ "End": true+ }+ }+ }+ """, new ArrayList<>());++ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.Runtime", execution.getError());+ assertEquals(EXPECTED_CAUSE, execution.getCause());+ }++ @Test+ @Timeout(60)+ void errorInsideASerialMapIterationFailsTheExecution() {+ Execution execution = runSync(mapOverItems(1), new ArrayList<>());++ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.Runtime", execution.getError());+ assertEquals(EXPECTED_CAUSE, execution.getCause());+ }++ @Test+ @Timeout(60)+ void errorInsideAConcurrentMapIterationFailsTheExecution() {+ Execution execution = runSync(mapOverItems(4), new ArrayList<>());++ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.Runtime", execution.getError());+ assertEquals(EXPECTED_CAUSE, execution.getCause());+ }++ /**+ * {@code maxConcurrency} 1 keeps the iterations on the calling thread; anything above it hands+ * them to {@code MapIterationScheduler}'s worker pool. Both routes have to terminate.+ */+ private static String mapOverItems(int maxConcurrency) {+ return """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Each",+ "States": {+ "Each": {+ "Type": "Map",+ "Items": "{% [1, 2, 3, 4] %}",+ "MaxConcurrency": __MAX_CONCURRENCY__,+ "ItemProcessor": {+ "StartAt": "Crash",+ "States": {+ "Crash": {"Type": "Pass", "Output": {"v": "{% 42 %}"}, "End": true}+ }+ },+ "End": true+ }+ }+ }+ """.replace("__MAX_CONCURRENCY__", String.valueOf(maxConcurrency));+ }++ private static String typesOf(List<HistoryEvent> history) {+ return history.stream().map(HistoryEvent::getType).toList().toString();+ }++ private Execution runSync(String definition, List<HistoryEvent> history) {+ StateMachine stateMachine = stateMachine(definition);+ Execution execution = startedExecution(stateMachine);+ executor.executeSync(stateMachine, execution, history, (updated, events) -> {+ });+ return execution;+ }++ private static StateMachine stateMachine(String definition) {+ StateMachine stateMachine = new StateMachine();+ stateMachine.setName("error-termination-test");+ stateMachine.setStateMachineArn(+ "arn:aws:states:%s:%s:stateMachine:error-termination-test".formatted(REGION, ACCOUNT));+ stateMachine.setRoleArn("arn:aws:iam::%s:role/test-role".formatted(ACCOUNT));+ stateMachine.setDefinition(definition);+ return stateMachine;+ }++ /** Mirrors {@code StepFunctionsService#startExecution}: the execution is RUNNING when it starts. */+ private static Execution startedExecution(StateMachine stateMachine) {+ Execution execution = new Execution();+ execution.setName("error-termination-execution");+ execution.setExecutionArn(+ "arn:aws:states:%s:%s:execution:error-termination-test:error-termination-execution"+ .formatted(REGION, ACCOUNT));+ execution.setStateMachineArn(stateMachine.getStateMachineArn());+ execution.setInput("{}");+ execution.setStatus("RUNNING");+ return execution;+ }+}