Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(stepfunctions): end the execution when a state throws an Error (#2693)

Facundo Paredesacf88794ca4e
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index 4589329543..2e0ab71ebf 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -217,7 +217,16 @@ public class AslExecutor {
MockedTestCase mockedTestCase,
BiConsumer<Execution, List<HistoryEvent>> onUpdate) {
registerMocks(exec, mockedTestCase);
- executor.submit(() -> runUnderExecutionAccount(sm, () -> doExecute(sm, exec, history, onUpdate)));
+ executor.submit(() -> {
+ try {
+ runUnderExecutionAccount(sm, () -> doExecute(sm, exec, history, onUpdate));
+ } catch (RuntimeException | Error e) {
+ // submit() parks whatever the task throws in a Future nobody reads, so without this
+ // the worker dies silently. doExecute has already published the terminal status by
+ // now; this is the only place the stack trace of what killed it reaches the log.
+ LOG.errorv(e, "ASL execution worker failed for {0}", exec.getExecutionArn());
+ }
+ });
}
/**
@@ -313,8 +322,10 @@ public class AslExecutor {
private void doExecute(StateMachine sm, Execution exec, List<HistoryEvent> history,
BiConsumer<Execution, List<HistoryEvent>> onUpdate) {
+ // Declared outside the try so the terminal handlers below can keep numbering the history
+ // where the execution left it.
+ AtomicLong eventId = new AtomicLong(history.size());
try {
- AtomicLong eventId = new AtomicLong(history.size());
JsonNode definition = objectMapper.readTree(sm.getDefinition());
JsonNode states = definition.path("States");
String startAt = definition.path("StartAt").asText();
@@ -364,17 +375,6 @@ public class AslExecutor {
failExecution(exec, history, eventId, e);
onUpdate.accept(exec, history);
return;
- } catch (Exception e) {
- String runtimeError = "States.Runtime";
- String runtimeCause = e.getMessage() != null ? e.getMessage() : "Unknown error";
- exec.setError(runtimeError);
- exec.setCause(runtimeCause);
- exec.setStopDate(System.currentTimeMillis() / 1000.0);
- exec.setStatus("FAILED");
- addEvent(history, eventId, "ExecutionFailed", null,
- Map.of("error", runtimeError, "cause", runtimeCause));
- onUpdate.accept(exec, history);
- return;
}
}
@@ -393,11 +393,19 @@ public class AslExecutor {
LOG.warnv("ASL execution failed for {0}: {1}", exec.getExecutionArn(), e.getMessage());
// This path previously set only the status, leaving error and cause null forever on an
// execution DescribeExecution reports as FAILED.
- exec.setError("States.Runtime");
- exec.setCause(e.getMessage() != null ? e.getMessage() : "Unknown error");
- exec.setStopDate(System.currentTimeMillis() / 1000.0);
- exec.setStatus("FAILED");
+ failExecution(exec, history, eventId, "States.Runtime",
+ e.getMessage() != null ? e.getMessage() : "Unknown error");
onUpdate.accept(exec, history);
+ } catch (Error e) {
+ // An Error is not a state failure: it says the runtime itself is broken, and no retry
+ // of the state machine can get past it. Publishing the same terminal FAILED an
+ // exception produces is what keeps DescribeExecution from reporting RUNNING forever,
+ // and the rethrow keeps the Error itself from being swallowed here. The cause carries
+ // toString() rather than getMessage(), because an Error's message is often null and
+ // the type name is the whole diagnostic.
+ failExecution(exec, history, eventId, "States.Runtime", e.toString());
+ onUpdate.accept(exec, history);
+ throw e;
} finally {
activeMocks.remove(exec.getExecutionArn());
}
@@ -1635,13 +1643,18 @@ public class AslExecutor {
throw e;
} catch (ExecutionException e) {
futures.forEach(future -> future.cancel(true));
- // Unwrap so a branch's FailStateException reaches the Parallel state's own
- // Retry and Catch handling instead of surfacing as States.Runtime. An Error
- // cause stays wrapped so the execution-level Exception handlers still
- // publish a terminal FAILED update instead of leaving the execution RUNNING.
+ // Unwrap so a branch's FailStateException reaches the Parallel state's own Retry and
+ // Catch handling instead of surfacing as States.Runtime, and so an Error reaches the
+ // execution-level handler as itself rather than as an ExecutionException wrapper. The
+ // reported cause is the same either way, since ExecutionException.getMessage() is the
+ // cause's toString(), but only the unwrapped Error is rethrown and logged with its
+ // stack trace.
if (e.getCause() instanceof Exception exception) {
throw exception;
}
+ if (e.getCause() instanceof Error error) {
+ throw error;
+ }
throw e;
} catch (Exception | Error e) {
futures.forEach(future -> future.cancel(true));
@@ -3110,14 +3123,23 @@ public class AslExecutor {
}
private void failExecution(Execution exec, List<HistoryEvent> history, AtomicLong eventId, FailStateException e) {
- String failError = e.error != null ? e.error : "States.Runtime";
- String failCause = e.cause != null ? e.cause : "";
- exec.setError(failError);
- exec.setCause(failCause);
+ failExecution(exec, history, eventId, e.error != null ? e.error : "States.Runtime",
+ e.cause != null ? e.cause : "");
+ }
+
+ /**
+ * The single terminal-failure write: every way an execution can fail leaves the same
+ * {@code error}, {@code cause} and {@code ExecutionFailed} event behind, so a client cannot
+ * tell a Fail state from a state that threw from a runtime Error by what it reads back.
+ */
+ private void failExecution(Execution exec, List<HistoryEvent> history, AtomicLong eventId,
+ String error, String cause) {
+ exec.setError(error);
+ exec.setCause(cause);
exec.setStopDate(System.currentTimeMillis() / 1000.0);
exec.setStatus("FAILED");
addEvent(history, eventId, "ExecutionFailed", null,
- Map.of("error", failError, "cause", failCause));
+ Map.of("error", error, "cause", cause));
}
private StateResult handleCatch(JsonNode stateDef, JsonNode input, FailStateException failure,
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorErrorTerminationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorErrorTerminationTest.java
new file mode 100644
index 0000000000..60c055cc83
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorErrorTerminationTest.java
@@ -0,0 +1,256 @@
+package io.github.hectorvent.floci.services.stepfunctions;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbJsonHandler;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;
+import io.github.hectorvent.floci.services.lambda.LambdaExecutorService;
+import io.github.hectorvent.floci.services.lambda.LambdaFunctionStore;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;
+import io.github.hectorvent.floci.services.stepfunctions.model.Execution;
+import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;
+import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;
+import io.quarkus.test.junit.QuarkusTest;
+import io.vertx.mutiny.core.Vertx;
+import jakarta.enterprise.inject.Instance;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Guards that an {@link Error} escaping a state ends the execution instead of leaving it RUNNING
+ * forever, the failure reported in issue #2666.
+ *
+ * <p>The trigger described there is native-image only: the JSONata {@code ??} and {@code ?:}
+ * operators clone their parser through Java serialization, which a GraalVM image answers with an
+ * {@code UnsupportedFeatureError}. The defect underneath it is not native-specific, so these tests
+ * inject the same shape on the JVM: a {@link JsonataEvaluator} whose template resolution throws a
+ * {@link LinkageError}, which is what evaluating {@code Output} does inside the image.
+ *
+ * <p>Every case is bounded by a timeout, because the symptom being guarded is an execution that
+ * never terminates.
+ */
+@QuarkusTest
+class AslExecutorErrorTerminationTest {
+
+ private static final String REGION = "us-east-2";
+ private static final String ACCOUNT = "000000000000";
+ private static final String ERROR_MESSAGE = "parser clone is unavailable in this image";
+ private static final String EXPECTED_CAUSE = "java.lang.LinkageError: " + ERROR_MESSAGE;
+
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private AslExecutor executor;
+
+ @Inject
+ Vertx vertx;
+
+ /** Throws where the native image throws: resolving a state's JSONata {@code Output}. */
+ private static final class CrashingJsonataEvaluator extends JsonataEvaluator {
+ CrashingJsonataEvaluator(ObjectMapper objectMapper) {
+ super(objectMapper);
+ }
+
+ @Override
+ JsonNode resolveTemplate(JsonNode template, JsonNode statesVar, JsonNode variables) {
+ throw new LinkageError(ERROR_MESSAGE);
+ }
+ }
+
+ @BeforeEach
+ void setUp() {
+ executor = new AslExecutor(
+ mock(LambdaExecutorService.class),
+ mock(LambdaFunctionStore.class),
+ mock(DynamoDbService.class),
+ mock(DynamoDbJsonHandler.class),
+ mock(SqsJsonHandler.class),
+ mock(io.github.hectorvent.floci.services.cloudformation.CloudFormationQueryHandler.class),
+ mock(io.github.hectorvent.floci.services.ec2.Ec2Service.class),
+ mock(S3Service.class),
+ mock(io.github.hectorvent.floci.services.ecs.EcsService.class),
+ mock(io.github.hectorvent.floci.services.ecs.EcsJsonHandler.class),
+ objectMapper,
+ new CrashingJsonataEvaluator(objectMapper),
+ mock(Instance.class), mock(EmulatorConfig.class), vertx);
+ }
+
+ private static final String PASS_WITH_JSONATA_OUTPUT = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Crash",
+ "States": {
+ "Crash": {
+ "Type": "Pass",
+ "Output": {"v": "{% 42 %}"},
+ "End": true
+ }
+ }
+ }
+ """;
+
+ @Test
+ @Timeout(60)
+ void errorInsideAStateFailsTheExecutionWithTheErrorAsCause() {
+ List<HistoryEvent> history = new ArrayList<>();
+ Execution execution = runSync(PASS_WITH_JSONATA_OUTPUT, history);
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.Runtime", execution.getError());
+ assertEquals(EXPECTED_CAUSE, execution.getCause());
+ assertNotNull(execution.getStopDate());
+ }
+
+ @Test
+ @Timeout(60)
+ void errorInsideAStateRecordsAnExecutionFailedEvent() {
+ List<HistoryEvent> history = new ArrayList<>();
+ runSync(PASS_WITH_JSONATA_OUTPUT, history);
+
+ HistoryEvent failed = history.stream()
+ .filter(event -> "ExecutionFailed".equals(event.getType()))
+ .findFirst()
+ .orElse(null);
+ assertNotNull(failed, "history has no ExecutionFailed event: " + typesOf(history));
+ assertEquals("States.Runtime", failed.getDetails().get("error"));
+ assertEquals(EXPECTED_CAUSE, failed.getDetails().get("cause"));
+ }
+
+ @Test
+ @Timeout(60)
+ void errorInsideAStatePublishesTheTerminalUpdateToTheListener() throws Exception {
+ StateMachine stateMachine = stateMachine(PASS_WITH_JSONATA_OUTPUT);
+ Execution execution = startedExecution(stateMachine);
+ List<HistoryEvent> history = new ArrayList<>();
+ CountDownLatch published = new CountDownLatch(1);
+
+ executor.executeAsync(stateMachine, execution, history,
+ (updated, events) -> published.countDown());
+
+ assertTrue(published.await(30, TimeUnit.SECONDS),
+ "no terminal update was published; execution is still " + execution.getStatus());
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals(EXPECTED_CAUSE, execution.getCause());
+ }
+
+ @Test
+ @Timeout(60)
+ void errorInsideAParallelBranchFailsTheExecutionWithTheSameCause() {
+ Execution execution = runSync("""
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Fan",
+ "States": {
+ "Fan": {
+ "Type": "Parallel",
+ "Branches": [{
+ "StartAt": "Crash",
+ "States": {
+ "Crash": {"Type": "Pass", "Output": {"v": "{% 42 %}"}, "End": true}
+ }
+ }],
+ "End": true
+ }
+ }
+ }
+ """, new ArrayList<>());
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.Runtime", execution.getError());
+ assertEquals(EXPECTED_CAUSE, execution.getCause());
+ }
+
+ @Test
+ @Timeout(60)
+ void errorInsideASerialMapIterationFailsTheExecution() {
+ Execution execution = runSync(mapOverItems(1), new ArrayList<>());
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.Runtime", execution.getError());
+ assertEquals(EXPECTED_CAUSE, execution.getCause());
+ }
+
+ @Test
+ @Timeout(60)
+ void errorInsideAConcurrentMapIterationFailsTheExecution() {
+ Execution execution = runSync(mapOverItems(4), new ArrayList<>());
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.Runtime", execution.getError());
+ assertEquals(EXPECTED_CAUSE, execution.getCause());
+ }
+
+ /**
+ * {@code maxConcurrency} 1 keeps the iterations on the calling thread; anything above it hands
+ * them to {@code MapIterationScheduler}'s worker pool. Both routes have to terminate.
+ */
+ private static String mapOverItems(int maxConcurrency) {
+ return """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Each",
+ "States": {
+ "Each": {
+ "Type": "Map",
+ "Items": "{% [1, 2, 3, 4] %}",
+ "MaxConcurrency": __MAX_CONCURRENCY__,
+ "ItemProcessor": {
+ "StartAt": "Crash",
+ "States": {
+ "Crash": {"Type": "Pass", "Output": {"v": "{% 42 %}"}, "End": true}
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """.replace("__MAX_CONCURRENCY__", String.valueOf(maxConcurrency));
+ }
+
+ private static String typesOf(List<HistoryEvent> history) {
+ return history.stream().map(HistoryEvent::getType).toList().toString();
+ }
+
+ private Execution runSync(String definition, List<HistoryEvent> history) {
+ StateMachine stateMachine = stateMachine(definition);
+ Execution execution = startedExecution(stateMachine);
+ executor.executeSync(stateMachine, execution, history, (updated, events) -> {
+ });
+ return execution;
+ }
+
+ private static StateMachine stateMachine(String definition) {
+ StateMachine stateMachine = new StateMachine();
+ stateMachine.setName("error-termination-test");
+ stateMachine.setStateMachineArn(
+ "arn:aws:states:%s:%s:stateMachine:error-termination-test".formatted(REGION, ACCOUNT));
+ stateMachine.setRoleArn("arn:aws:iam::%s:role/test-role".formatted(ACCOUNT));
+ stateMachine.setDefinition(definition);
+ return stateMachine;
+ }
+
+ /** Mirrors {@code StepFunctionsService#startExecution}: the execution is RUNNING when it starts. */
+ private static Execution startedExecution(StateMachine stateMachine) {
+ Execution execution = new Execution();
+ execution.setName("error-termination-execution");
+ execution.setExecutionArn(
+ "arn:aws:states:%s:%s:execution:error-termination-test:error-termination-execution"
+ .formatted(REGION, ACCOUNT));
+ execution.setStateMachineArn(stateMachine.getStateMachineArn());
+ execution.setInput("{}");
+ execution.setStatus("RUNNING");
+ return execution;
+ }
+}