Light, fluffy, and always free - The AWS Local Emulator alternative
fix(stepfunctions): bound a .sync Task by its TimeoutSeconds, not by a poll count (#4519)
mitsukado-shinagawaac41ff339a7a
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.mdindex 4cdf670564..6466972280 100644--- a/docs/services/step-functions.md+++ b/docs/services/step-functions.md@@ -230,9 +230,10 @@ One deviation. Every pause is capped at `floci.services.stepfunctions.max-wait-s## TimeoutsASL carries two `TimeoutSeconds` fields and Floci enforces both, in the two terminal shapes-AWS uses. The state machine's own field bounds every state; a `Task`'s own field only bounds-one that waits for a task token — an activity, or a `.waitForTaskToken` integration. A Lambda-or other SDK task that returns directly is not bound by it.+AWS uses. The state machine's own field bounds every state; a `Task`'s own field bounds one that+waits: for a task token (an activity, or a `.waitForTaskToken` integration) or for a job to end+(`ecs:runTask.sync`, `states:startExecution.sync` and `.sync:2`). A Lambda or other SDK task that+returns directly is not bound by it.The state machine's own `TimeoutSeconds` is the whole execution's budget. It is checked beforeevery state and inside a `Wait`, so a `Wait` longer than what is left is cut rather than slept@@ -254,6 +255,16 @@ reads `FAILED` with that same error. A `Catch` on a heartbeat expiry matches undfailure reports one. A `Task` that declares no `TimeoutSeconds` waits 300 seconds, where AWSwaits a year.+A `Task` that waits for a `.sync` job runs under the same two clocks and no other: the execution's+budget ends it `TIMED_OUT`, its own `TimeoutSeconds` ends it `FAILED` with a `TaskTimedOut` event+carrying `States.Timeout` and no cause, and a job that takes longer than either simply runs until+the earlier clock fires. The default of 300 seconds applies here too when the state declares none.+When the `Task`'s own clock fires, the job it was waiting on is stopped the way AWS stops it: the+ECS task reads `stopCode: UserInitiated`, the child execution reads `ABORTED` with no error, and+both carry the cause `The Task state in AWS Step Functions execution [<arn>] which was managing+this resource was aborted`. A `StopExecution` that lands while the state waits ends the wait and+stops the job with the same cause. When the execution's budget fires instead, the job is left running.+One deviation. AWS starts the `TimeoutSeconds` clock when a worker picks the task up, the instantit emits `ActivityStarted`. Floci emits `ActivityStarted` at schedule time, so both clocks startwhen the task is scheduled.diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.javaindex c1ba77ceca..0107db00d2 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java@@ -153,9 +153,9 @@ public class AslExecutor {}private static final Logger LOG = Logger.getLogger(AslExecutor.class);- // How long a Task waits for its token when the state declares no TimeoutSeconds. AWS lets it- // run for a year; the emulator would rather free the worker thread.- private static final int DEFAULT_TASK_TOKEN_TIMEOUT_SECONDS = 300;+ // How long a Task waits for its token, or for a .sync job to end, when the state declares no+ // TimeoutSeconds. AWS lets it run for a year; the emulator would rather free the worker thread.+ private static final int DEFAULT_TASK_TIMEOUT_SECONDS = 300;/*** AWS ends an execution once its history reaches this many events. The count is neither reset@@ -173,9 +173,9 @@ public class AslExecutor {private static final int INLINE_MAP_MAX_CONCURRENCY = 40;private static final int DISTRIBUTED_MAP_MAX_CONCURRENCY = 10_000;- // ecs:runTask.sync polling — wait up to ~60s for the task to reach STOPPED.- private static final int ECS_SYNC_POLL_ATTEMPTS = 600;- private static final long ECS_SYNC_POLL_INTERVAL_MS = 100;+ // How often a .sync integration (ecs:runTask, states:startExecution) re-reads the job it waits+ // on. The wait itself is bounded by the Task's TimeoutSeconds and the execution's budget.+ private static final long SYNC_POLL_INTERVAL_MS = 100;// AWS caps the string input of States.Base64Encode/Base64Decode/Hash at 10,000 characters// (measured here in Unicode code points).@@ -532,6 +532,9 @@ public class AslExecutor {} catch (ExecutionTimedOutException e) {timeOutExecution(exec, chain);onUpdate.accept(exec, history);+ } catch (ExecutionAbortedException e) {+ // StopExecution already wrote ABORTED and sealed the history; the worker only stops.+ onUpdate.accept(exec, history);} catch (Exception e) {LOG.warnv("ASL execution failed for {0}: {1}", exec.getExecutionArn(), e.getMessage());// This path previously set only the status, leaving error and cause null forever on an@@ -797,6 +800,9 @@ public class AslExecutor {var tokenFuture = needsToken ? sfnService.get().registerPendingToken(taskToken) : null;var profile = taskEventProfile(resource, isActivity);JsonNode taskResult;+ // Read before the scheduled event is built, so a large input or a slow history callback+ // does not extend the Task's own TimeoutSeconds.+ long taskDeadlineNanos = taskDeadlineNanos(stateDef);try {addTaskScheduledEvent(chain, profile, stateDef, effectiveInput, sm);addTaskStartedEvent(chain, profile);@@ -804,9 +810,12 @@ public class AslExecutor {taskResult = mockedInvocation != null? mockedTaskResult(mockedInvocation.steps(), stateName, mockedInvocation.responseIndex()): invokeResource(effectiveResource, effectiveInput, sm, taskToken,- executionDeadlineNanos, jsonata ? null : stateDef.path("Parameters"));+ context.path("Execution").path("Id").asText(null),+ executionDeadlineNanos, taskDeadlineNanos,+ jsonata ? null : stateDef.path("Parameters"));if (tokenFuture != null) {- taskResult = awaitToken(tokenFuture, stateDef, taskToken, executionDeadlineNanos);+ taskResult = awaitToken(tokenFuture, stateDef, taskToken, executionDeadlineNanos,+ taskDeadlineNanos);}} catch (ExecutionTimedOutException e) {// The state machine's TimeoutSeconds budget ran out while this task was waiting. AWS@@ -817,6 +826,10 @@ public class AslExecutor {} catch (TaskTimedOutException e) {addTaskTimedOutEvent(chain, profile);throw e;+ } catch (ExecutionAbortedException e) {+ // StopExecution ended the execution while this task waited on its job. The history+ // was sealed by the abort; nothing more is written about the state.+ throw e;} catch (InterruptedException e) {// The task of a branch that was cut. AWS records nothing for it.throw e;@@ -900,13 +913,8 @@ public class AslExecutor {* schedule time, so there is no later instant to anchor on here.*/private JsonNode awaitToken(CompletableFuture<JsonNode> future, JsonNode stateDef, String taskToken,- long executionDeadlineNanos) throws Exception {- int timeoutSeconds = stateDef.path("TimeoutSeconds").asInt(0);- if (timeoutSeconds <= 0) {- timeoutSeconds = DEFAULT_TASK_TOKEN_TIMEOUT_SECONDS;- }+ long executionDeadlineNanos, long timeoutDeadlineNanos) throws Exception {int heartbeatSeconds = stateDef.path("HeartbeatSeconds").asInt(0);- long timeoutDeadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds);try {while (true) {long wakeAtNanos = Math.min(executionDeadlineNanos, Math.min(timeoutDeadlineNanos,@@ -945,6 +953,19 @@ public class AslExecutor {}}+ /**+ * The instant a Task's own {@code TimeoutSeconds} runs out, counted from now: the bound on a+ * task-token wait and on a {@code .sync} job wait alike. A state that declares none gets the+ * emulator's default rather than the year AWS allows.+ */+ private long taskDeadlineNanos(JsonNode stateDef) {+ int timeoutSeconds = stateDef.path("TimeoutSeconds").asInt(0);+ if (timeoutSeconds <= 0) {+ timeoutSeconds = DEFAULT_TASK_TIMEOUT_SECONDS;+ }+ return System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds);+ }+/*** When the worker's silence becomes too long: its last heartbeat plus the state's* {@code HeartbeatSeconds}, or never for a state that declares none.@@ -1041,7 +1062,8 @@ public class AslExecutor {}private JsonNode invokeResource(String resource, JsonNode input, StateMachine sm, String taskToken,- long executionDeadlineNanos, JsonNode rawParameters) throws Exception {+ String executionArn, long executionDeadlineNanos, long taskDeadlineNanos,+ JsonNode rawParameters) throws Exception {// Support Lambda resources: direct ARN or optimized integrationString functionName = null;String functionRef = null;@@ -1190,7 +1212,7 @@ public class AslExecutor {? ".waitForTaskToken": integration.suffix();String region = extractRegionFromArn(sm.getStateMachineArn());- return invokeEcsRunTask(mode, input, region, executionDeadlineNanos);+ return invokeEcsRunTask(mode, input, region, executionArn, executionDeadlineNanos, taskDeadlineNanos);}// AWS SDK service integrations: Step Functions@@ -1215,7 +1237,8 @@ public class AslExecutor {if (integration.isAnySuffix("states", "startExecution")) {String mode = integration.suffix();String region = extractRegionFromArn(sm.getStateMachineArn());- return invokeNestedStateMachine(mode, input, region, executionDeadlineNanos, rawParameters);+ return invokeNestedStateMachine(mode, input, region, executionArn, executionDeadlineNanos,+ taskDeadlineNanos, rawParameters);}throw new FailStateException("States.TaskFailed", "Unsupported resource: " + resource);@@ -1632,8 +1655,9 @@ public class AslExecutor {return envelope;}- private JsonNode invokeNestedStateMachine(String mode, JsonNode input, String region,- long executionDeadlineNanos, JsonNode rawParameters) throws Exception {+ private JsonNode invokeNestedStateMachine(String mode, JsonNode input, String region, String executionArn,+ long executionDeadlineNanos, long taskDeadlineNanos,+ JsonNode rawParameters) throws Exception {String smArn = input.path("StateMachineArn").asText(null);if (smArn == null || smArn.isBlank()) {throw new FailStateException("States.TaskFailed",@@ -1679,10 +1703,23 @@ public class AslExecutor {return result;}- // .sync or .sync:2 — poll until terminal, or until the parent execution's TimeoutSeconds- // budget runs out, which ends the parent as TIMED_OUT and leaves the child running.- for (int i = 0; i < 600; i++) {- sleepOrTimeOutExecution(TimeUnit.MILLISECONDS.toNanos(100), executionDeadlineNanos);+ // .sync or .sync:2 polls until terminal, or until one of the two clocks runs out: the parent+ // execution's TimeoutSeconds budget ends the parent as TIMED_OUT and leaves the child running;+ // the Task's own TimeoutSeconds fails the state with States.Timeout and aborts the child, the+ // way AWS does (measured: child ABORTED, no error, the cause below). A StopExecution on the+ // parent ends the wait the same way, so a stopped execution does not keep a polling worker.+ while (true) {+ try {+ sleepOrTimeOutTask(TimeUnit.MILLISECONDS.toNanos(SYNC_POLL_INTERVAL_MS),+ executionDeadlineNanos, taskDeadlineNanos);+ } catch (TaskTimedOutException e) {+ abortChildExecution(execArn, executionArn);+ throw e;+ }+ if (abortedByCaller(executionArn)) {+ abortChildExecution(execArn, executionArn);+ throw new ExecutionAbortedException();+ }io.github.hectorvent.floci.services.stepfunctions.model.Execution current =sfnService.get().describeExecution(execArn);String status = current.getStatus();@@ -1717,8 +1754,6 @@ public class AslExecutor {current.getCause() != null ? current.getCause(): "Nested execution ended with status: " + status);}- throw new FailStateException("States.TaskFailed",- "Nested execution timed out: " + execArn);}/**@@ -1732,8 +1767,8 @@ public class AslExecutor {* STOPPED, or ".waitForTaskToken" to launch and let the token future carry the result* (both ".sync" and ".waitForTaskToken" fail the state on a placement failure).*/- private JsonNode invokeEcsRunTask(String mode, JsonNode input, String region,- long executionDeadlineNanos) throws Exception {+ private JsonNode invokeEcsRunTask(String mode, JsonNode input, String region, String executionArn,+ long executionDeadlineNanos, long taskDeadlineNanos) throws Exception {String taskDefinition = input.path("TaskDefinition").asText(null);if (taskDefinition == null || taskDefinition.isBlank()) {throw new FailStateException("States.TaskFailed",@@ -1802,11 +1837,24 @@ public class AslExecutor {// .sync — wait until every launched task reaches STOPPED, then surface success or failure.// All tasks must be polled (not just the first): with Count > 1, a failure in any task must- // fail the state, otherwise tasks beyond the first would run unmonitored.+ // fail the state, otherwise tasks beyond the first would run unmonitored. The wait is bounded+ // by the Task's TimeoutSeconds and by the execution's budget, never by a poll count of its own.+ // The Task's clock also stops the tasks it launched, the way AWS does (measured: the ECS task+ // reads stopCode UserInitiated with the cause below), and so does a StopExecution on the+ // execution, which otherwise would leave this worker polling until a clock ran out.List<String> taskArns = launched.stream().map(EcsTask::getTaskArn).toList();- for (int i = 0; i < ECS_SYNC_POLL_ATTEMPTS; i++) {- sleepOrTimeOutExecution(TimeUnit.MILLISECONDS.toNanos(ECS_SYNC_POLL_INTERVAL_MS),- executionDeadlineNanos);+ while (true) {+ try {+ sleepOrTimeOutTask(TimeUnit.MILLISECONDS.toNanos(SYNC_POLL_INTERVAL_MS),+ executionDeadlineNanos, taskDeadlineNanos);+ } catch (TaskTimedOutException e) {+ stopEcsTasks(cluster, taskArns, executionArn, region);+ throw e;+ }+ if (abortedByCaller(executionArn)) {+ stopEcsTasks(cluster, taskArns, executionArn, region);+ throw new ExecutionAbortedException();+ }List<EcsTask> described = ecsService.describeTasks(cluster, taskArns, region);boolean allStopped = described.size() == taskArns.size()&& described.stream().allMatch(t -> "STOPPED".equals(t.getLastStatus()));@@ -1831,8 +1879,38 @@ public class AslExecutor {}return arr;}- throw new FailStateException("States.Timeout",- "ecs:runTask.sync timed out waiting for tasks to stop: " + taskArns);+ }++ /**+ * The reason AWS writes on a job it stops because the {@code .sync} Task waiting on it ran out+ * of its {@code TimeoutSeconds}: verbatim, as the ECS task's {@code stoppedReason} and the child+ * execution's {@code cause}.+ */+ private static String syncAbortCause(String executionArn) {+ return "The Task state in AWS Step Functions execution [" + executionArn+ + "] which was managing this resource was aborted";+ }++ /** Best effort, like AWS: a task that is already gone does not change the timeout being reported. */+ private void stopEcsTasks(String cluster, List<String> taskArns, String executionArn, String region) {+ for (String taskArn : taskArns) {+ try {+ ecsService.stopTask(cluster, taskArn, syncAbortCause(executionArn), region);+ } catch (RuntimeException e) {+ LOG.warnv("ecs:runTask.sync timed out but the task {0} could not be stopped: {1}",+ taskArn, e.getMessage());+ }+ }+ }++ /** Best effort, like AWS: a child that already ended does not change the timeout being reported. */+ private void abortChildExecution(String childExecutionArn, String executionArn) {+ try {+ sfnService.get().stopExecution(childExecutionArn, syncAbortCause(executionArn), null);+ } catch (RuntimeException e) {+ LOG.warnv("states:startExecution.sync timed out but the child {0} could not be stopped: {1}",+ childExecutionArn, e.getMessage());+ }}/** A failure cause if the ECS task did not complete cleanly (non-zero exit or no container ran), or null on success. */@@ -2363,13 +2441,29 @@ public class AslExecutor {*/private void sleepOrTimeOutExecution(long pauseNanos, long executionDeadlineNanos)throws InterruptedException {- long remainingNanos = executionDeadlineNanos - System.nanoTime();+ sleepOrTimeOutTask(pauseNanos, executionDeadlineNanos, Long.MAX_VALUE);+ }++ /**+ * One poll interval of a {@code .sync} job wait, under the same two clocks {@link #awaitToken}+ * parks on. The execution's budget is read first: when it is the clock that ran out, the+ * execution ends TIMED_OUT and the Task's own timeout never applies; otherwise the Task's+ * {@code TimeoutSeconds} ends the state as {@code States.Timeout} with no cause.+ */+ private void sleepOrTimeOutTask(long pauseNanos, long executionDeadlineNanos, long taskDeadlineNanos)+ throws InterruptedException {+ long remainingNanos = Math.min(executionDeadlineNanos, taskDeadlineNanos) - System.nanoTime();if (pauseNanos < remainingNanos) {sleeper.sleep(pauseNanos);return;}sleeper.sleep(Math.max(remainingNanos, 0));- throw new ExecutionTimedOutException();+ // The execution's budget wins when it is the clock that was parked on, and also when a late+ // wake finds it spent: the execution ends TIMED_OUT and the Task's own timeout never applies.+ if (executionDeadlineNanos <= taskDeadlineNanos || System.nanoTime() >= executionDeadlineNanos) {+ throw new ExecutionTimedOutException();+ }+ throw new TaskTimedOutException("States.Timeout");}private StateResult executeSucceedState(JsonNode stateDef, JsonNode input, boolean jsonata, JsonNode context,@@ -5391,6 +5485,24 @@ public class AslExecutor {}}+ /**+ * The same read for a wait that has only the execution's ARN: a {@code .sync} poll loop. An+ * execution the store no longer holds is treated as stopped, since nothing is waiting for the+ * result either way.+ */+ private boolean abortedByCaller(String executionArn) {+ if (executionArn == null) {+ return false;+ }+ try {+ return abortedByCaller(sfnService.get().describeExecution(executionArn));+ } catch (AwsException e) {+ LOG.warnv("Execution {0} vanished while a .sync task was waiting on its job; ending the wait ({1})",+ executionArn, e.getMessage());+ return true;+ }+ }+private StateResult handleCatch(JsonNode stateDef, JsonNode input, FailStateException failure,boolean jsonata, JsonNode context, ObjectNode variables) throws Exception {JsonNode catchers = stateDef.path("Catch");@@ -5493,6 +5605,17 @@ public class AslExecutor {}}+ /**+ * Thrown when a wait finds that StopExecution has already ended the execution. Not a+ * {@link FailStateException}: no Catch sees it, no Retry re-runs the state, and nothing is+ * written, because the abort sealed the history and ABORTED is the status that stands.+ */+ static class ExecutionAbortedException extends RuntimeException {+ ExecutionAbortedException() {+ super("Execution aborted by StopExecution");+ }+ }+static class FailStateException extends RuntimeException {static final String ATTRIBUTION_PREFIX = "An error occurred while executing the state '%s' (entered at the event id #%d). ";diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorEcsRunTaskModeTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorEcsRunTaskModeTest.javaindex cc1886da4b..e34445d10a 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorEcsRunTaskModeTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorEcsRunTaskModeTest.java@@ -8,6 +8,8 @@ import io.github.hectorvent.floci.services.dynamodb.DynamoDbJsonHandler;import io.github.hectorvent.floci.services.ecs.EcsJsonHandler;import io.github.hectorvent.floci.services.ecs.EcsService;import io.github.hectorvent.floci.services.ecs.container.HostVolumePolicy;+import io.github.hectorvent.floci.services.ecs.model.Container;+import io.github.hectorvent.floci.services.ecs.model.EcsTask;import io.github.hectorvent.floci.services.ecs.model.NetworkConfiguration;import io.github.hectorvent.floci.services.lambda.LambdaExecutorService;import io.github.hectorvent.floci.services.lambda.LambdaFunctionStore;@@ -21,16 +23,22 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.mockito.ArgumentCaptor;+import java.time.Clock;import java.util.ArrayList;import java.util.List;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicInteger;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyInt;+import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;@@ -43,7 +51,9 @@ import static org.mockito.Mockito.when;* <li>{@code .sync} (and {@code .waitForTaskToken}) fail the state on a placement failure,* using the AWS error name {@code AmazonECS.Unknown};</li>* <li>an awsvpc {@code NetworkConfiguration} in the parameters is parsed and passed to runTask- * rather than dropped.</li>+ * rather than dropped;</li>+ * <li>a {@code .sync} wait is bounded by the Task's {@code TimeoutSeconds} and by the execution's+ * budget, and by nothing else: no poll count of its own cuts a long-running task short.</li>* </ul>*/class AslExecutorEcsRunTaskModeTest {@@ -53,16 +63,28 @@ class AslExecutorEcsRunTaskModeTest {private final ObjectMapper objectMapper = new ObjectMapper();private EcsService ecsService;+ private StepFunctionsService sfnService;private AslExecutor executor;+ private List<HistoryEvent> history;@BeforeEachvoid setUp() {ecsService = mock(EcsService.class);+ // The .sync wait reads the execution back to notice a StopExecution; RUNNING unless a test says otherwise.+ sfnService = mock(StepFunctionsService.class);+ when(sfnService.describeExecution(any())).thenAnswer(invocation -> parent(invocation.getArgument(0), "RUNNING"));+ executor = newExecutor(TimeUnit.NANOSECONDS::sleep);+ }++ @SuppressWarnings("unchecked")+ private AslExecutor newExecutor(AslExecutor.Sleeper sleeper) {// A real handler so parseNetworkConfiguration / parseContainerOverrides actually run.EcsJsonHandler ecsJsonHandler = new EcsJsonHandler(ecsService, objectMapper,new HostVolumePolicy(mock(EmulatorConfig.class, RETURNS_DEEP_STUBS)));+ Instance<StepFunctionsService> instance = mock(Instance.class);+ when(instance.get()).thenReturn(sfnService);- executor = new AslExecutor(+ return new AslExecutor(mock(LambdaExecutorService.class),mock(LambdaFunctionStore.class),mock(DynamoDbFacade.class),@@ -78,9 +100,12 @@ class AslExecutorEcsRunTaskModeTest {mock(io.github.hectorvent.floci.services.scheduler.SchedulerController.class),objectMapper,new JsonataEvaluator(objectMapper),- mock(Instance.class),+ instance,mock(EmulatorConfig.class),null,+ null,+ Clock.systemUTC(),+ sleeper,null);}@@ -144,16 +169,142 @@ class AslExecutorEcsRunTaskModeTest {assertEquals("ENABLED", passed.getAwsvpcConfiguration().getAssignPublicIp());}+ @Test+ void syncFailsWithStatesTimeoutWhenTheTaskOutlivesTimeoutSeconds() throws Exception {+ launchOneTask();+ when(ecsService.describeTasks(any(), any(), any())).thenReturn(List.of(task("RUNNING")));++ Execution execution = runDefinition("""+ {+ "StartAt": "RunTask",+ "States": {+ "RunTask": { "Type": "Task", "Resource": "arn:aws:states:::ecs:runTask.sync",+ "TimeoutSeconds": 1, "End": true }+ }+ }+ """, "{\"TaskDefinition\":\"my-task-def\"}");++ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.Timeout", execution.getError());+ assertNull(execution.getCause(), "a timeout carries no cause");+ HistoryEvent timedOut = history.stream()+ .filter(event -> "TaskTimedOut".equals(event.getType()))+ .findFirst().orElseThrow(() -> new AssertionError("no TaskTimedOut event in " + history));+ assertEquals("ecs", timedOut.getDetails().get("resourceType"));+ assertEquals("States.Timeout", timedOut.getDetails().get("error"));+ // AWS stops the task it was waiting on, with this exact stoppedReason (measured).+ verify(ecsService).stopTask(any(), eq(task("RUNNING").getTaskArn()),+ eq("The Task state in AWS Step Functions execution [" + execution.getExecutionArn()+ + "] which was managing this resource was aborted"), eq(REGION));+ }++ @Test+ void syncIsNotCappedByAPollCountWhenNoTimeoutSecondsIsDeclared() throws Exception {+ // The former loop gave up after 600 polls. A task that stops on the 700th read must still+ // succeed: without a TimeoutSeconds only the emulator's default bound applies, and the+ // no-op sleeper keeps that from being real time.+ executor = newExecutor(nanos -> { });+ launchOneTask();+ AtomicInteger polls = new AtomicInteger();+ when(ecsService.describeTasks(any(), any(), any()))+ .thenAnswer(invocation -> List.of(task(polls.incrementAndGet() < 700 ? "RUNNING" : "STOPPED")));++ Execution execution = run("arn:aws:states:::ecs:runTask.sync",+ "{\"TaskDefinition\":\"my-task-def\"}");++ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());+ assertEquals(700, polls.get());+ assertEquals("STOPPED", objectMapper.readTree(execution.getOutput()).path("LastStatus").asText());+ }++ @Test+ void executionBudgetCutsASyncWaitBeforeTheTasksOwnTimeout() throws Exception {+ launchOneTask();+ when(ecsService.describeTasks(any(), any(), any())).thenReturn(List.of(task("RUNNING")));++ Execution execution = runDefinition("""+ {+ "StartAt": "RunTask",+ "TimeoutSeconds": 1,+ "States": {+ "RunTask": { "Type": "Task", "Resource": "arn:aws:states:::ecs:runTask.sync",+ "TimeoutSeconds": 60, "End": true }+ }+ }+ """, "{\"TaskDefinition\":\"my-task-def\"}");++ assertEquals("TIMED_OUT", execution.getStatus());+ assertNull(execution.getError());+ verify(ecsService, never()).stopTask(any(), any(), any(), any());+ assertTrue(history.stream().noneMatch(event -> "TaskTimedOut".equals(event.getType())),+ "the execution's budget writes nothing about the state it cut");+ assertTrue(history.stream().anyMatch(event -> "ExecutionTimedOut".equals(event.getType())));+ }++ @Test+ void stopExecutionEndsTheSyncWaitAndStopsTheTask() throws Exception {+ executor = newExecutor(nanos -> { });+ launchOneTask();+ AtomicInteger parentReads = new AtomicInteger();+ when(sfnService.describeExecution(any()))+ .thenAnswer(invocation -> parent(invocation.getArgument(0), parentReads.incrementAndGet() < 3 ? "RUNNING" : "ABORTED"));+ AtomicInteger polls = new AtomicInteger();+ when(ecsService.describeTasks(any(), any(), any())).thenAnswer(invocation -> {+ polls.incrementAndGet();+ return List.of(task("RUNNING"));+ });++ Execution execution = run("arn:aws:states:::ecs:runTask.sync",+ "{\"TaskDefinition\":\"my-task-def\"}");++ assertEquals(2, polls.get(), "polling stops as soon as the execution reads ABORTED");+ verify(ecsService).stopTask(any(), eq(task("RUNNING").getTaskArn()),+ eq("The Task state in AWS Step Functions execution [" + execution.getExecutionArn()+ + "] which was managing this resource was aborted"), eq(REGION));+ List<String> types = history.stream().map(HistoryEvent::getType).toList();+ assertTrue(types.stream().noneMatch(type -> type.equals("TaskTimedOut") || type.equals("TaskFailed")+ || type.equals("ExecutionFailed") || type.equals("ExecutionSucceeded")),+ "an aborted execution gets no terminal event from the worker: " + types);+ }++ private static Execution parent(String executionArn, String status) {+ Execution parent = new Execution();+ parent.setExecutionArn(executionArn);+ parent.setStatus(status);+ return parent;+ }++ private void launchOneTask() {+ when(ecsService.runTask(any(), any(), anyInt(), any(), any(), any(), any(), any(), any()))+ .thenReturn(List.of(task("PENDING")));+ }++ private static EcsTask task(String lastStatus) {+ EcsTask task = new EcsTask();+ task.setTaskArn("arn:aws:ecs:%s:%s:task/default/0123456789abcdef".formatted(REGION, ACCOUNT));+ task.setClusterArn("arn:aws:ecs:%s:%s:cluster/default".formatted(REGION, ACCOUNT));+ task.setLastStatus(lastStatus);+ Container container = new Container();+ container.setName("runner");+ if ("STOPPED".equals(lastStatus)) {+ container.setExitCode(0);+ }+ task.setContainers(List.of(container));+ return task;+ }+private Execution run(String resource, String input) {- String definition = """+ return runDefinition("""{"StartAt": "RunTask","States": {"RunTask": { "Type": "Task", "Resource": "%s", "End": true }}}- """.formatted(resource);+ """.formatted(resource), input);+ }+ private Execution runDefinition(String definition, String input) {StateMachine stateMachine = new StateMachine();stateMachine.setName("ecs-runtask-test");stateMachine.setStateMachineArn("arn:aws:states:%s:%s:stateMachine:ecs-runtask-test".formatted(REGION, ACCOUNT));@@ -166,7 +317,7 @@ class AslExecutorEcsRunTaskModeTest {execution.setStateMachineArn(stateMachine.getStateMachineArn());execution.setInput(input);- List<HistoryEvent> history = new ArrayList<>();+ history = new ArrayList<>();executor.executeSync(stateMachine, execution, history, (updated, events) -> { });return execution;}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.javanew file mode 100644index 0000000000..ca840a9738--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.java@@ -0,0 +1,230 @@+package io.github.hectorvent.floci.services.stepfunctions;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.services.cloudformation.CloudFormationQueryHandler;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbFacade;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbJsonHandler;+import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.github.hectorvent.floci.services.ecs.EcsJsonHandler;+import io.github.hectorvent.floci.services.ecs.EcsService;+import io.github.hectorvent.floci.services.eventbridge.EventBridgeHandler;+import io.github.hectorvent.floci.services.lambda.LambdaExecutorService;+import io.github.hectorvent.floci.services.lambda.LambdaFunctionStore;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.scheduler.SchedulerController;+import io.github.hectorvent.floci.services.scheduler.SchedulerService;+import io.github.hectorvent.floci.services.sns.SnsJsonHandler;+import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;+import io.github.hectorvent.floci.services.stepfunctions.model.Execution;+import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;+import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;+import jakarta.enterprise.inject.Instance;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.time.Clock;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicInteger;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.ArgumentMatchers.isNull;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++/**+ * A {@code states:startExecution.sync} wait is bounded by the Task's {@code TimeoutSeconds}, the+ * way AWS bounds it, and no longer by a fixed poll count of its own. The timeout aborts the child,+ * a {@code Catch} on {@code States.Timeout} takes it, and a StopExecution on the parent ends the+ * wait and the child alike.+ */+class AslExecutorNestedSyncTimeoutTest {++ private static final String REGION = "us-east-2";+ private static final String ACCOUNT = "000000000000";+ private static final String CHILD_SM_ARN =+ "arn:aws:states:%s:%s:stateMachine:child".formatted(REGION, ACCOUNT);+ private static final String CHILD_ARN =+ "arn:aws:states:%s:%s:execution:child:run-1".formatted(REGION, ACCOUNT);+ private static final String PARENT_ARN =+ "arn:aws:states:%s:%s:execution:parent:parent-run".formatted(REGION, ACCOUNT);+ private static final String ABORT_CAUSE =+ "The Task state in AWS Step Functions execution [" + PARENT_ARN+ + "] which was managing this resource was aborted";++ private final ObjectMapper objectMapper = new ObjectMapper();+ private StepFunctionsService sfnService;+ private List<HistoryEvent> history;++ @BeforeEach+ void setUp() {+ sfnService = mock(StepFunctionsService.class);+ when(sfnService.startExecution(any(), any(), any(), any())).thenReturn(child("RUNNING"));+ when(sfnService.describeExecution(PARENT_ARN)).thenReturn(parent("RUNNING"));+ }++ @Test+ void syncFailsWithStatesTimeoutWhenTheChildOutlivesTimeoutSeconds() {+ when(sfnService.describeExecution(CHILD_ARN)).thenReturn(child("RUNNING"));++ Execution execution = run(newExecutor(TimeUnit.NANOSECONDS::sleep), ".sync", 1, false);++ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.Timeout", execution.getError());+ assertNull(execution.getCause());+ HistoryEvent timedOut = history.stream()+ .filter(event -> "TaskTimedOut".equals(event.getType()))+ .findFirst().orElseThrow(() -> new AssertionError("no TaskTimedOut event in " + history));+ assertEquals("states", timedOut.getDetails().get("resourceType"));+ // AWS aborts the child it was waiting on: no error, this exact cause (measured).+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());+ }++ @Test+ void sync2TimeoutIsCaughtUnderStatesTimeout() throws Exception {+ when(sfnService.describeExecution(CHILD_ARN)).thenReturn(child("RUNNING"));++ Execution execution = run(newExecutor(TimeUnit.NANOSECONDS::sleep), ".sync:2", 1, true);++ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());+ JsonNode output = objectMapper.readTree(execution.getOutput());+ assertEquals("States.Timeout", output.path("Error").asText());+ assertTrue(history.stream().anyMatch(event -> "TaskTimedOut".equals(event.getType())));+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());+ }++ @Test+ void syncIsNotCappedByAPollCountWhenNoTimeoutSecondsIsDeclared() throws Exception {+ AtomicInteger polls = new AtomicInteger();+ when(sfnService.describeExecution(CHILD_ARN)).thenAnswer(invocation -> {+ if (polls.incrementAndGet() < 700) {+ return child("RUNNING");+ }+ Execution done = child("SUCCEEDED");+ done.setOutput("{\"answer\":42}");+ return done;+ });++ Execution execution = run(newExecutor(nanos -> { }), ".sync", 0, false);++ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());+ assertEquals(700, polls.get());+ assertTrue(objectMapper.readTree(execution.getOutput()).path("output").asText().contains("42"));+ verify(sfnService, never()).stopExecution(any(), any(), any());+ }++ @Test+ void stopExecutionOnTheParentEndsTheWaitAndAbortsTheChild() {+ AtomicInteger parentReads = new AtomicInteger();+ when(sfnService.describeExecution(PARENT_ARN))+ .thenAnswer(invocation -> parent(parentReads.incrementAndGet() < 3 ? "RUNNING" : "ABORTED"));+ AtomicInteger childPolls = new AtomicInteger();+ when(sfnService.describeExecution(CHILD_ARN)).thenAnswer(invocation -> {+ childPolls.incrementAndGet();+ return child("RUNNING");+ });++ run(newExecutor(nanos -> { }), ".sync", 0, false);++ assertEquals(2, childPolls.get(), "polling stops as soon as the parent reads ABORTED");+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());+ List<String> types = history.stream().map(HistoryEvent::getType).toList();+ assertTrue(types.stream().noneMatch(type -> type.equals("TaskTimedOut")+ || type.equals("TaskFailed") || type.startsWith("Execution")+ && !type.equals("ExecutionStarted")), "an aborted execution gets no terminal event from the worker: " + types);+ }++ private static Execution child(String status) {+ Execution current = new Execution();+ current.setExecutionArn(CHILD_ARN);+ current.setStateMachineArn(CHILD_SM_ARN);+ current.setName("run-1");+ current.setStatus(status);+ current.setStartDate(1.0);+ return current;+ }++ private static Execution parent(String status) {+ Execution current = new Execution();+ current.setExecutionArn(PARENT_ARN);+ current.setStatus(status);+ return current;+ }++ @SuppressWarnings("unchecked")+ private AslExecutor newExecutor(AslExecutor.Sleeper sleeper) {+ Instance<StepFunctionsService> instance = mock(Instance.class);+ when(instance.get()).thenReturn(sfnService);+ return new AslExecutor(+ mock(LambdaExecutorService.class),+ mock(LambdaFunctionStore.class),+ mock(DynamoDbFacade.class),+ mock(DynamoDbJsonHandler.class),+ mock(SqsJsonHandler.class), mock(SnsJsonHandler.class),+ mock(CloudFormationQueryHandler.class),+ mock(Ec2Service.class),+ mock(S3Service.class),+ mock(EcsService.class),+ mock(EcsJsonHandler.class),+ mock(EventBridgeHandler.class),+ mock(SchedulerService.class),+ mock(SchedulerController.class),+ objectMapper,+ new JsonataEvaluator(objectMapper),+ instance,+ mock(EmulatorConfig.class),+ null,+ null,+ Clock.systemUTC(),+ sleeper,+ null);+ }++ private Execution run(AslExecutor executor, String mode, int timeoutSeconds, boolean catchTimeout) {+ String timeout = timeoutSeconds > 0 ? "\"TimeoutSeconds\": %d,".formatted(timeoutSeconds) : "";+ String catcher = catchTimeout+ ? "\"Catch\": [{\"ErrorEquals\": [\"States.Timeout\"], \"Next\": \"Recover\"}],"+ : "";+ String definition = """+ {+ "StartAt": "Child",+ "States": {+ "Child": {+ "Type": "Task",+ "Resource": "arn:aws:states:::states:startExecution%s",+ %s+ %s+ "Parameters": { "StateMachineArn": "%s" },+ "End": true+ },+ "Recover": { "Type": "Pass", "End": true }+ }+ }+ """.formatted(mode, timeout, catcher, CHILD_SM_ARN);++ StateMachine stateMachine = new StateMachine();+ stateMachine.setName("parent");+ stateMachine.setStateMachineArn("arn:aws:states:%s:%s:stateMachine:parent".formatted(REGION, ACCOUNT));+ stateMachine.setRoleArn("arn:aws:iam::%s:role/test-role".formatted(ACCOUNT));+ stateMachine.setDefinition(definition);++ Execution execution = new Execution();+ execution.setName("parent-run");+ execution.setExecutionArn(PARENT_ARN);+ execution.setStateMachineArn(stateMachine.getStateMachineArn());+ execution.setInput("{}");++ history = new ArrayList<>();+ executor.executeSync(stateMachine, execution, history, (updated, events) -> { });+ return execution;+ }+}