Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(stepfunctions): bound a .sync Task by its TimeoutSeconds, not by a poll count (#4519)

mitsukado-shinagawaac41ff339a7a
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.md
index 4cdf670564..6466972280 100644
--- a/docs/services/step-functions.md
+++ b/docs/services/step-functions.md
@@ -230,9 +230,10 @@ One deviation. Every pause is capped at `floci.services.stepfunctions.max-wait-s
## Timeouts
ASL carries two `TimeoutSeconds` fields and Floci enforces both, in the two terminal shapes
-AWS uses. The state machine's own field bounds every state; a `Task`'s own field only bounds
-one that waits for a task token — an activity, or a `.waitForTaskToken` integration. A Lambda
-or other SDK task that returns directly is not bound by it.
+AWS uses. The state machine's own field bounds every state; a `Task`'s own field bounds one that
+waits: for a task token (an activity, or a `.waitForTaskToken` integration) or for a job to end
+(`ecs:runTask.sync`, `states:startExecution.sync` and `.sync:2`). A Lambda or other SDK task that
+returns directly is not bound by it.
The state machine's own `TimeoutSeconds` is the whole execution's budget. It is checked before
every state and inside a `Wait`, so a `Wait` longer than what is left is cut rather than slept
@@ -254,6 +255,16 @@ reads `FAILED` with that same error. A `Catch` on a heartbeat expiry matches und
failure reports one. A `Task` that declares no `TimeoutSeconds` waits 300 seconds, where AWS
waits a year.
+A `Task` that waits for a `.sync` job runs under the same two clocks and no other: the execution's
+budget ends it `TIMED_OUT`, its own `TimeoutSeconds` ends it `FAILED` with a `TaskTimedOut` event
+carrying `States.Timeout` and no cause, and a job that takes longer than either simply runs until
+the earlier clock fires. The default of 300 seconds applies here too when the state declares none.
+When the `Task`'s own clock fires, the job it was waiting on is stopped the way AWS stops it: the
+ECS task reads `stopCode: UserInitiated`, the child execution reads `ABORTED` with no error, and
+both carry the cause `The Task state in AWS Step Functions execution [<arn>] which was managing
+this resource was aborted`. A `StopExecution` that lands while the state waits ends the wait and
+stops the job with the same cause. When the execution's budget fires instead, the job is left running.
+
One deviation. AWS starts the `TimeoutSeconds` clock when a worker picks the task up, the instant
it emits `ActivityStarted`. Floci emits `ActivityStarted` at schedule time, so both clocks start
when the task is scheduled.
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index c1ba77ceca..0107db00d2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -153,9 +153,9 @@ public class AslExecutor {
}
private static final Logger LOG = Logger.getLogger(AslExecutor.class);
- // How long a Task waits for its token when the state declares no TimeoutSeconds. AWS lets it
- // run for a year; the emulator would rather free the worker thread.
- private static final int DEFAULT_TASK_TOKEN_TIMEOUT_SECONDS = 300;
+ // How long a Task waits for its token, or for a .sync job to end, when the state declares no
+ // TimeoutSeconds. AWS lets it run for a year; the emulator would rather free the worker thread.
+ private static final int DEFAULT_TASK_TIMEOUT_SECONDS = 300;
/**
* AWS ends an execution once its history reaches this many events. The count is neither reset
@@ -173,9 +173,9 @@ public class AslExecutor {
private static final int INLINE_MAP_MAX_CONCURRENCY = 40;
private static final int DISTRIBUTED_MAP_MAX_CONCURRENCY = 10_000;
- // ecs:runTask.sync polling — wait up to ~60s for the task to reach STOPPED.
- private static final int ECS_SYNC_POLL_ATTEMPTS = 600;
- private static final long ECS_SYNC_POLL_INTERVAL_MS = 100;
+ // How often a .sync integration (ecs:runTask, states:startExecution) re-reads the job it waits
+ // on. The wait itself is bounded by the Task's TimeoutSeconds and the execution's budget.
+ private static final long SYNC_POLL_INTERVAL_MS = 100;
// AWS caps the string input of States.Base64Encode/Base64Decode/Hash at 10,000 characters
// (measured here in Unicode code points).
@@ -532,6 +532,9 @@ public class AslExecutor {
} catch (ExecutionTimedOutException e) {
timeOutExecution(exec, chain);
onUpdate.accept(exec, history);
+ } catch (ExecutionAbortedException e) {
+ // StopExecution already wrote ABORTED and sealed the history; the worker only stops.
+ onUpdate.accept(exec, history);
} catch (Exception e) {
LOG.warnv("ASL execution failed for {0}: {1}", exec.getExecutionArn(), e.getMessage());
// This path previously set only the status, leaving error and cause null forever on an
@@ -797,6 +800,9 @@ public class AslExecutor {
var tokenFuture = needsToken ? sfnService.get().registerPendingToken(taskToken) : null;
var profile = taskEventProfile(resource, isActivity);
JsonNode taskResult;
+ // Read before the scheduled event is built, so a large input or a slow history callback
+ // does not extend the Task's own TimeoutSeconds.
+ long taskDeadlineNanos = taskDeadlineNanos(stateDef);
try {
addTaskScheduledEvent(chain, profile, stateDef, effectiveInput, sm);
addTaskStartedEvent(chain, profile);
@@ -804,9 +810,12 @@ public class AslExecutor {
taskResult = mockedInvocation != null
? mockedTaskResult(mockedInvocation.steps(), stateName, mockedInvocation.responseIndex())
: invokeResource(effectiveResource, effectiveInput, sm, taskToken,
- executionDeadlineNanos, jsonata ? null : stateDef.path("Parameters"));
+ context.path("Execution").path("Id").asText(null),
+ executionDeadlineNanos, taskDeadlineNanos,
+ jsonata ? null : stateDef.path("Parameters"));
if (tokenFuture != null) {
- taskResult = awaitToken(tokenFuture, stateDef, taskToken, executionDeadlineNanos);
+ taskResult = awaitToken(tokenFuture, stateDef, taskToken, executionDeadlineNanos,
+ taskDeadlineNanos);
}
} catch (ExecutionTimedOutException e) {
// The state machine's TimeoutSeconds budget ran out while this task was waiting. AWS
@@ -817,6 +826,10 @@ public class AslExecutor {
} catch (TaskTimedOutException e) {
addTaskTimedOutEvent(chain, profile);
throw e;
+ } catch (ExecutionAbortedException e) {
+ // StopExecution ended the execution while this task waited on its job. The history
+ // was sealed by the abort; nothing more is written about the state.
+ throw e;
} catch (InterruptedException e) {
// The task of a branch that was cut. AWS records nothing for it.
throw e;
@@ -900,13 +913,8 @@ public class AslExecutor {
* schedule time, so there is no later instant to anchor on here.
*/
private JsonNode awaitToken(CompletableFuture<JsonNode> future, JsonNode stateDef, String taskToken,
- long executionDeadlineNanos) throws Exception {
- int timeoutSeconds = stateDef.path("TimeoutSeconds").asInt(0);
- if (timeoutSeconds <= 0) {
- timeoutSeconds = DEFAULT_TASK_TOKEN_TIMEOUT_SECONDS;
- }
+ long executionDeadlineNanos, long timeoutDeadlineNanos) throws Exception {
int heartbeatSeconds = stateDef.path("HeartbeatSeconds").asInt(0);
- long timeoutDeadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds);
try {
while (true) {
long wakeAtNanos = Math.min(executionDeadlineNanos, Math.min(timeoutDeadlineNanos,
@@ -945,6 +953,19 @@ public class AslExecutor {
}
}
+ /**
+ * The instant a Task's own {@code TimeoutSeconds} runs out, counted from now: the bound on a
+ * task-token wait and on a {@code .sync} job wait alike. A state that declares none gets the
+ * emulator's default rather than the year AWS allows.
+ */
+ private long taskDeadlineNanos(JsonNode stateDef) {
+ int timeoutSeconds = stateDef.path("TimeoutSeconds").asInt(0);
+ if (timeoutSeconds <= 0) {
+ timeoutSeconds = DEFAULT_TASK_TIMEOUT_SECONDS;
+ }
+ return System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds);
+ }
+
/**
* When the worker's silence becomes too long: its last heartbeat plus the state's
* {@code HeartbeatSeconds}, or never for a state that declares none.
@@ -1041,7 +1062,8 @@ public class AslExecutor {
}
private JsonNode invokeResource(String resource, JsonNode input, StateMachine sm, String taskToken,
- long executionDeadlineNanos, JsonNode rawParameters) throws Exception {
+ String executionArn, long executionDeadlineNanos, long taskDeadlineNanos,
+ JsonNode rawParameters) throws Exception {
// Support Lambda resources: direct ARN or optimized integration
String functionName = null;
String functionRef = null;
@@ -1190,7 +1212,7 @@ public class AslExecutor {
? ".waitForTaskToken"
: integration.suffix();
String region = extractRegionFromArn(sm.getStateMachineArn());
- return invokeEcsRunTask(mode, input, region, executionDeadlineNanos);
+ return invokeEcsRunTask(mode, input, region, executionArn, executionDeadlineNanos, taskDeadlineNanos);
}
// AWS SDK service integrations: Step Functions
@@ -1215,7 +1237,8 @@ public class AslExecutor {
if (integration.isAnySuffix("states", "startExecution")) {
String mode = integration.suffix();
String region = extractRegionFromArn(sm.getStateMachineArn());
- return invokeNestedStateMachine(mode, input, region, executionDeadlineNanos, rawParameters);
+ return invokeNestedStateMachine(mode, input, region, executionArn, executionDeadlineNanos,
+ taskDeadlineNanos, rawParameters);
}
throw new FailStateException("States.TaskFailed", "Unsupported resource: " + resource);
@@ -1632,8 +1655,9 @@ public class AslExecutor {
return envelope;
}
- private JsonNode invokeNestedStateMachine(String mode, JsonNode input, String region,
- long executionDeadlineNanos, JsonNode rawParameters) throws Exception {
+ private JsonNode invokeNestedStateMachine(String mode, JsonNode input, String region, String executionArn,
+ long executionDeadlineNanos, long taskDeadlineNanos,
+ JsonNode rawParameters) throws Exception {
String smArn = input.path("StateMachineArn").asText(null);
if (smArn == null || smArn.isBlank()) {
throw new FailStateException("States.TaskFailed",
@@ -1679,10 +1703,23 @@ public class AslExecutor {
return result;
}
- // .sync or .sync:2 — poll until terminal, or until the parent execution's TimeoutSeconds
- // budget runs out, which ends the parent as TIMED_OUT and leaves the child running.
- for (int i = 0; i < 600; i++) {
- sleepOrTimeOutExecution(TimeUnit.MILLISECONDS.toNanos(100), executionDeadlineNanos);
+ // .sync or .sync:2 polls until terminal, or until one of the two clocks runs out: the parent
+ // execution's TimeoutSeconds budget ends the parent as TIMED_OUT and leaves the child running;
+ // the Task's own TimeoutSeconds fails the state with States.Timeout and aborts the child, the
+ // way AWS does (measured: child ABORTED, no error, the cause below). A StopExecution on the
+ // parent ends the wait the same way, so a stopped execution does not keep a polling worker.
+ while (true) {
+ try {
+ sleepOrTimeOutTask(TimeUnit.MILLISECONDS.toNanos(SYNC_POLL_INTERVAL_MS),
+ executionDeadlineNanos, taskDeadlineNanos);
+ } catch (TaskTimedOutException e) {
+ abortChildExecution(execArn, executionArn);
+ throw e;
+ }
+ if (abortedByCaller(executionArn)) {
+ abortChildExecution(execArn, executionArn);
+ throw new ExecutionAbortedException();
+ }
io.github.hectorvent.floci.services.stepfunctions.model.Execution current =
sfnService.get().describeExecution(execArn);
String status = current.getStatus();
@@ -1717,8 +1754,6 @@ public class AslExecutor {
current.getCause() != null ? current.getCause()
: "Nested execution ended with status: " + status);
}
- throw new FailStateException("States.TaskFailed",
- "Nested execution timed out: " + execArn);
}
/**
@@ -1732,8 +1767,8 @@ public class AslExecutor {
* STOPPED, or ".waitForTaskToken" to launch and let the token future carry the result
* (both ".sync" and ".waitForTaskToken" fail the state on a placement failure).
*/
- private JsonNode invokeEcsRunTask(String mode, JsonNode input, String region,
- long executionDeadlineNanos) throws Exception {
+ private JsonNode invokeEcsRunTask(String mode, JsonNode input, String region, String executionArn,
+ long executionDeadlineNanos, long taskDeadlineNanos) throws Exception {
String taskDefinition = input.path("TaskDefinition").asText(null);
if (taskDefinition == null || taskDefinition.isBlank()) {
throw new FailStateException("States.TaskFailed",
@@ -1802,11 +1837,24 @@ public class AslExecutor {
// .sync — wait until every launched task reaches STOPPED, then surface success or failure.
// All tasks must be polled (not just the first): with Count > 1, a failure in any task must
- // fail the state, otherwise tasks beyond the first would run unmonitored.
+ // fail the state, otherwise tasks beyond the first would run unmonitored. The wait is bounded
+ // by the Task's TimeoutSeconds and by the execution's budget, never by a poll count of its own.
+ // The Task's clock also stops the tasks it launched, the way AWS does (measured: the ECS task
+ // reads stopCode UserInitiated with the cause below), and so does a StopExecution on the
+ // execution, which otherwise would leave this worker polling until a clock ran out.
List<String> taskArns = launched.stream().map(EcsTask::getTaskArn).toList();
- for (int i = 0; i < ECS_SYNC_POLL_ATTEMPTS; i++) {
- sleepOrTimeOutExecution(TimeUnit.MILLISECONDS.toNanos(ECS_SYNC_POLL_INTERVAL_MS),
- executionDeadlineNanos);
+ while (true) {
+ try {
+ sleepOrTimeOutTask(TimeUnit.MILLISECONDS.toNanos(SYNC_POLL_INTERVAL_MS),
+ executionDeadlineNanos, taskDeadlineNanos);
+ } catch (TaskTimedOutException e) {
+ stopEcsTasks(cluster, taskArns, executionArn, region);
+ throw e;
+ }
+ if (abortedByCaller(executionArn)) {
+ stopEcsTasks(cluster, taskArns, executionArn, region);
+ throw new ExecutionAbortedException();
+ }
List<EcsTask> described = ecsService.describeTasks(cluster, taskArns, region);
boolean allStopped = described.size() == taskArns.size()
&& described.stream().allMatch(t -> "STOPPED".equals(t.getLastStatus()));
@@ -1831,8 +1879,38 @@ public class AslExecutor {
}
return arr;
}
- throw new FailStateException("States.Timeout",
- "ecs:runTask.sync timed out waiting for tasks to stop: " + taskArns);
+ }
+
+ /**
+ * The reason AWS writes on a job it stops because the {@code .sync} Task waiting on it ran out
+ * of its {@code TimeoutSeconds}: verbatim, as the ECS task's {@code stoppedReason} and the child
+ * execution's {@code cause}.
+ */
+ private static String syncAbortCause(String executionArn) {
+ return "The Task state in AWS Step Functions execution [" + executionArn
+ + "] which was managing this resource was aborted";
+ }
+
+ /** Best effort, like AWS: a task that is already gone does not change the timeout being reported. */
+ private void stopEcsTasks(String cluster, List<String> taskArns, String executionArn, String region) {
+ for (String taskArn : taskArns) {
+ try {
+ ecsService.stopTask(cluster, taskArn, syncAbortCause(executionArn), region);
+ } catch (RuntimeException e) {
+ LOG.warnv("ecs:runTask.sync timed out but the task {0} could not be stopped: {1}",
+ taskArn, e.getMessage());
+ }
+ }
+ }
+
+ /** Best effort, like AWS: a child that already ended does not change the timeout being reported. */
+ private void abortChildExecution(String childExecutionArn, String executionArn) {
+ try {
+ sfnService.get().stopExecution(childExecutionArn, syncAbortCause(executionArn), null);
+ } catch (RuntimeException e) {
+ LOG.warnv("states:startExecution.sync timed out but the child {0} could not be stopped: {1}",
+ childExecutionArn, e.getMessage());
+ }
}
/** A failure cause if the ECS task did not complete cleanly (non-zero exit or no container ran), or null on success. */
@@ -2363,13 +2441,29 @@ public class AslExecutor {
*/
private void sleepOrTimeOutExecution(long pauseNanos, long executionDeadlineNanos)
throws InterruptedException {
- long remainingNanos = executionDeadlineNanos - System.nanoTime();
+ sleepOrTimeOutTask(pauseNanos, executionDeadlineNanos, Long.MAX_VALUE);
+ }
+
+ /**
+ * One poll interval of a {@code .sync} job wait, under the same two clocks {@link #awaitToken}
+ * parks on. The execution's budget is read first: when it is the clock that ran out, the
+ * execution ends TIMED_OUT and the Task's own timeout never applies; otherwise the Task's
+ * {@code TimeoutSeconds} ends the state as {@code States.Timeout} with no cause.
+ */
+ private void sleepOrTimeOutTask(long pauseNanos, long executionDeadlineNanos, long taskDeadlineNanos)
+ throws InterruptedException {
+ long remainingNanos = Math.min(executionDeadlineNanos, taskDeadlineNanos) - System.nanoTime();
if (pauseNanos < remainingNanos) {
sleeper.sleep(pauseNanos);
return;
}
sleeper.sleep(Math.max(remainingNanos, 0));
- throw new ExecutionTimedOutException();
+ // The execution's budget wins when it is the clock that was parked on, and also when a late
+ // wake finds it spent: the execution ends TIMED_OUT and the Task's own timeout never applies.
+ if (executionDeadlineNanos <= taskDeadlineNanos || System.nanoTime() >= executionDeadlineNanos) {
+ throw new ExecutionTimedOutException();
+ }
+ throw new TaskTimedOutException("States.Timeout");
}
private StateResult executeSucceedState(JsonNode stateDef, JsonNode input, boolean jsonata, JsonNode context,
@@ -5391,6 +5485,24 @@ public class AslExecutor {
}
}
+ /**
+ * The same read for a wait that has only the execution's ARN: a {@code .sync} poll loop. An
+ * execution the store no longer holds is treated as stopped, since nothing is waiting for the
+ * result either way.
+ */
+ private boolean abortedByCaller(String executionArn) {
+ if (executionArn == null) {
+ return false;
+ }
+ try {
+ return abortedByCaller(sfnService.get().describeExecution(executionArn));
+ } catch (AwsException e) {
+ LOG.warnv("Execution {0} vanished while a .sync task was waiting on its job; ending the wait ({1})",
+ executionArn, e.getMessage());
+ return true;
+ }
+ }
+
private StateResult handleCatch(JsonNode stateDef, JsonNode input, FailStateException failure,
boolean jsonata, JsonNode context, ObjectNode variables) throws Exception {
JsonNode catchers = stateDef.path("Catch");
@@ -5493,6 +5605,17 @@ public class AslExecutor {
}
}
+ /**
+ * Thrown when a wait finds that StopExecution has already ended the execution. Not a
+ * {@link FailStateException}: no Catch sees it, no Retry re-runs the state, and nothing is
+ * written, because the abort sealed the history and ABORTED is the status that stands.
+ */
+ static class ExecutionAbortedException extends RuntimeException {
+ ExecutionAbortedException() {
+ super("Execution aborted by StopExecution");
+ }
+ }
+
static class FailStateException extends RuntimeException {
static final String ATTRIBUTION_PREFIX = "An error occurred while executing the state '%s' (entered at the event id #%d). ";
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorEcsRunTaskModeTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorEcsRunTaskModeTest.java
index cc1886da4b..e34445d10a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorEcsRunTaskModeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorEcsRunTaskModeTest.java
@@ -8,6 +8,8 @@ import io.github.hectorvent.floci.services.dynamodb.DynamoDbJsonHandler;
import io.github.hectorvent.floci.services.ecs.EcsJsonHandler;
import io.github.hectorvent.floci.services.ecs.EcsService;
import io.github.hectorvent.floci.services.ecs.container.HostVolumePolicy;
+import io.github.hectorvent.floci.services.ecs.model.Container;
+import io.github.hectorvent.floci.services.ecs.model.EcsTask;
import io.github.hectorvent.floci.services.ecs.model.NetworkConfiguration;
import io.github.hectorvent.floci.services.lambda.LambdaExecutorService;
import io.github.hectorvent.floci.services.lambda.LambdaFunctionStore;
@@ -21,16 +23,22 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
+import java.time.Clock;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -43,7 +51,9 @@ import static org.mockito.Mockito.when;
* <li>{@code .sync} (and {@code .waitForTaskToken}) fail the state on a placement failure,
* using the AWS error name {@code AmazonECS.Unknown};</li>
* <li>an awsvpc {@code NetworkConfiguration} in the parameters is parsed and passed to runTask
- * rather than dropped.</li>
+ * rather than dropped;</li>
+ * <li>a {@code .sync} wait is bounded by the Task's {@code TimeoutSeconds} and by the execution's
+ * budget, and by nothing else: no poll count of its own cuts a long-running task short.</li>
* </ul>
*/
class AslExecutorEcsRunTaskModeTest {
@@ -53,16 +63,28 @@ class AslExecutorEcsRunTaskModeTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private EcsService ecsService;
+ private StepFunctionsService sfnService;
private AslExecutor executor;
+ private List<HistoryEvent> history;
@BeforeEach
void setUp() {
ecsService = mock(EcsService.class);
+ // The .sync wait reads the execution back to notice a StopExecution; RUNNING unless a test says otherwise.
+ sfnService = mock(StepFunctionsService.class);
+ when(sfnService.describeExecution(any())).thenAnswer(invocation -> parent(invocation.getArgument(0), "RUNNING"));
+ executor = newExecutor(TimeUnit.NANOSECONDS::sleep);
+ }
+
+ @SuppressWarnings("unchecked")
+ private AslExecutor newExecutor(AslExecutor.Sleeper sleeper) {
// A real handler so parseNetworkConfiguration / parseContainerOverrides actually run.
EcsJsonHandler ecsJsonHandler = new EcsJsonHandler(ecsService, objectMapper,
new HostVolumePolicy(mock(EmulatorConfig.class, RETURNS_DEEP_STUBS)));
+ Instance<StepFunctionsService> instance = mock(Instance.class);
+ when(instance.get()).thenReturn(sfnService);
- executor = new AslExecutor(
+ return new AslExecutor(
mock(LambdaExecutorService.class),
mock(LambdaFunctionStore.class),
mock(DynamoDbFacade.class),
@@ -78,9 +100,12 @@ class AslExecutorEcsRunTaskModeTest {
mock(io.github.hectorvent.floci.services.scheduler.SchedulerController.class),
objectMapper,
new JsonataEvaluator(objectMapper),
- mock(Instance.class),
+ instance,
mock(EmulatorConfig.class),
null,
+ null,
+ Clock.systemUTC(),
+ sleeper,
null);
}
@@ -144,16 +169,142 @@ class AslExecutorEcsRunTaskModeTest {
assertEquals("ENABLED", passed.getAwsvpcConfiguration().getAssignPublicIp());
}
+ @Test
+ void syncFailsWithStatesTimeoutWhenTheTaskOutlivesTimeoutSeconds() throws Exception {
+ launchOneTask();
+ when(ecsService.describeTasks(any(), any(), any())).thenReturn(List.of(task("RUNNING")));
+
+ Execution execution = runDefinition("""
+ {
+ "StartAt": "RunTask",
+ "States": {
+ "RunTask": { "Type": "Task", "Resource": "arn:aws:states:::ecs:runTask.sync",
+ "TimeoutSeconds": 1, "End": true }
+ }
+ }
+ """, "{\"TaskDefinition\":\"my-task-def\"}");
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.Timeout", execution.getError());
+ assertNull(execution.getCause(), "a timeout carries no cause");
+ HistoryEvent timedOut = history.stream()
+ .filter(event -> "TaskTimedOut".equals(event.getType()))
+ .findFirst().orElseThrow(() -> new AssertionError("no TaskTimedOut event in " + history));
+ assertEquals("ecs", timedOut.getDetails().get("resourceType"));
+ assertEquals("States.Timeout", timedOut.getDetails().get("error"));
+ // AWS stops the task it was waiting on, with this exact stoppedReason (measured).
+ verify(ecsService).stopTask(any(), eq(task("RUNNING").getTaskArn()),
+ eq("The Task state in AWS Step Functions execution [" + execution.getExecutionArn()
+ + "] which was managing this resource was aborted"), eq(REGION));
+ }
+
+ @Test
+ void syncIsNotCappedByAPollCountWhenNoTimeoutSecondsIsDeclared() throws Exception {
+ // The former loop gave up after 600 polls. A task that stops on the 700th read must still
+ // succeed: without a TimeoutSeconds only the emulator's default bound applies, and the
+ // no-op sleeper keeps that from being real time.
+ executor = newExecutor(nanos -> { });
+ launchOneTask();
+ AtomicInteger polls = new AtomicInteger();
+ when(ecsService.describeTasks(any(), any(), any()))
+ .thenAnswer(invocation -> List.of(task(polls.incrementAndGet() < 700 ? "RUNNING" : "STOPPED")));
+
+ Execution execution = run("arn:aws:states:::ecs:runTask.sync",
+ "{\"TaskDefinition\":\"my-task-def\"}");
+
+ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());
+ assertEquals(700, polls.get());
+ assertEquals("STOPPED", objectMapper.readTree(execution.getOutput()).path("LastStatus").asText());
+ }
+
+ @Test
+ void executionBudgetCutsASyncWaitBeforeTheTasksOwnTimeout() throws Exception {
+ launchOneTask();
+ when(ecsService.describeTasks(any(), any(), any())).thenReturn(List.of(task("RUNNING")));
+
+ Execution execution = runDefinition("""
+ {
+ "StartAt": "RunTask",
+ "TimeoutSeconds": 1,
+ "States": {
+ "RunTask": { "Type": "Task", "Resource": "arn:aws:states:::ecs:runTask.sync",
+ "TimeoutSeconds": 60, "End": true }
+ }
+ }
+ """, "{\"TaskDefinition\":\"my-task-def\"}");
+
+ assertEquals("TIMED_OUT", execution.getStatus());
+ assertNull(execution.getError());
+ verify(ecsService, never()).stopTask(any(), any(), any(), any());
+ assertTrue(history.stream().noneMatch(event -> "TaskTimedOut".equals(event.getType())),
+ "the execution's budget writes nothing about the state it cut");
+ assertTrue(history.stream().anyMatch(event -> "ExecutionTimedOut".equals(event.getType())));
+ }
+
+ @Test
+ void stopExecutionEndsTheSyncWaitAndStopsTheTask() throws Exception {
+ executor = newExecutor(nanos -> { });
+ launchOneTask();
+ AtomicInteger parentReads = new AtomicInteger();
+ when(sfnService.describeExecution(any()))
+ .thenAnswer(invocation -> parent(invocation.getArgument(0), parentReads.incrementAndGet() < 3 ? "RUNNING" : "ABORTED"));
+ AtomicInteger polls = new AtomicInteger();
+ when(ecsService.describeTasks(any(), any(), any())).thenAnswer(invocation -> {
+ polls.incrementAndGet();
+ return List.of(task("RUNNING"));
+ });
+
+ Execution execution = run("arn:aws:states:::ecs:runTask.sync",
+ "{\"TaskDefinition\":\"my-task-def\"}");
+
+ assertEquals(2, polls.get(), "polling stops as soon as the execution reads ABORTED");
+ verify(ecsService).stopTask(any(), eq(task("RUNNING").getTaskArn()),
+ eq("The Task state in AWS Step Functions execution [" + execution.getExecutionArn()
+ + "] which was managing this resource was aborted"), eq(REGION));
+ List<String> types = history.stream().map(HistoryEvent::getType).toList();
+ assertTrue(types.stream().noneMatch(type -> type.equals("TaskTimedOut") || type.equals("TaskFailed")
+ || type.equals("ExecutionFailed") || type.equals("ExecutionSucceeded")),
+ "an aborted execution gets no terminal event from the worker: " + types);
+ }
+
+ private static Execution parent(String executionArn, String status) {
+ Execution parent = new Execution();
+ parent.setExecutionArn(executionArn);
+ parent.setStatus(status);
+ return parent;
+ }
+
+ private void launchOneTask() {
+ when(ecsService.runTask(any(), any(), anyInt(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(List.of(task("PENDING")));
+ }
+
+ private static EcsTask task(String lastStatus) {
+ EcsTask task = new EcsTask();
+ task.setTaskArn("arn:aws:ecs:%s:%s:task/default/0123456789abcdef".formatted(REGION, ACCOUNT));
+ task.setClusterArn("arn:aws:ecs:%s:%s:cluster/default".formatted(REGION, ACCOUNT));
+ task.setLastStatus(lastStatus);
+ Container container = new Container();
+ container.setName("runner");
+ if ("STOPPED".equals(lastStatus)) {
+ container.setExitCode(0);
+ }
+ task.setContainers(List.of(container));
+ return task;
+ }
+
private Execution run(String resource, String input) {
- String definition = """
+ return runDefinition("""
{
"StartAt": "RunTask",
"States": {
"RunTask": { "Type": "Task", "Resource": "%s", "End": true }
}
}
- """.formatted(resource);
+ """.formatted(resource), input);
+ }
+ private Execution runDefinition(String definition, String input) {
StateMachine stateMachine = new StateMachine();
stateMachine.setName("ecs-runtask-test");
stateMachine.setStateMachineArn("arn:aws:states:%s:%s:stateMachine:ecs-runtask-test".formatted(REGION, ACCOUNT));
@@ -166,7 +317,7 @@ class AslExecutorEcsRunTaskModeTest {
execution.setStateMachineArn(stateMachine.getStateMachineArn());
execution.setInput(input);
- List<HistoryEvent> history = new ArrayList<>();
+ history = new ArrayList<>();
executor.executeSync(stateMachine, execution, history, (updated, events) -> { });
return execution;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.java
new file mode 100644
index 0000000000..ca840a9738
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.java
@@ -0,0 +1,230 @@
+package io.github.hectorvent.floci.services.stepfunctions;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.cloudformation.CloudFormationQueryHandler;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbFacade;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbJsonHandler;
+import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.services.ecs.EcsJsonHandler;
+import io.github.hectorvent.floci.services.ecs.EcsService;
+import io.github.hectorvent.floci.services.eventbridge.EventBridgeHandler;
+import io.github.hectorvent.floci.services.lambda.LambdaExecutorService;
+import io.github.hectorvent.floci.services.lambda.LambdaFunctionStore;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.scheduler.SchedulerController;
+import io.github.hectorvent.floci.services.scheduler.SchedulerService;
+import io.github.hectorvent.floci.services.sns.SnsJsonHandler;
+import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;
+import io.github.hectorvent.floci.services.stepfunctions.model.Execution;
+import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;
+import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;
+import jakarta.enterprise.inject.Instance;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * A {@code states:startExecution.sync} wait is bounded by the Task's {@code TimeoutSeconds}, the
+ * way AWS bounds it, and no longer by a fixed poll count of its own. The timeout aborts the child,
+ * a {@code Catch} on {@code States.Timeout} takes it, and a StopExecution on the parent ends the
+ * wait and the child alike.
+ */
+class AslExecutorNestedSyncTimeoutTest {
+
+ private static final String REGION = "us-east-2";
+ private static final String ACCOUNT = "000000000000";
+ private static final String CHILD_SM_ARN =
+ "arn:aws:states:%s:%s:stateMachine:child".formatted(REGION, ACCOUNT);
+ private static final String CHILD_ARN =
+ "arn:aws:states:%s:%s:execution:child:run-1".formatted(REGION, ACCOUNT);
+ private static final String PARENT_ARN =
+ "arn:aws:states:%s:%s:execution:parent:parent-run".formatted(REGION, ACCOUNT);
+ private static final String ABORT_CAUSE =
+ "The Task state in AWS Step Functions execution [" + PARENT_ARN
+ + "] which was managing this resource was aborted";
+
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private StepFunctionsService sfnService;
+ private List<HistoryEvent> history;
+
+ @BeforeEach
+ void setUp() {
+ sfnService = mock(StepFunctionsService.class);
+ when(sfnService.startExecution(any(), any(), any(), any())).thenReturn(child("RUNNING"));
+ when(sfnService.describeExecution(PARENT_ARN)).thenReturn(parent("RUNNING"));
+ }
+
+ @Test
+ void syncFailsWithStatesTimeoutWhenTheChildOutlivesTimeoutSeconds() {
+ when(sfnService.describeExecution(CHILD_ARN)).thenReturn(child("RUNNING"));
+
+ Execution execution = run(newExecutor(TimeUnit.NANOSECONDS::sleep), ".sync", 1, false);
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.Timeout", execution.getError());
+ assertNull(execution.getCause());
+ HistoryEvent timedOut = history.stream()
+ .filter(event -> "TaskTimedOut".equals(event.getType()))
+ .findFirst().orElseThrow(() -> new AssertionError("no TaskTimedOut event in " + history));
+ assertEquals("states", timedOut.getDetails().get("resourceType"));
+ // AWS aborts the child it was waiting on: no error, this exact cause (measured).
+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());
+ }
+
+ @Test
+ void sync2TimeoutIsCaughtUnderStatesTimeout() throws Exception {
+ when(sfnService.describeExecution(CHILD_ARN)).thenReturn(child("RUNNING"));
+
+ Execution execution = run(newExecutor(TimeUnit.NANOSECONDS::sleep), ".sync:2", 1, true);
+
+ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());
+ JsonNode output = objectMapper.readTree(execution.getOutput());
+ assertEquals("States.Timeout", output.path("Error").asText());
+ assertTrue(history.stream().anyMatch(event -> "TaskTimedOut".equals(event.getType())));
+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());
+ }
+
+ @Test
+ void syncIsNotCappedByAPollCountWhenNoTimeoutSecondsIsDeclared() throws Exception {
+ AtomicInteger polls = new AtomicInteger();
+ when(sfnService.describeExecution(CHILD_ARN)).thenAnswer(invocation -> {
+ if (polls.incrementAndGet() < 700) {
+ return child("RUNNING");
+ }
+ Execution done = child("SUCCEEDED");
+ done.setOutput("{\"answer\":42}");
+ return done;
+ });
+
+ Execution execution = run(newExecutor(nanos -> { }), ".sync", 0, false);
+
+ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());
+ assertEquals(700, polls.get());
+ assertTrue(objectMapper.readTree(execution.getOutput()).path("output").asText().contains("42"));
+ verify(sfnService, never()).stopExecution(any(), any(), any());
+ }
+
+ @Test
+ void stopExecutionOnTheParentEndsTheWaitAndAbortsTheChild() {
+ AtomicInteger parentReads = new AtomicInteger();
+ when(sfnService.describeExecution(PARENT_ARN))
+ .thenAnswer(invocation -> parent(parentReads.incrementAndGet() < 3 ? "RUNNING" : "ABORTED"));
+ AtomicInteger childPolls = new AtomicInteger();
+ when(sfnService.describeExecution(CHILD_ARN)).thenAnswer(invocation -> {
+ childPolls.incrementAndGet();
+ return child("RUNNING");
+ });
+
+ run(newExecutor(nanos -> { }), ".sync", 0, false);
+
+ assertEquals(2, childPolls.get(), "polling stops as soon as the parent reads ABORTED");
+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());
+ List<String> types = history.stream().map(HistoryEvent::getType).toList();
+ assertTrue(types.stream().noneMatch(type -> type.equals("TaskTimedOut")
+ || type.equals("TaskFailed") || type.startsWith("Execution")
+ && !type.equals("ExecutionStarted")), "an aborted execution gets no terminal event from the worker: " + types);
+ }
+
+ private static Execution child(String status) {
+ Execution current = new Execution();
+ current.setExecutionArn(CHILD_ARN);
+ current.setStateMachineArn(CHILD_SM_ARN);
+ current.setName("run-1");
+ current.setStatus(status);
+ current.setStartDate(1.0);
+ return current;
+ }
+
+ private static Execution parent(String status) {
+ Execution current = new Execution();
+ current.setExecutionArn(PARENT_ARN);
+ current.setStatus(status);
+ return current;
+ }
+
+ @SuppressWarnings("unchecked")
+ private AslExecutor newExecutor(AslExecutor.Sleeper sleeper) {
+ Instance<StepFunctionsService> instance = mock(Instance.class);
+ when(instance.get()).thenReturn(sfnService);
+ return new AslExecutor(
+ mock(LambdaExecutorService.class),
+ mock(LambdaFunctionStore.class),
+ mock(DynamoDbFacade.class),
+ mock(DynamoDbJsonHandler.class),
+ mock(SqsJsonHandler.class), mock(SnsJsonHandler.class),
+ mock(CloudFormationQueryHandler.class),
+ mock(Ec2Service.class),
+ mock(S3Service.class),
+ mock(EcsService.class),
+ mock(EcsJsonHandler.class),
+ mock(EventBridgeHandler.class),
+ mock(SchedulerService.class),
+ mock(SchedulerController.class),
+ objectMapper,
+ new JsonataEvaluator(objectMapper),
+ instance,
+ mock(EmulatorConfig.class),
+ null,
+ null,
+ Clock.systemUTC(),
+ sleeper,
+ null);
+ }
+
+ private Execution run(AslExecutor executor, String mode, int timeoutSeconds, boolean catchTimeout) {
+ String timeout = timeoutSeconds > 0 ? "\"TimeoutSeconds\": %d,".formatted(timeoutSeconds) : "";
+ String catcher = catchTimeout
+ ? "\"Catch\": [{\"ErrorEquals\": [\"States.Timeout\"], \"Next\": \"Recover\"}],"
+ : "";
+ String definition = """
+ {
+ "StartAt": "Child",
+ "States": {
+ "Child": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::states:startExecution%s",
+ %s
+ %s
+ "Parameters": { "StateMachineArn": "%s" },
+ "End": true
+ },
+ "Recover": { "Type": "Pass", "End": true }
+ }
+ }
+ """.formatted(mode, timeout, catcher, CHILD_SM_ARN);
+
+ StateMachine stateMachine = new StateMachine();
+ stateMachine.setName("parent");
+ stateMachine.setStateMachineArn("arn:aws:states:%s:%s:stateMachine:parent".formatted(REGION, ACCOUNT));
+ stateMachine.setRoleArn("arn:aws:iam::%s:role/test-role".formatted(ACCOUNT));
+ stateMachine.setDefinition(definition);
+
+ Execution execution = new Execution();
+ execution.setName("parent-run");
+ execution.setExecutionArn(PARENT_ARN);
+ execution.setStateMachineArn(stateMachine.getStateMachineArn());
+ execution.setInput("{}");
+
+ history = new ArrayList<>();
+ executor.executeSync(stateMachine, execution, history, (updated, events) -> { });
+ return execution;
+ }
+}