Light, fluffy, and always free - The AWS Local Emulator alternative
fix(stepfunctions): bound a running execution the four ways AWS does (#2782)
Facundo Paredes92887696074b
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.mdindex 0f5a366c5c..f06d0e0e6f 100644--- a/docs/services/step-functions.md+++ b/docs/services/step-functions.md@@ -111,6 +111,37 @@ uniformly between zero and the computed delay, as on AWS. One deviation. The delbetween attempts is capped at 30 seconds, the same cap Floci applies to `Wait` states,so emulated runs stay fast.+## Timeouts++ASL carries two `TimeoutSeconds` fields and Floci enforces both, in the two terminal shapes+AWS uses. The state machine's own field bounds every state; a `Task`'s own field only bounds+one that waits for a task token — an activity, or a `.waitForTaskToken` integration. A Lambda+or other SDK task that returns directly is not bound by it.++The state machine's own `TimeoutSeconds` is the whole execution's budget. It is checked before+every state and inside a `Wait`, so a `Wait` longer than what is left is cut rather than slept+out. The execution ends `TIMED_OUT` with `stopDate` set and no `error` and no `cause` at all;+`States.Timeout` is named only in the single `ExecutionTimedOut` event, whose `previousEventId`+is `0`. The state that was cut gets no `*StateExited` event. A `Parallel` or `Map` branch runs+on its own thread and is not cut mid-state: the budget is enforced again as soon as the branch+returns.++A `Task` that waits for a task token — an activity, or a `.waitForTaskToken` integration — has+two independent bounds. `TimeoutSeconds` is the whole wait; `HeartbeatSeconds` is the longest gap+allowed between two `SendTaskHeartbeat` calls, and every heartbeat pushes that gap forward, so a+worker that reports as often as the definition asks runs until `TimeoutSeconds` runs out. Either+clock ends the state the same way: an `ActivityTimedOut` event — `TaskTimedOut` for a+`.waitForTaskToken` integration — carrying `States.Timeout` and no cause, then an execution that+reads `FAILED` with that same error. A `Catch` on a heartbeat expiry matches under+`States.HeartbeatTimeout` and under `States.Timeout` alike. Neither timeout carries a cause:+`DescribeExecution` and the `ExecutionFailed` event both leave the key out, where every other+failure reports one. A `Task` that declares no `TimeoutSeconds` waits 300 seconds, where AWS+waits a year.++One deviation. AWS starts the `TimeoutSeconds` clock when a worker picks the task up, the instant+it emits `ActivityStarted`. Floci emits `ActivityStarted` at schedule time, so both clocks start+when the task is scheduled.+## JSONata nullsAn expression that evaluates to JSON `null` produces a value, not a missing one. It keeps its keydiff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.javaindex d8aad8aedb..997bb5e5a1 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java@@ -108,6 +108,19 @@ public class AslExecutor {private static final Logger LOG = Logger.getLogger(AslExecutor.class);private static final int MAX_WAIT_SECONDS = 30;+ // How long a Task waits for its token when the state declares no TimeoutSeconds. AWS lets it+ // run for a year; the emulator would rather free the worker thread.+ private static final int DEFAULT_TASK_TOKEN_TIMEOUT_SECONDS = 300;++ /**+ * AWS ends an execution once its history reaches this many events. The count is neither reset+ * nor offset: the event that ends the execution is number 25,000 itself, so the last event the+ * state machine produced is 24,999.+ */+ private static final int MAX_HISTORY_EVENTS = 25_000;+ private static final String HISTORY_EVENT_LIMIT_CAUSE =+ "The execution reached the maximum number of history events (" + MAX_HISTORY_EVENTS + ").";+private static final int INLINE_MAP_MAX_CONCURRENCY = 40;private static final int DISTRIBUTED_MAP_MAX_CONCURRENCY = 10_000;@@ -354,9 +367,10 @@ public class AslExecutor {private void doExecute(StateMachine sm, Execution exec, List<HistoryEvent> history,BiConsumer<Execution, List<HistoryEvent>> onUpdate) {- // Declared outside the try so the terminal handlers below can keep numbering the history- // where the execution left it.- AtomicLong eventId = new AtomicLong(history.size());+ // Shared with every Parallel branch and every inline Map iteration of this execution: the+ // 25,000-event limit is the execution's, not the thread's. It starts where the history+ // already is, because ExecutionStarted is an event of this execution too.+ AtomicLong producedEventCount = new AtomicLong(history.size());var firstState = true;try {JsonNode definition = objectMapper.readTree(sm.getDefinition());@@ -364,20 +378,30 @@ public class AslExecutor {String startAt = definition.path("StartAt").asText();String topLevelQueryLanguage = definition.path("QueryLanguage").asText("JSONPath");JsonNode currentInput = parseInput(exec.getInput());+ // The state machine's total budget, computed once so every state measures against the+ // same instant. Long.MAX_VALUE stands for a definition with no TimeoutSeconds.+ int timeoutSeconds = definition.path("TimeoutSeconds").asInt(0);+ long executionDeadlineNanos = timeoutSeconds > 0+ ? System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds)+ : Long.MAX_VALUE;JsonNode execContext = buildContext(exec, sm);// Execution-scoped JSONata variables (the Assign field). Mutated in place as states// assign, so later states observe earlier assignments.ObjectNode variables = objectMapper.createObjectNode();String currentStateName = startAt;- while (currentStateName != null) {+ while (currentStateName != null && !abortedByCaller(exec)) {+ if (System.nanoTime() >= executionDeadlineNanos) {+ throw new ExecutionTimedOutException();+ }JsonNode stateDef = states.path(currentStateName);if (stateDef.isMissingNode()) {throw new RuntimeException("State not found: " + currentStateName);}String type = stateDef.path("Type").asText();- addEvent(history, eventId, stateEnteredEventType(type), firstState ? 0L : eventId.get(),+ publishStateEnteredEvent(history, producedEventCount, stateEnteredEventType(type),+ firstState ? 0L : history.size(),Map.of("name", currentStateName, "input", currentInput.toString(),"inputDetails", Map.of("truncated", false)));firstState = false;@@ -388,8 +412,9 @@ public class AslExecutor {var jsonata = isJsonata(stateDef, topLevelQueryLanguage);try {var stateResult = executeStateWithRetry(currentStateName, type, stateDef, currentInput,- history, eventId, sm, jsonata, topLevelQueryLanguage, execContext, variables);- addEvent(history, eventId, stateExitedEventType(type),+ history, producedEventCount, sm, jsonata, topLevelQueryLanguage, execContext,+ variables, executionDeadlineNanos);+ publishEvent(history, producedEventCount, stateExitedEventType(type),Map.of("name", currentStateName, "output", stateResult.output().toString(),"outputDetails", Map.of("truncated", false)));@@ -411,35 +436,36 @@ public class AslExecutor {failure = catchClauseFailure;}if (caught != null) {- addEvent(history, eventId, stateExitedEventType(type),+ publishEvent(history, producedEventCount, stateExitedEventType(type),Map.of("name", currentStateName, "output", caught.output().toString(),"outputDetails", Map.of("truncated", false)));currentInput = caught.output();currentStateName = caught.nextState();continue;}- failExecution(exec, history, eventId, failure);+ failExecution(exec, history, failure);onUpdate.accept(exec, history);return;}}- // Status is the publication point, so it is set last. describeExecution hands out this- // same live Execution, so a client polling for SUCCEEDED between setStatus and setOutput- // would read a terminal execution with a null output, which real Step Functions never- // returns. The same ordering applies to every terminal path below.- exec.setOutput(currentInput.toString());- exec.setStopDate(System.currentTimeMillis() / 1000.0);- exec.setStatus("SUCCEEDED");- addEvent(history, eventId, "ExecutionSucceeded",- Map.of("output", currentInput.toString(), "outputDetails", Map.of("truncated", false)));+ succeedExecution(exec, history, currentInput);onUpdate.accept(exec, history);+ } catch (ExecutionTimedOutException e) {+ timeOutExecution(exec, history);+ onUpdate.accept(exec, history);+ } catch (FailStateException e) {+ // A state's own failure is handled inside the loop, where its Catch clauses apply. What+ // reaches here is a failure raised while recording a state's entered event, outside the+ // per-state try: the execution hit the history-event limit.+ failExecution(exec, history, e);+ onUpdate.accept(exec, history);} catch (Exception e) {LOG.warnv("ASL execution failed for {0}: {1}", exec.getExecutionArn(), e.getMessage());// This path previously set only the status, leaving error and cause null forever on an// execution DescribeExecution reports as FAILED.- failExecution(exec, history, eventId, "States.Runtime",+ failExecution(exec, history, "States.Runtime",e.getMessage() != null ? e.getMessage() : "Unknown error");onUpdate.accept(exec, history);} catch (Error e) {@@ -449,7 +475,7 @@ public class AslExecutor {// and the rethrow keeps the Error itself from being swallowed here. The cause carries// toString() rather than getMessage(), because an Error's message is often null and// the type name is the whole diagnostic.- failExecution(exec, history, eventId, "States.Runtime", e.toString());+ failExecution(exec, history, "States.Runtime", e.toString());onUpdate.accept(exec, history);throw e;} finally {@@ -470,16 +496,17 @@ public class AslExecutor {* caller's Catch handling, preserving Retry-before-Catch order.*/private StateResult executeStateWithRetry(String name, String type, JsonNode stateDef, JsonNode input,- List<HistoryEvent> history, AtomicLong eventId, StateMachine sm,- boolean jsonata, String topLevelQueryLanguage, JsonNode context,- ObjectNode variables) throws Exception {+ List<HistoryEvent> history, AtomicLong producedEventCount,+ StateMachine sm, boolean jsonata, String topLevelQueryLanguage,+ JsonNode context, ObjectNode variables,+ long executionDeadlineNanos) throws Exception {var retriers = stateDef.path("Retry");var attemptsPerRetrier = new HashMap<Integer, Integer>();var attempt = 0;while (true) {try {- return executeState(name, type, stateDef, input, history, eventId, sm, jsonata,- topLevelQueryLanguage, context, variables, attempt);+ return executeState(name, type, stateDef, input, history, producedEventCount, sm, jsonata,+ topLevelQueryLanguage, context, variables, attempt, executionDeadlineNanos);} catch (FailStateException e) {var retrierIndex = findMatchingRetrier(retriers, e);if (retrierIndex < 0) {@@ -490,7 +517,7 @@ public class AslExecutor {if (attemptsUsed > retrier.path("MaxAttempts").asInt(3)) {throw e;}- sleepBeforeRetry(retrier, attemptsUsed);+ sleepBeforeRetry(retrier, attemptsUsed, executionDeadlineNanos);attempt++;updateRetryCount(context, attempt);}@@ -501,20 +528,25 @@ public class AslExecutor {if (!retriers.isArray()) {return -1;}- var error = failure.error != null ? failure.error : "States.Runtime";for (var i = 0; i < retriers.size(); i++) {- if (catchMatches(retriers.get(i), error)) {+ if (catchMatches(retriers.get(i), failure)) {return i;}}return -1;}- private void sleepBeforeRetry(JsonNode retrier, int attemptsUsed) throws InterruptedException {+ /**+ * Backs off before the next attempt. The backoff is a pause inside the state, so a retrier+ * whose interval outlasts the state machine's {@code TimeoutSeconds} budget ends the execution+ * where the budget runs out rather than attempting again past it, which is what AWS does. The+ * deadline is read even when the delay is zero, so a state that already spent the budget stops+ * instead of retrying instantly.+ */+ private void sleepBeforeRetry(JsonNode retrier, int attemptsUsed, long executionDeadlineNanos)+ throws InterruptedException {var delaySeconds = retryDelaySeconds(retrier, attemptsUsed, ThreadLocalRandom.current().nextDouble());- if (delaySeconds > 0) {- Thread.sleep((long) (delaySeconds * 1000));- }+ sleepOrTimeOutExecution((long) (delaySeconds * 1_000_000_000L), executionDeadlineNanos);}/**@@ -542,19 +574,22 @@ public class AslExecutor {}private StateResult executeState(String name, String type, JsonNode stateDef, JsonNode input,- List<HistoryEvent> history, AtomicLong eventId, StateMachine sm,- boolean jsonata, String topLevelQueryLanguage, JsonNode context,- ObjectNode variables, int attempt) throws Exception {+ List<HistoryEvent> history, AtomicLong producedEventCount,+ StateMachine sm, boolean jsonata, String topLevelQueryLanguage,+ JsonNode context, ObjectNode variables, int attempt,+ long executionDeadlineNanos) throws Exception {return switch (type) {case "Pass" -> executePassState(stateDef, input, jsonata, context, variables);- case "Task" -> executeTaskState(name, stateDef, input, history, eventId, sm, jsonata, context,- variables, attempt);+ case "Task" -> executeTaskState(name, stateDef, input, history, producedEventCount, sm,+ jsonata, context, variables, attempt, executionDeadlineNanos);case "Choice" -> executeChoiceState(stateDef, input, jsonata, context, variables);- case "Wait" -> executeWaitState(stateDef, input, jsonata, context, variables);+ case "Wait" -> executeWaitState(stateDef, input, jsonata, context, variables, executionDeadlineNanos);case "Succeed" -> executeSucceedState(stateDef, input, jsonata, context, variables);case "Fail" -> executeFail(stateDef, input, jsonata, context, variables);- case "Parallel" -> executeParallelState(name, stateDef, input, sm, jsonata, topLevelQueryLanguage, context, variables);- case "Map" -> executeMapState(name, stateDef, input, sm, jsonata, topLevelQueryLanguage, context, variables);+ case "Parallel" -> executeParallelState(name, stateDef, input, producedEventCount, sm, jsonata,+ topLevelQueryLanguage, context, variables, executionDeadlineNanos);+ case "Map" -> executeMapState(name, stateDef, input, producedEventCount, sm, jsonata,+ topLevelQueryLanguage, context, variables, executionDeadlineNanos);default -> new StateResult(input, stateDef.path("Next").asText(null));};}@@ -585,9 +620,10 @@ public class AslExecutor {}private StateResult executeTaskState(String stateName, JsonNode stateDef, JsonNode input,- List<HistoryEvent> history, AtomicLong eventId, StateMachine sm,- boolean jsonata, JsonNode context, ObjectNode variables,- int attempt) throws Exception {+ List<HistoryEvent> history, AtomicLong producedEventCount,+ StateMachine sm, boolean jsonata, JsonNode context,+ ObjectNode variables, int attempt,+ long executionDeadlineNanos) throws Exception {var resource = stateDef.path("Resource").asText();var isWaitForToken = resource.endsWith(".waitForTaskToken");var effectiveResource = isWaitForToken@@ -623,25 +659,44 @@ public class AslExecutor {}var profile = taskEventProfile(resource, isActivity);- addTaskScheduledEvent(history, eventId, profile, stateDef, effectiveInput, sm);- addTaskStartedEvent(history, eventId, profile);-JsonNode taskResult;try {- taskResult = mockedSteps != null- ? mockedTaskResult(mockedSteps, stateName, attempt)- : invokeResource(effectiveResource, effectiveInput, sm, taskToken);- if (tokenFuture != null) {- taskResult = awaitToken(tokenFuture, stateDef);+ addTaskScheduledEvent(history, producedEventCount, profile, stateDef, effectiveInput, sm);+ addTaskStartedEvent(history, producedEventCount, profile);+ try {+ taskResult = mockedSteps != null+ ? mockedTaskResult(mockedSteps, stateName, attempt)+ : invokeResource(effectiveResource, effectiveInput, sm, taskToken, executionDeadlineNanos);+ if (tokenFuture != null) {+ taskResult = awaitToken(tokenFuture, stateDef, taskToken, executionDeadlineNanos);+ }+ } catch (ExecutionTimedOutException e) {+ // The state machine's TimeoutSeconds budget ran out while this task was waiting. AWS+ // ends the execution there and writes nothing about the state it cut: the history of+ // a task still waiting on its token is ExecutionStarted, TaskStateEntered,+ // ActivityScheduled, ExecutionTimedOut, with no TaskFailed and no TaskTimedOut.+ throw e;+ } catch (TaskTimedOutException e) {+ addTaskTimedOutEvent(history, producedEventCount, profile);+ throw e;+ } catch (Exception e) {+ var failure = e instanceof FailStateException f ? f : null;+ addTaskFailedEvent(history, producedEventCount, profile,+ failure != null && failure.error != null ? failure.error : "States.Runtime",+ failure != null ? failure.cause : e.getMessage());+ throw e;}} catch (Exception e) {- var failure = e instanceof FailStateException f ? f : null;- addTaskFailedEvent(history, eventId, profile,- failure != null && failure.error != null ? failure.error : "States.Runtime",- failure != null ? failure.cause : e.getMessage());+ // A token registered above is normally discarded by awaitToken's own finally. Anything+ // that throws before awaitToken runs — the scheduled/started events themselves, or the+ // resource invocation — would otherwise leave it pending forever; the discard here is a+ // no-op once awaitToken already ran it.+ if (needsToken) {+ sfnService.get().discardPendingToken(taskToken);+ }throw e;}- addTaskSucceededEvent(history, eventId, profile, taskResult);+ addTaskSucceededEvent(history, producedEventCount, profile, taskResult);if (jsonata) {JsonNode output = applyJsonataOutput(stateDef, input, taskResult, context, variables);@@ -685,17 +740,51 @@ public class AslExecutor {"No mocked response defined for attempt " + attempt + " of state '" + stateName + "'");}- private JsonNode awaitToken(CompletableFuture<JsonNode> future, JsonNode stateDef) throws Exception {- int timeout = stateDef.path("HeartbeatSeconds").asInt(0);- if (timeout <= 0) {- timeout = 300;+ /**+ * Waits for the worker to answer the task token under the two independent bounds AWS enforces:+ * {@code TimeoutSeconds} is the whole wait, and {@code HeartbeatSeconds} is the longest gap+ * allowed between two SendTaskHeartbeat calls, each of which pushes that gap forward. Either+ * clock ends the state as a {@link TaskTimedOutException}: the error is {@code States.Timeout}+ * and there is no cause.+ *+ * <p>Both clocks start when the task is scheduled. AWS starts TimeoutSeconds when a worker+ * picks the task up, which is the instant it emits ActivityStarted; Floci emits that event at+ * schedule time, so there is no later instant to anchor on here.+ */+ private JsonNode awaitToken(CompletableFuture<JsonNode> future, JsonNode stateDef, String taskToken,+ long executionDeadlineNanos) throws Exception {+ int timeoutSeconds = stateDef.path("TimeoutSeconds").asInt(0);+ if (timeoutSeconds <= 0) {+ timeoutSeconds = DEFAULT_TASK_TOKEN_TIMEOUT_SECONDS;}+ int heartbeatSeconds = stateDef.path("HeartbeatSeconds").asInt(0);+ long timeoutDeadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds);try {- return future.get(timeout, TimeUnit.SECONDS);- } catch (java.util.concurrent.TimeoutException e) {- future.cancel(true);- throw new FailStateException("States.HeartbeatTimeout",- "Task timed out after " + timeout + " seconds");+ while (true) {+ long wakeAtNanos = Math.min(executionDeadlineNanos, Math.min(timeoutDeadlineNanos,+ heartbeatDeadlineNanos(taskToken, heartbeatSeconds)));+ try {+ return future.get(wakeAtNanos - System.nanoTime(), TimeUnit.NANOSECONDS);+ } catch (java.util.concurrent.TimeoutException e) {+ // The execution's budget is read first: when it is the clock that ran out, the+ // execution ends as TIMED_OUT and the task's own timeout never applies.+ if (System.nanoTime() >= executionDeadlineNanos) {+ future.cancel(true);+ throw new ExecutionTimedOutException();+ }+ if (System.nanoTime() >= timeoutDeadlineNanos) {+ future.cancel(true);+ throw new TaskTimedOutException("States.Timeout");+ }+ // Read the gap again rather than trusting the one this thread parked on: a+ // heartbeat that landed meanwhile has already moved it past now, and the task+ // goes back to waiting on the later deadline.+ if (System.nanoTime() >= heartbeatDeadlineNanos(taskToken, heartbeatSeconds)) {+ future.cancel(true);+ throw new TaskTimedOutException("States.HeartbeatTimeout");+ }+ }+ }} catch (ExecutionException e) {Throwable cause = e.getCause();if (cause instanceof FailStateException fse) {@@ -703,9 +792,21 @@ public class AslExecutor {}throw new FailStateException("States.TaskFailed",cause != null ? cause.getMessage() : "Task failed");+ } finally {+ sfnService.get().discardPendingToken(taskToken);}}+ /**+ * When the worker's silence becomes too long: its last heartbeat plus the state's+ * {@code HeartbeatSeconds}, or never for a state that declares none.+ */+ private long heartbeatDeadlineNanos(String taskToken, int heartbeatSeconds) {+ return heartbeatSeconds > 0+ ? sfnService.get().lastTaskHeartbeatNanos(taskToken) + TimeUnit.SECONDS.toNanos(heartbeatSeconds)+ : Long.MAX_VALUE;+ }+/*** Extracts the Lambda function name from a reference that may be a bare name, a name with a* version/alias qualifier (e.g. "name:$LATEST"), or a full/partial function ARN@@ -745,7 +846,8 @@ public class AslExecutor {CustomResourceLiveness.tokenIn(payload).ifPresent(customResourceLiveness::touch);}- private JsonNode invokeResource(String resource, JsonNode input, StateMachine sm, String taskToken) throws Exception {+ private JsonNode invokeResource(String resource, JsonNode input, StateMachine sm, String taskToken,+ long executionDeadlineNanos) throws Exception {// Support Lambda resources: direct ARN or optimized integrationString functionName = null;JsonNode lambdaPayload = input;@@ -869,7 +971,7 @@ public class AslExecutor {? ".waitForTaskToken": resource.substring("arn:aws:states:::ecs:runTask".length());String region = extractRegionFromArn(sm.getStateMachineArn());- return invokeEcsRunTask(mode, input, region);+ return invokeEcsRunTask(mode, input, region, executionDeadlineNanos);}// AWS SDK service integrations: Step Functions@@ -896,7 +998,7 @@ public class AslExecutor {if (resource.startsWith("arn:aws:states:::states:startExecution")) {String mode = resource.substring("arn:aws:states:::states:startExecution".length());String region = extractRegionFromArn(sm.getStateMachineArn());- return invokeNestedStateMachine(mode, input, region);+ return invokeNestedStateMachine(mode, input, region, executionDeadlineNanos);}// Activity resource: arn:aws:states:{region}:{account}:activity:{name}@@ -1229,7 +1331,8 @@ public class AslExecutor {return envelope;}- private JsonNode invokeNestedStateMachine(String mode, JsonNode input, String region) throws Exception {+ private JsonNode invokeNestedStateMachine(String mode, JsonNode input, String region,+ long executionDeadlineNanos) throws Exception {String smArn = input.path("StateMachineArn").asText(null);if (smArn == null || smArn.isBlank()) {throw new FailStateException("States.TaskFailed",@@ -1250,9 +1353,10 @@ public class AslExecutor {return result;}- // .sync or .sync:2 — poll until terminal+ // .sync or .sync:2 — poll until terminal, or until the parent execution's TimeoutSeconds+ // budget runs out, which ends the parent as TIMED_OUT and leaves the child running.for (int i = 0; i < 600; i++) {- Thread.sleep(100);+ sleepOrTimeOutExecution(TimeUnit.MILLISECONDS.toNanos(100), executionDeadlineNanos);io.github.hectorvent.floci.services.stepfunctions.model.Execution current =sfnService.get().describeExecution(execArn);String status = current.getStatus();@@ -1302,7 +1406,8 @@ public class AslExecutor {* STOPPED, or ".waitForTaskToken" to launch and let the token future carry the result* (both ".sync" and ".waitForTaskToken" fail the state on a placement failure).*/- private JsonNode invokeEcsRunTask(String mode, JsonNode input, String region) throws Exception {+ private JsonNode invokeEcsRunTask(String mode, JsonNode input, String region,+ long executionDeadlineNanos) throws Exception {String taskDefinition = input.path("TaskDefinition").asText(null);if (taskDefinition == null || taskDefinition.isBlank()) {throw new FailStateException("States.TaskFailed",@@ -1374,7 +1479,8 @@ public class AslExecutor {// fail the state, otherwise tasks beyond the first would run unmonitored.List<String> taskArns = launched.stream().map(EcsTask::getTaskArn).toList();for (int i = 0; i < ECS_SYNC_POLL_ATTEMPTS; i++) {- Thread.sleep(ECS_SYNC_POLL_INTERVAL_MS);+ sleepOrTimeOutExecution(TimeUnit.MILLISECONDS.toNanos(ECS_SYNC_POLL_INTERVAL_MS),+ executionDeadlineNanos);List<EcsTask> described = ecsService.describeTasks(cluster, taskArns, region);boolean allStopped = described.size() == taskArns.size()&& described.stream().allMatch(t -> "STOPPED".equals(t.getLastStatus()));@@ -1845,7 +1951,8 @@ public class AslExecutor {}private StateResult executeWaitState(JsonNode stateDef, JsonNode input, boolean jsonata, JsonNode context,- ObjectNode variables) throws InterruptedException {+ ObjectNode variables, long executionDeadlineNanos)+ throws InterruptedException {int seconds = 0;if (jsonata) {if (stateDef.has("Seconds")) {@@ -1869,7 +1976,7 @@ public class AslExecutor {}// Timestamp and TimestampPath: wait until that time or now, whichever is soonerif (seconds > 0) {- TimeUnit.SECONDS.sleep(seconds);+ sleepOrTimeOutExecution(TimeUnit.SECONDS.toNanos(seconds), executionDeadlineNanos);}if (jsonata) {JsonNode output = applyJsonataOutput(stateDef, input, null, context, variables);@@ -1878,6 +1985,23 @@ public class AslExecutor {return new StateResult(input, stateDef.path("Next").asText(null));}+ /**+ * Sleeps out a pause the definition asked for, ending the execution instead when the state+ * machine's {@code TimeoutSeconds} budget runs out first. The two pauses long enough to+ * outlast that budget are a Wait and a Retry's backoff, and both leave the state they cut+ * without its Exited event, the same way AWS does.+ */+ private void sleepOrTimeOutExecution(long pauseNanos, long executionDeadlineNanos)+ throws InterruptedException {+ long remainingNanos = executionDeadlineNanos - System.nanoTime();+ if (pauseNanos < remainingNanos) {+ TimeUnit.NANOSECONDS.sleep(pauseNanos);+ return;+ }+ TimeUnit.NANOSECONDS.sleep(Math.max(remainingNanos, 0));+ throw new ExecutionTimedOutException();+ }+private StateResult executeSucceedState(JsonNode stateDef, JsonNode input, boolean jsonata, JsonNode context,ObjectNode variables) {if (jsonata) {@@ -1890,7 +2014,9 @@ public class AslExecutor {private StateResult executeFail(JsonNode stateDef, JsonNode input, boolean jsonata, JsonNode context,ObjectNode variables) {String error = stateDef.path("Error").asText(null);- String cause = stateDef.path("Cause").asText(null);+ // A Fail state that declares no Cause reports an empty one, not a missing key. A task that+ // ran out of one of its clocks is the only failure that omits the key.+ String cause = stateDef.path("Cause").asText("");if (jsonata) {JsonNode statesVar = buildStatesVar(input, null, context);if (error != null && JsonataEvaluator.isExpression(error)) {@@ -1903,9 +2029,11 @@ public class AslExecutor {throw new FailStateException(error, cause);}- private StateResult executeParallelState(String name, JsonNode stateDef, JsonNode input, StateMachine sm,- boolean jsonata, String topLevelQueryLanguage, JsonNode context,- ObjectNode variables) throws Exception {+ private StateResult executeParallelState(String name, JsonNode stateDef, JsonNode input,+ AtomicLong producedEventCount, StateMachine sm, boolean jsonata,+ String topLevelQueryLanguage, JsonNode context,+ ObjectNode variables, long executionDeadlineNanos)+ throws Exception {JsonNode branches = stateDef.path("Branches");List<Future<JsonNode>> futures = new ArrayList<>();@@ -1924,28 +2052,29 @@ public class AslExecutor {// scope is thread-bound, so without this a branch's Task integrations would resolve to// the default account rather than the execution's.futures.add(executor.submit(() -> callUnderExecutionAccount(sm,- () -> executeBranch(startAt, branchStates, capturedInput, sm, topLevelQueryLanguage,- branchContext, branchVariables))));+ () -> executeBranch(startAt, branchStates, capturedInput, producedEventCount, sm,+ topLevelQueryLanguage, branchContext, branchVariables))));}int timeoutSeconds = stateDef.path("TimeoutSeconds").asInt(0);- long deadlineNanos = timeoutSeconds > 0+ long stateDeadlineNanos = timeoutSeconds > 0? System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds): Long.MAX_VALUE;+ // Two clocks can end this wait, and the join stops at whichever comes first: the state's+ // own TimeoutSeconds, and the state machine's budget for the whole execution.+ long joinDeadlineNanos = Math.min(stateDeadlineNanos, executionDeadlineNanos);ArrayNode results = objectMapper.createArrayNode();try {for (Future<JsonNode> future : futures) {- long remainingNanos = deadlineNanos - System.nanoTime();+ long remainingNanos = joinDeadlineNanos - System.nanoTime();if (remainingNanos <= 0) {- throw new FailStateException("States.Timeout",- "Parallel state timed out after " + timeoutSeconds + " seconds");+ throw parallelJoinExpired(stateDeadlineNanos, timeoutSeconds);}try {results.add(future.get(remainingNanos, TimeUnit.NANOSECONDS));} catch (java.util.concurrent.TimeoutException e) {- throw new FailStateException("States.Timeout",- "Parallel state timed out after " + timeoutSeconds + " seconds");+ throw parallelJoinExpired(stateDeadlineNanos, timeoutSeconds);}}} catch (InterruptedException e) {@@ -1986,9 +2115,24 @@ public class AslExecutor {return new StateResult(output, stateDef.path("Next").asText(null));}- private StateResult executeMapState(String name, JsonNode stateDef, JsonNode input, StateMachine sm,- boolean jsonata, String topLevelQueryLanguage, JsonNode context,- ObjectNode variables) throws Exception {+ /**+ * Names the clock that ended a Parallel's join. The state's own {@code TimeoutSeconds} fails+ * the state, so its Retry and Catch still apply; the state machine's budget ends the whole+ * execution and no Catch sees it.+ */+ private RuntimeException parallelJoinExpired(long stateDeadlineNanos, int timeoutSeconds) {+ if (System.nanoTime() < stateDeadlineNanos) {+ return new ExecutionTimedOutException();+ }+ return new FailStateException("States.Timeout",+ "Parallel state timed out after " + timeoutSeconds + " seconds");+ }++ private StateResult executeMapState(String name, JsonNode stateDef, JsonNode input,+ AtomicLong producedEventCount, StateMachine sm, boolean jsonata,+ String topLevelQueryLanguage, JsonNode context,+ ObjectNode variables, long executionDeadlineNanos)+ throws Exception {String processorMode = stateDef.path("ItemProcessor").path("ProcessorConfig").path("Mode").asText("INLINE");boolean distributed = "DISTRIBUTED".equals(processorMode);@@ -2055,8 +2199,13 @@ public class AslExecutor {if (hasResultWriter) {childInputsByIndex[i] = iterInput;}- JsonNode branchOutput = executeBranch(startAt, iteratorStates, iterInput, sm,- topLevelQueryLanguage, iterContext, variables.deepCopy());+ // A Distributed Map runs each item as a child execution, and a child execution has a+ // history of its own: the item's events count against its own limit, not the parent's.+ // An inline Map's iterations are part of this execution and count here.+ AtomicLong childExecutionEventCount = distributed ? new AtomicLong() : producedEventCount;+ JsonNode branchOutput = executeBranch(startAt, iteratorStates, iterInput,+ childExecutionEventCount, sm, topLevelQueryLanguage, iterContext,+ variables.deepCopy());if (hasResultWriter) {childTimingsByIndex[i] = new long[]{startMs, System.currentTimeMillis()};}@@ -2064,9 +2213,17 @@ public class AslExecutor {};if (itemCount > 0) {- List<JsonNode> itemOutputs = MapIterationScheduler.execute(- itemCount, Math.max(1, effectiveConcurrency),- i -> () -> callUnderExecutionAccount(sm, makeTask.apply(i)));+ List<JsonNode> itemOutputs;+ try {+ itemOutputs = MapIterationScheduler.execute(+ itemCount, Math.max(1, effectiveConcurrency),+ i -> () -> callUnderExecutionAccount(sm, makeTask.apply(i)),+ executionDeadlineNanos);+ } catch (java.util.concurrent.TimeoutException e) {+ // The only deadline the scheduler is given is the state machine's budget, so its+ // expiry ends the execution rather than failing the Map state.+ throw new ExecutionTimedOutException();+ }results.addAll(itemOutputs);}@@ -2499,10 +2656,17 @@ public class AslExecutor {return limited;}- private JsonNode executeBranch(String startAt, JsonNode states, JsonNode input, StateMachine sm,- String topLevelQueryLanguage, JsonNode context, ObjectNode variables) throws Exception {- List<HistoryEvent> ignored = new ArrayList<>();- AtomicLong eventId = new AtomicLong(0);+ /**+ * Runs the states of one Parallel branch or one Map iteration. floci does not publish their+ * events, but they are events of the execution all the same, so each one is counted against its+ * history-event limit: a branch that never reaches a terminal state ends the whole execution at+ * event 25,000, exactly as one in the top-level flow does. A null history is what tells the+ * states below they are running inside a branch.+ */+ private JsonNode executeBranch(String startAt, JsonNode states, JsonNode input,+ AtomicLong producedEventCount, StateMachine sm,+ String topLevelQueryLanguage, JsonNode context,+ ObjectNode variables) throws Exception {JsonNode currentInput = input;String currentState = startAt;@@ -2517,10 +2681,15 @@ public class AslExecutor {String type = stateDef.path("Type").asText();boolean stateJsonata = isJsonata(stateDef, topLevelQueryLanguage);updateStateContext(context, currentState);+ countTowardsHistoryEventLimit(producedEventCount);StateResult result;try {- result = executeStateWithRetry(currentState, type, stateDef, currentInput, ignored, eventId, sm,- stateJsonata, topLevelQueryLanguage, context, variables);+ // A Parallel or Map branch runs on its own thread and is not cut mid-state by the+ // execution's TimeoutSeconds: the state loop that resumes once the branch returns+ // is where the budget is enforced.+ result = executeStateWithRetry(currentState, type, stateDef, currentInput,+ null, producedEventCount, sm, stateJsonata, topLevelQueryLanguage, context,+ variables, Long.MAX_VALUE);} catch (FailStateException e) {StateResult caught = handleCatch(stateDef, currentInput, e, stateJsonata, context, variables);if (caught == null) {@@ -2528,6 +2697,7 @@ public class AslExecutor {}result = caught;}+ countTowardsHistoryEventLimit(producedEventCount);currentInput = result.output();currentState = result.nextState();if ("Succeed".equals(type) || stateDef.path("End").asBoolean(false)) {@@ -3458,18 +3628,76 @@ public class AslExecutor {// ──────────────────────────── History helpers ────────────────────────────- private void addEvent(List<HistoryEvent> history, AtomicLong counter, String type, Map<String, Object> details) {- addEvent(history, counter, type, counter.get(), details);+ /**+ * Counts one event towards the limit AWS puts on an execution's history, leaving the last slot+ * free: it belongs to the event that ends the execution. The count is taken before it is+ * judged, so however many branches and Map iterations are producing events at once, exactly one+ * of them takes event 24,999 and every other one finds the limit reached.+ *+ * <p>Reaching it raises {@code States.Runtime} at the state that produced the event, and that+ * ends the whole execution: {@link #catchMatches} refuses {@code States.Runtime} before it+ * reads {@code ErrorEquals}, so a Retry and a Catch the state declares for it both stand down.+ */+ static void countTowardsHistoryEventLimit(AtomicLong producedEventCount) {+ if (producedEventCount.incrementAndGet() >= MAX_HISTORY_EVENTS) {+ throw new FailStateException("States.Runtime", HISTORY_EVENT_LIMIT_CAUSE);+ }+ }++ /**+ * Records an event the state machine produced: counted against the history-event limit, then+ * published.+ *+ * <p>{@code history} is null inside a Parallel branch or a Map iteration. Their states are+ * states of this execution and their events count against its limit, but floci does not publish+ * them, so there is nothing to build for them beyond the count.+ */+ private void publishEvent(List<HistoryEvent> history, AtomicLong producedEventCount, String type,+ Map<String, Object> details) {+ countTowardsHistoryEventLimit(producedEventCount);+ if (history == null) {+ return;+ }+ appendEvent(history, type, history.size(), details);+ }++ /**+ * Records a state's Entered event with the previousEventId the top-level flow works out: AWS+ * leaves the Entered event of the state an execution starts in unchained, at previousEventId 0,+ * rather than pointing it at the ExecutionStarted event before it. Only the top-level flow+ * publishes these, so its history is never null.+ */+ private void publishStateEnteredEvent(List<HistoryEvent> history, AtomicLong producedEventCount,+ String type, long previousEventId,+ Map<String, Object> details) {+ countTowardsHistoryEventLimit(producedEventCount);+ appendEvent(history, type, previousEventId, details);}- private void addEvent(List<HistoryEvent> history, AtomicLong counter, String type, long previousEventId,- Map<String, Object> details) {- var event = new HistoryEvent();- event.setId(counter.incrementAndGet());- event.setPreviousEventId(previousEventId);- event.setType(type);- event.setDetails(details);- history.add(event);+ /**+ * Records the event that ends the execution. It does not count towards the history-event limit:+ * an execution always gets to say how it ended, in the slot {@link #publishEvent} leaves free.+ */+ private void publishTerminalEvent(List<HistoryEvent> history, String type, Map<String, Object> details) {+ appendEvent(history, type, history.size(), details);+ }++ /**+ * Appends an event and numbers it from the end of the history: the published history is the one+ * authority for an event's id, so an event's id is its position in the list. Held under the+ * history's own monitor, because StopExecution appends the terminal event of an aborted+ * execution from another thread and seals the history against anything after it.+ */+ private void appendEvent(List<HistoryEvent> history, String type, long previousEventId,+ Map<String, Object> details) {+ synchronized (history) {+ var event = new HistoryEvent();+ event.setId(history.size() + 1L);+ event.setPreviousEventId(previousEventId);+ event.setType(type);+ event.setDetails(details);+ history.add(event);+ }}private record TaskEventProfile(String prefix, String resourceType, String resource) {}@@ -3492,8 +3720,9 @@ public class AslExecutor {return new TaskEventProfile("Task", resource, resource);}- private void addTaskScheduledEvent(List<HistoryEvent> history, AtomicLong eventId, TaskEventProfile profile,- JsonNode stateDef, JsonNode effectiveInput, StateMachine sm) {+ private void addTaskScheduledEvent(List<HistoryEvent> history, AtomicLong producedEventCount,+ TaskEventProfile profile, JsonNode stateDef, JsonNode effectiveInput,+ StateMachine sm) {var details = new LinkedHashMap<String, Object>();if (profile.resourceType() != null) {details.put("resourceType", profile.resourceType());@@ -3512,33 +3741,34 @@ public class AslExecutor {if (stateDef.path("HeartbeatSeconds").isNumber()) {details.put("heartbeatInSeconds", stateDef.path("HeartbeatSeconds").asLong());}- addEvent(history, eventId, profile.prefix() + "Scheduled", details);+ publishEvent(history, producedEventCount, profile.prefix() + "Scheduled", details);}- private void addTaskStartedEvent(List<HistoryEvent> history, AtomicLong eventId, TaskEventProfile profile) {+ private void addTaskStartedEvent(List<HistoryEvent> history, AtomicLong producedEventCount,+ TaskEventProfile profile) {if ("Task".equals(profile.prefix())) {- addEvent(history, eventId, profile.prefix() + "Started",+ publishEvent(history, producedEventCount, profile.prefix() + "Started",Map.of("resourceType", profile.resourceType(), "resource", profile.resource()));} else {- addEvent(history, eventId, profile.prefix() + "Started", null);+ publishEvent(history, producedEventCount, profile.prefix() + "Started", null);}}- private void addTaskSucceededEvent(List<HistoryEvent> history, AtomicLong eventId, TaskEventProfile profile,- JsonNode taskResult) {+ private void addTaskSucceededEvent(List<HistoryEvent> history, AtomicLong producedEventCount,+ TaskEventProfile profile, JsonNode taskResult) {var output = taskResult.toString();if ("Task".equals(profile.prefix())) {- addEvent(history, eventId, profile.prefix() + "Succeeded",+ publishEvent(history, producedEventCount, profile.prefix() + "Succeeded",Map.of("resourceType", profile.resourceType(), "resource", profile.resource(),"output", output, "outputDetails", Map.of("truncated", false)));} else {- addEvent(history, eventId, profile.prefix() + "Succeeded",+ publishEvent(history, producedEventCount, profile.prefix() + "Succeeded",Map.of("output", output, "outputDetails", Map.of("truncated", false)));}}- private void addTaskFailedEvent(List<HistoryEvent> history, AtomicLong eventId, TaskEventProfile profile,- String error, String cause) {+ private void addTaskFailedEvent(List<HistoryEvent> history, AtomicLong producedEventCount,+ TaskEventProfile profile, String error, String cause) {var details = new LinkedHashMap<String, Object>();if ("Task".equals(profile.prefix())) {details.put("resourceType", profile.resourceType());@@ -3550,27 +3780,103 @@ public class AslExecutor {if (cause != null) {details.put("cause", cause);}- addEvent(history, eventId, profile.prefix() + "Failed", details);+ publishEvent(history, producedEventCount, profile.prefix() + "Failed", details);}- private void failExecution(Execution exec, List<HistoryEvent> history, AtomicLong eventId, FailStateException e) {- failExecution(exec, history, eventId, e.error != null ? e.error : "States.Runtime",- e.cause != null ? e.cause : "");+ /**+ * The event a Task leaves when one of its clocks runs out. It names {@code States.Timeout} for+ * both {@code TimeoutSeconds} and {@code HeartbeatSeconds}, and carries no cause.+ */+ private void addTaskTimedOutEvent(List<HistoryEvent> history, AtomicLong producedEventCount,+ TaskEventProfile profile) {+ var details = new LinkedHashMap<String, Object>();+ if ("Task".equals(profile.prefix())) {+ details.put("resourceType", profile.resourceType());+ details.put("resource", profile.resource());+ }+ details.put("error", "States.Timeout");+ publishEvent(history, producedEventCount, profile.prefix() + "TimedOut", details);+ }++ private void failExecution(Execution exec, List<HistoryEvent> history, FailStateException e) {+ failExecution(exec, history, e.error != null ? e.error : "States.Runtime", e.cause);}/*** The single terminal-failure write: every way an execution can fail leaves the same* {@code error}, {@code cause} and {@code ExecutionFailed} event behind, so a client cannot* tell a Fail state from a state that threw from a runtime Error by what it reads back.+ *+ * <p>A null {@code cause} is the failure saying it has none, and both DescribeExecution and the+ * ExecutionFailed event leave the key out rather than reporting it empty. Only a task that ran+ * out of its TimeoutSeconds or HeartbeatSeconds budget arrives here without one.*/- private void failExecution(Execution exec, List<HistoryEvent> history, AtomicLong eventId,- String error, String cause) {- exec.setError(error);- exec.setCause(cause);- exec.setStopDate(System.currentTimeMillis() / 1000.0);- exec.setStatus("FAILED");- addEvent(history, eventId, "ExecutionFailed",- Map.of("error", error, "cause", cause));+ private void failExecution(Execution exec, List<HistoryEvent> history, String error, String cause) {+ synchronized (exec) {+ if (abortedByCaller(exec)) {+ return;+ }+ exec.setError(error);+ exec.setCause(cause);+ exec.setStopDate(System.currentTimeMillis() / 1000.0);+ exec.setStatus("FAILED");+ }+ var details = new LinkedHashMap<String, Object>();+ details.put("error", error);+ if (cause != null) {+ details.put("cause", cause);+ }+ publishTerminalEvent(history, "ExecutionFailed", details);+ }++ /**+ * The third terminal write. A timed out execution carries neither error nor cause:+ * DescribeExecution leaves both keys out, and States.Timeout is named only inside the+ * ExecutionTimedOut event, which points at the start of the execution rather than at the state+ * it cut. The event is appended rather than published, because it is what ends the execution+ * and the history-event limit leaves the last slot free for exactly that.+ */+ private void timeOutExecution(Execution exec, List<HistoryEvent> history) {+ synchronized (exec) {+ if (abortedByCaller(exec)) {+ return;+ }+ exec.setStopDate(System.currentTimeMillis() / 1000.0);+ exec.setStatus("TIMED_OUT");+ }+ appendEvent(history, "ExecutionTimedOut", 0L, Map.of("error", "States.Timeout"));+ }++ /**+ * The single terminal-success write, the mirror of {@link #failExecution}.+ *+ * <p>Status is the publication point, so it is set last. describeExecution hands out this same+ * live Execution, so a client polling for SUCCEEDED between setStatus and setOutput would read+ * a terminal execution with a null output, which real Step Functions never returns.+ */+ private void succeedExecution(Execution exec, List<HistoryEvent> history, JsonNode output) {+ synchronized (exec) {+ if (abortedByCaller(exec)) {+ return;+ }+ exec.setOutput(output.toString());+ exec.setStopDate(System.currentTimeMillis() / 1000.0);+ exec.setStatus("SUCCEEDED");+ }+ publishTerminalEvent(history, "ExecutionSucceeded",+ Map.of("output", output.toString(), "outputDetails", Map.of("truncated", false)));+ }++ /**+ * True once StopExecution published ABORTED on this execution. The state loop reads it between+ * states and every terminal write here reads it before publishing, so the worker's own status+ * loses the race against a stop that arrived while it was still stepping: what a caller has+ * already read back from DescribeExecution is what stands.+ */+ private static boolean abortedByCaller(Execution exec) {+ synchronized (exec) {+ return "ABORTED".equals(exec.getStatus());+ }}private StateResult handleCatch(JsonNode stateDef, JsonNode input, FailStateException failure,@@ -3583,7 +3889,7 @@ public class AslExecutor {String cause = failure.cause != null ? failure.cause : "";for (int i = 0; i < catchers.size(); i++) {JsonNode catcher = catchers.get(i);- if (!catchMatches(catcher, error)) {+ if (!catchMatches(catcher, failure)) {continue;}String next = catcher.path("Next").asText(null);@@ -3607,11 +3913,12 @@ public class AslExecutor {return null;}- private boolean catchMatches(JsonNode catcher, String error) {+ private boolean catchMatches(JsonNode catcher, FailStateException failure) {var errors = catcher.path("ErrorEquals");if (!errors.isArray()) {return false;}+ var error = failure.error != null ? failure.error : "States.Runtime";// States.Runtime is never retried or caught, even when named explicitly in// ErrorEquals. Verified against real AWS: the execution fails immediately.if ("States.Runtime".equals(error)) {@@ -3619,7 +3926,7 @@ public class AslExecutor {}for (JsonNode candidate : errors) {var expected = candidate.asText();- if (expected.equals(error)) {+ if (failure.isNamedBy(expected)) {return true;}if ("States.TaskFailed".equals(expected)@@ -3663,6 +3970,17 @@ public class AslExecutor {record StateResult(JsonNode output, String nextState) {}+ /**+ * Thrown when the state machine's top-level {@code TimeoutSeconds} budget runs out. It is not a+ * {@link FailStateException} on purpose: a Catch clause never sees it, no Retry re-runs the+ * state it cut, and the execution ends TIMED_OUT rather than FAILED.+ */+ static class ExecutionTimedOutException extends RuntimeException {+ ExecutionTimedOutException() {+ super("States.Timeout");+ }+ }+static class FailStateException extends RuntimeException {final String error;final String cause;@@ -3672,5 +3990,34 @@ public class AslExecutor {this.error = error;this.cause = cause;}++ /**+ * Whether an {@code ErrorEquals} entry spelling {@code errorName} names this failure. A+ * failure answers to the error it reports, and a task timeout answers to the name of the+ * clock that ran out as well.+ */+ boolean isNamedBy(String errorName) {+ return errorName.equals(error);+ }+ }++ /**+ * Thrown when a Task ran out of one of the two clocks bounding its wait for a task token. Both+ * report {@code States.Timeout} with no cause and emit a {@code TimedOut} history event; a+ * {@code HeartbeatSeconds} expiry is also caught by an {@code ErrorEquals} naming+ * {@code States.HeartbeatTimeout}.+ */+ static class TaskTimedOutException extends FailStateException {+ private final String expiredClockError;++ TaskTimedOutException(String expiredClockError) {+ super("States.Timeout", null);+ this.expiredClockError = expiredClockError;+ }++ @Override+ boolean isNamedBy(String errorName) {+ return super.isNamedBy(errorName) || errorName.equals(expiredClockError);+ }}}diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationScheduler.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationScheduler.javaindex d8cb258798..77b6a8450f 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationScheduler.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationScheduler.java@@ -12,6 +12,8 @@ import java.util.concurrent.ExecutorCompletionService;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;import java.util.function.IntFunction;/** Runs Map iterations with a bounded in-flight window while preserving input order. */@@ -20,8 +22,15 @@ final class MapIterationScheduler {private MapIterationScheduler() {}+ /**+ * @param deadlineNanos a {@link System#nanoTime()} reading the iterations must all finish+ * before, or {@link Long#MAX_VALUE} for no deadline. Reaching it throws+ * {@link TimeoutException}, leaving the caller to say what the expiry+ * means for the state it belongs to.+ */static <T> List<T> execute(int itemCount, int maxConcurrency,- IntFunction<Callable<T>> taskFactory) throws Exception {+ IntFunction<Callable<T>> taskFactory, long deadlineNanos)+ throws Exception {if (itemCount < 0) {throw new IllegalArgumentException("itemCount must not be negative");}@@ -31,12 +40,8 @@ final class MapIterationScheduler {if (itemCount == 0) {return List.of();}- if (itemCount == 1 || maxConcurrency == 1) {- List<T> results = new ArrayList<>(itemCount);- for (int i = 0; i < itemCount; i++) {- results.add(taskFactory.apply(i).call());- }- return results;+ if (System.nanoTime() >= deadlineNanos) {+ throw new TimeoutException();}int inFlightLimit = Math.min(itemCount, maxConcurrency);@@ -53,7 +58,12 @@ final class MapIterationScheduler {inFlight.add(submit(completions, taskFactory, nextIndex++));}while (completed < itemCount) {- Future<IndexedResult<T>> future = completions.take();+ Future<IndexedResult<T>> future =+ completions.poll(deadlineNanos - System.nanoTime(), TimeUnit.NANOSECONDS);+ if (future == null) {+ cancel(inFlight);+ throw new TimeoutException();+ }inFlight.remove(future);IndexedResult<T> result = future.get();results.set(result.index(), result.value());diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.javaindex ed56bf86cb..fe51d8bd4c 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java@@ -410,7 +410,9 @@ public class StepFunctionsJsonHandler {private Response handleSendTaskFailure(JsonNode request) {service.sendTaskFailure(request.path("taskToken").asText(),- request.path("cause").asText(null),+ // A SendTaskFailure that names no cause fails the task with an empty one, not with+ // a missing key.+ request.path("cause").asText(""),request.path("error").asText(null));return Response.ok(objectMapper.createObjectNode()).build();diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.javaindex 479b7666ff..b878a74f04 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java@@ -46,9 +46,12 @@ public class StepFunctionsService implements Resettable, ResourceProvider {private final StorageBackend<String, Execution> executionStore;private final StorageBackend<String, Activity> activityStore;private final StorageBackend<String, MapRun> mapRunStore;- private final Map<String, List<HistoryEvent>> historyCache = new ConcurrentHashMap<>();+ private final Map<String, ExecutionHistory> historyCache = new ConcurrentHashMap<>();private final Map<String, BlockingQueue<ActivityTask>> activityQueues = new ConcurrentHashMap<>();private final Map<String, CompletableFuture<JsonNode>> pendingTaskTokens = new ConcurrentHashMap<>();+ // When each pending token last showed progress, so a Task's HeartbeatSeconds can bound the gap+ // between heartbeats instead of the whole wait.+ private final Map<String, Long> taskHeartbeatNanos = new ConcurrentHashMap<>();private final RegionResolver regionResolver;private final AslExecutor aslExecutor;private final ObjectMapper objectMapper;@@ -104,6 +107,7 @@ public class StepFunctionsService implements Resettable, ResourceProvider {activityQueues.clear();pendingTaskTokens.values().forEach(f -> f.completeExceptionally(new RuntimeException("StepFunctionsService cleared")));pendingTaskTokens.clear();+ taskHeartbeatNanos.clear();}@Override@@ -524,7 +528,7 @@ public class StepFunctionsService implements Resettable, ResourceProvider {executionStore.put(arn, exec);- var history = new ArrayList<HistoryEvent>();+ var history = new ExecutionHistory();var startEvent = new HistoryEvent();startEvent.setId(1L);startEvent.setPreviousEventId(0L);@@ -537,9 +541,10 @@ public class StepFunctionsService implements Resettable, ResourceProvider {LOG.infov("Started execution: {0}", arn);+ // The executor appends to this same history, so the cache entry put above is already the+ // finished history by the time this runs.aslExecutor.executeAsync(sm, exec, history, mockedTestCase, (updatedExec, updatedHistory) -> {executionStore.put(updatedExec.getExecutionArn(), updatedExec);- historyCache.put(updatedExec.getExecutionArn(), updatedHistory);LOG.infov("Execution {0} completed with status {1}", updatedExec.getExecutionArn(), updatedExec.getStatus());});@@ -660,30 +665,80 @@ public class StepFunctionsService implements Resettable, ResourceProvider {.map(e -> e.getStateMachineArn().equals(stateMachineArn)).orElse(false));}+ /**+ * Aborts a running execution. The caller's {@code error} and {@code cause} land on the+ * Execution itself, not only on the history event, so DescribeExecution reports them.+ *+ * <p>The worker thread may still be inside a state when this runs. It shares this Execution+ * instance and reads the status it publishes here, so the writes are made under the same+ * monitor the worker's terminal write takes: ABORTED is the status that stands.+ *+ * <p>ExecutionAborted seals the history for the same reason: the worker still has the state it+ * is inside left to record, and those events belong to an execution the caller has already+ * been told is finished.+ */public void stopExecution(String arn, String cause, String error) {Execution exec = describeExecution(arn);- if (!"RUNNING".equals(exec.getStatus())) {- return;+ synchronized (exec) {+ if (!"RUNNING".equals(exec.getStatus())) {+ return;+ }+ exec.setError(error);+ exec.setCause(cause);+ exec.setStopDate(System.currentTimeMillis() / 1000.0);+ exec.setStatus("ABORTED");}- exec.setStopDate(System.currentTimeMillis() / 1000.0);- exec.setStatus("ABORTED");executionStore.put(arn, exec);- List<HistoryEvent> history = historyCache.getOrDefault(arn, new ArrayList<>());- HistoryEvent event = new HistoryEvent();- event.setId(history.size() + 1L);- event.setPreviousEventId((long) history.size());- event.setType("ExecutionAborted");Map<String, Object> details = new HashMap<>();if (error != null) details.put("error", error);if (cause != null) details.put("cause", cause);- event.setDetails(details);- history.add(event);+ // An execution can outlive its history: the executions are stored, the histories are held in+ // memory only, so a restart in persistent mode brings a RUNNING execution back with nothing+ // behind it. The abort still gets recorded, against a history that starts here.+ historyCache.computeIfAbsent(arn, key -> new ExecutionHistory())+ .sealWith("ExecutionAborted", details);}public List<HistoryEvent> getExecutionHistory(String arn) {describeExecution(arn);- return historyCache.getOrDefault(arn, Collections.emptyList());+ ExecutionHistory history = historyCache.get(arn);+ return history != null ? history : Collections.emptyList();+ }++ /**+ * The history of one execution, appended to by the worker thread running the state machine and+ * by the thread serving StopExecution. Sealing it with the terminal event is what makes that+ * event the last one: the worker is still inside a state when the abort lands, and the events+ * it has left to record belong to an execution the caller has already been told is finished.+ *+ * <p>The seal is enforced in {@link #add}, the one way the worker appends. {@code addAll} does+ * not route through it, so a bulk append would write past the seal; nothing does one today.+ */+ static final class ExecutionHistory extends ArrayList<HistoryEvent> {++ private static final long serialVersionUID = 1L;++ private boolean sealed;++ @Override+ public synchronized boolean add(HistoryEvent event) {+ if (sealed) {+ return false;+ }+ return super.add(event);+ }++ /** Appends the terminal event, numbered from the end of the history, and takes no more. */+ synchronized void sealWith(String terminalEventType, Map<String, Object> details) {+ HistoryEvent terminalEvent = new HistoryEvent();+ terminalEvent.setId(size() + 1L);+ terminalEvent.setPreviousEventId((long) size());+ terminalEvent.setType(terminalEventType);+ terminalEvent.setDetails(details);+ add(terminalEvent);+ sealed = true;+ }}// ──────────────────────────── Activities ────────────────────────────@@ -744,9 +799,32 @@ public class StepFunctionsService implements Resettable, ResourceProvider {public CompletableFuture<JsonNode> registerPendingToken(String token) {CompletableFuture<JsonNode> future = new CompletableFuture<>();pendingTaskTokens.put(token, future);+ taskHeartbeatNanos.put(token, System.nanoTime());return future;}+ /**+ * When the token last showed progress: the moment its task was scheduled, then every+ * SendTaskHeartbeat that named it. A token that is no longer pending reads as having just+ * reported, so a task whose result already arrived is never failed on a heartbeat gap.+ */+ public long lastTaskHeartbeatNanos(String taskToken) {+ Long reportedAt = taskToken != null ? taskHeartbeatNanos.get(taskToken) : null;+ return reportedAt != null ? reportedAt : System.nanoTime();+ }++ /**+ * Forgets a token the waiting task has stopped listening for, so a later SendTaskSuccess,+ * SendTaskFailure or SendTaskHeartbeat naming it reports no pending task.+ */+ public void discardPendingToken(String taskToken) {+ if (taskToken == null) {+ return;+ }+ pendingTaskTokens.remove(taskToken);+ taskHeartbeatNanos.remove(taskToken);+ }+// ──────────────────────────── Tasks ────────────────────────────/**@@ -782,8 +860,16 @@ public class StepFunctionsService implements Resettable, ResourceProvider {return true;}+ /**+ * Resets the gap a Task's {@code HeartbeatSeconds} allows. A heartbeat for a token nobody is+ * waiting for is logged and changes nothing, as in {@link #sendTaskSuccess(String, String)}.+ */public void sendTaskHeartbeat(String taskToken) {- LOG.debugv("Task heartbeat for token {0}", taskToken);+ if (taskToken == null || !pendingTaskTokens.containsKey(taskToken)) {+ LOG.warnv("SendTaskHeartbeat: no pending task for token {0}", taskToken);+ return;+ }+ taskHeartbeatNanos.put(taskToken, System.nanoTime());}// ──────────────────────────── Map runs ────────────────────────────diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHistoryEventLimitTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHistoryEventLimitTest.javanew file mode 100644index 0000000000..49c3a225d0--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHistoryEventLimitTest.java@@ -0,0 +1,377 @@+package io.github.hectorvent.floci.services.stepfunctions;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbJsonHandler;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;+import io.github.hectorvent.floci.services.lambda.LambdaExecutorService;+import io.github.hectorvent.floci.services.lambda.LambdaFunctionStore;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;+import io.github.hectorvent.floci.services.stepfunctions.model.Execution;+import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;+import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;+import io.quarkus.test.junit.QuarkusTest;+import io.vertx.mutiny.core.Vertx;+import jakarta.enterprise.inject.Instance;+import jakarta.inject.Inject;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicLong;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.mock;++/**+ * Guards the bound AWS puts on a running execution: a state machine that never reaches a terminal+ * state is ended at 25,000 history events with {@code States.Runtime}. The cut is an event count,+ * not a wall clock, and the counter is neither reset nor offset: the {@code ExecutionFailed} event+ * is event 25,000 itself, so the last event the machine produced is 24,999.+ *+ * <p>A {@code Choice} that loops back through a {@code Pass} reaches it in a few hundred+ * milliseconds, which is why nothing here waits on the clock.+ */+@QuarkusTest+class AslExecutorHistoryEventLimitTest {++ private static final String REGION = "us-east-2";+ private static final String ACCOUNT = "000000000000";++ /** A Choice that spins forever, four history events per lap and no terminal state. */+ private static final String RUNAWAY_LOOP = """+ {+ "StartAt": "Loop",+ "States": {+ "Loop": {+ "Type": "Choice",+ "Choices": [{"Variable": "$.spin", "BooleanEquals": true, "Next": "Tick"}],+ "Default": "Done"+ },+ "Tick": {"Type": "Pass", "Next": "Loop"},+ "Done": {"Type": "Succeed"}+ }+ }+ """;++ /**+ * A Parallel branch that laps 10,000 times, four history events per lap. The lap count is high+ * enough that the branch alone produces far more than 25,000 events, and finite so the run ends+ * either way: bounded, at event 25,000; unbounded, by finishing every lap.+ */+ private static final String RUNAWAY_PARALLEL_BRANCH = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Seed",+ "States": {+ "Seed": {"Type": "Pass", "Assign": {"laps": 0}, "Next": "Fan"},+ "Fan": {+ "Type": "Parallel",+ "Branches": [{+ "StartAt": "Spin",+ "States": {+ "Spin": {+ "Type": "Choice",+ "Choices": [{"Condition": "{% $laps < 10000 %}", "Next": "Tick"}],+ "Default": "Stop"+ },+ "Tick": {"Type": "Pass", "Assign": {"laps": "{% $laps + 1 %}"}, "Next": "Spin"},+ "Stop": {"Type": "Succeed"}+ }+ }],+ "End": true+ }+ }+ }+ """;++ /**+ * The same runaway branch under a Parallel that declares a Retry and a Catch for States.ALL,+ * the two clauses AWS allows on a Parallel. Reaching the limit is States.Runtime, which+ * catchMatches refuses before it reads ErrorEquals, so neither clause runs.+ */+ private static final String RUNAWAY_PARALLEL_BRANCH_UNDER_RETRY_AND_CATCH = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Seed",+ "States": {+ "Seed": {"Type": "Pass", "Assign": {"laps": 0}, "Next": "Fan"},+ "Fan": {+ "Type": "Parallel",+ "Retry": [{"ErrorEquals": ["States.ALL"], "IntervalSeconds": 30, "MaxAttempts": 3}],+ "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "Recovered"}],+ "Branches": [{+ "StartAt": "Spin",+ "States": {+ "Spin": {+ "Type": "Choice",+ "Choices": [{"Condition": "{% $laps < 10000 %}", "Next": "Tick"}],+ "Default": "Stop"+ },+ "Tick": {"Type": "Pass", "Assign": {"laps": "{% $laps + 1 %}"}, "Next": "Spin"},+ "Stop": {"Type": "Succeed"}+ }+ }],+ "Next": "Recovered"+ },+ "Recovered": {"Type": "Succeed"}+ }+ }+ """;++ /**+ * More items than the 25,000-event limit has room for: each one enters and exits a single Pass,+ * so 13,000 items are 26,000 events if the parent is charged for them.+ */+ private static final int DISTRIBUTED_MAP_ITEMS = 13_000;++ /** One item per iteration, run one at a time so the run is the same on every machine. */+ private static final String DISTRIBUTED_MAP_OVER_MANY_ITEMS = """+ {+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "ItemsPath": "$.items",+ "MaxConcurrency": 1,+ "ItemProcessor": {+ "ProcessorConfig": {"Mode": "DISTRIBUTED", "ExecutionType": "STANDARD"},+ "StartAt": "Handle",+ "States": {"Handle": {"Type": "Pass", "End": true}}+ },+ "End": true+ }+ }+ }+ """;++ /**+ * The same items run inline, 40 at a time. An inline Map's iterations are part of this+ * execution, so together they run it out of history events while 40 of them are in flight.+ */+ private static final String INLINE_MAP_OVER_MANY_ITEMS = """+ {+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Map",+ "ItemsPath": "$.items",+ "MaxConcurrency": 40,+ "ItemProcessor": {+ "StartAt": "Handle",+ "States": {"Handle": {"Type": "Pass", "End": true}}+ },+ "End": true+ }+ }+ }+ """;++ private final ObjectMapper objectMapper = new ObjectMapper();+ private final List<HistoryEvent> history = new ArrayList<>();+ private AslExecutor executor;++ @Inject+ Vertx vertx;++ @BeforeEach+ void setUp() {+ executor = new AslExecutor(+ mock(LambdaExecutorService.class),+ mock(LambdaFunctionStore.class),+ mock(DynamoDbService.class),+ mock(DynamoDbJsonHandler.class),+ mock(SqsJsonHandler.class),+ mock(io.github.hectorvent.floci.services.cloudformation.CloudFormationQueryHandler.class),+ mock(io.github.hectorvent.floci.services.ec2.Ec2Service.class),+ mock(S3Service.class),+ mock(io.github.hectorvent.floci.services.ecs.EcsService.class),+ mock(io.github.hectorvent.floci.services.ecs.EcsJsonHandler.class),+ mock(io.github.hectorvent.floci.services.eventbridge.EventBridgeHandler.class),+ mock(io.github.hectorvent.floci.services.scheduler.SchedulerService.class),+ mock(io.github.hectorvent.floci.services.scheduler.SchedulerController.class),+ objectMapper,+ new JsonataEvaluator(objectMapper),+ mock(Instance.class), mock(EmulatorConfig.class), vertx, null);+ }++ @Test+ void runawayLoopFailsWithTheHistoryEventLimitError() {+ Execution execution = run(RUNAWAY_LOOP);++ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.Runtime", execution.getError());+ assertEquals("The execution reached the maximum number of history events (25000).",+ execution.getCause());+ assertNotNull(execution.getStopDate());+ }++ @Test+ void executionFailedIsTheTwentyFiveThousandthEvent() {+ run(RUNAWAY_LOOP);++ assertEquals(25000, history.size());+ HistoryEvent last = history.get(history.size() - 1);+ assertEquals("ExecutionFailed", last.getType());+ assertEquals(25000L, last.getId());+ // Not reset and not offset: 24,999 is the last event the machine itself produced.+ HistoryEvent lastFromTheMachine = history.get(history.size() - 2);+ assertEquals(24999L, lastFromTheMachine.getId());+ assertNotEquals("ExecutionFailed", lastFromTheMachine.getType());+ }++ @Test+ void eventsProducedInsideAParallelBranchCountTowardsTheLimit() {+ Execution execution = run(RUNAWAY_PARALLEL_BRANCH);++ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.Runtime", execution.getError());+ assertEquals("The execution reached the maximum number of history events (25000).",+ execution.getCause());+ }++ /**+ * The branch produced the events that ran the execution out of them, and floci publishes none+ * of them. What the caller reads back is the four events the top-level flow did produce, ending+ * in the failure: an unbroken chain, not four events numbered as if 25,000 had happened.+ */+ @Test+ void aParallelBranchLeavesThePublishedHistoryUnbroken() {+ run(RUNAWAY_PARALLEL_BRANCH);++ assertEquals(List.of("PassStateEntered", "PassStateExited", "ParallelStateEntered",+ "ExecutionFailed"),+ history.stream().map(HistoryEvent::getType).toList());+ for (int index = 0; index < history.size(); index++) {+ assertEquals(index + 1L, history.get(index).getId(), "unexpected id at index " + index);+ assertEquals((long) index, history.get(index).getPreviousEventId(),+ "unexpected previousEventId at index " + index);+ }+ }++ /**+ * A Distributed Map runs each item as a child execution, and a child execution has a history of+ * its own. The parent is charged for the Map run, not for what the items do inside it, so a+ * Distributed Map over more items than the limit has room for still succeeds.+ */+ @Test+ void aDistributedMapItemDoesNotSpendTheParentsHistoryEvents() {+ Execution execution = run(DISTRIBUTED_MAP_OVER_MANY_ITEMS, manyItemsInput());++ assertEquals("SUCCEEDED", execution.getStatus(),+ "error=" + execution.getError() + " cause=" + execution.getCause());+ }++ /**+ * The same items run inline are the execution's own, whatever the concurrency: 40 iterations+ * counting against one limit run it out of history events, where a Distributed Map's items do+ * not.+ */+ @Test+ void concurrentInlineMapIterationsSpendTheExecutionsHistoryEvents() {+ Execution execution = run(INLINE_MAP_OVER_MANY_ITEMS, manyItemsInput());++ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.Runtime", execution.getError());+ assertEquals("The execution reached the maximum number of history events (25000).",+ execution.getCause());+ }++ /**+ * The event that reaches the limit is what ends the execution, so exactly one of the iterations+ * racing for the last countable event may take it. Reading the count and then taking it lets+ * every racer through the check and spends more events than the execution has.+ */+ @Test+ void onlyOneOfTheThreadsRacingForTheLastCountableEventTakesIt() throws Exception {+ int racers = 40;+ // 24,998 events produced: one is left before the limit, and event 25,000 is the failure.+ AtomicLong producedEventCount = new AtomicLong(24_998);+ CountDownLatch ready = new CountDownLatch(racers);+ CountDownLatch go = new CountDownLatch(1);+ AtomicInteger counted = new AtomicInteger();++ List<Thread> racerThreads = new ArrayList<>();+ for (int racer = 0; racer < racers; racer++) {+ Thread thread = new Thread(() -> {+ ready.countDown();+ try {+ go.await();+ AslExecutor.countTowardsHistoryEventLimit(producedEventCount);+ counted.incrementAndGet();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ } catch (AslExecutor.FailStateException expected) {+ // The limit was already reached by another racer.+ }+ });+ racerThreads.add(thread);+ thread.start();+ }+ ready.await();+ go.countDown();+ for (Thread thread : racerThreads) {+ thread.join();+ }++ assertEquals(1, counted.get(), "more than one racer took the last countable event");+ }++ /** {@code {"items": [0, 1, ... ]}}, one item per iteration of the Distributed Map. */+ private static String manyItemsInput() {+ StringBuilder input = new StringBuilder("{\"items\": [0");+ for (int item = 1; item < DISTRIBUTED_MAP_ITEMS; item++) {+ input.append(',').append(item);+ }+ return input.append("]}").toString();+ }++ @Test+ void neitherRetryNorCatchInterceptsTheHistoryEventLimit() {+ long startedAt = System.currentTimeMillis();++ Execution execution = run(RUNAWAY_PARALLEL_BRANCH_UNDER_RETRY_AND_CATCH);++ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.Runtime", execution.getError());+ assertEquals("The execution reached the maximum number of history events (25000).",+ execution.getCause());+ assertEquals("ExecutionFailed", history.get(history.size() - 1).getType());+ assertNotEquals("SucceedStateEntered", history.get(history.size() - 2).getType());+ // Three attempts at IntervalSeconds 30 would put this past 90 seconds if the Retry ran.+ assertTrue(System.currentTimeMillis() - startedAt < 30_000,+ "the Retry's backoff ran");+ }++ private Execution run(String definition) {+ return run(definition, "{\"spin\": true}");+ }++ private Execution run(String definition, String input) {+ StateMachine stateMachine = new StateMachine();+ stateMachine.setName("history-event-limit-test");+ stateMachine.setStateMachineArn(+ "arn:aws:states:%s:%s:stateMachine:history-event-limit-test".formatted(REGION, ACCOUNT));+ stateMachine.setRoleArn("arn:aws:iam::%s:role/test-role".formatted(ACCOUNT));+ stateMachine.setDefinition(definition);++ Execution execution = new Execution();+ execution.setName("history-event-limit-execution");+ execution.setExecutionArn(+ "arn:aws:states:%s:%s:execution:history-event-limit-test:history-event-limit-execution"+ .formatted(REGION, ACCOUNT));+ execution.setStateMachineArn(stateMachine.getStateMachineArn());+ execution.setInput(input);++ executor.executeSync(stateMachine, execution, history, (updated, events) -> {+ });+ return execution;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTaskHistoryEventsTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTaskHistoryEventsTest.javaindex 4ca7bc7c8a..57c54717ad 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTaskHistoryEventsTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTaskHistoryEventsTest.java@@ -257,6 +257,48 @@ class AslExecutorTaskHistoryEventsTest {assertNull(started.getDetails());}+ /**+ * A Parallel branch and a Map iteration run states of their own whose events floci does not+ * publish. The published history is still one unbroken chain: the ids of the events around them+ * do not skip the numbers those states used.+ */+ @Test+ void parallelAndMapPublishAnUnbrokenEventChain() throws Exception {+ var history = new ArrayList<HistoryEvent>();+ var execution = run("""+ {+ "StartAt": "Fan",+ "States": {+ "Fan": {+ "Type": "Parallel",+ "Branches": [{+ "StartAt": "BranchStep",+ "States": {"BranchStep": {"Type": "Pass", "Next": "BranchTail"},+ "BranchTail": {"Type": "Pass", "End": true}}+ }],+ "Next": "Iterate"+ },+ "Iterate": {+ "Type": "Map",+ "ItemsPath": "$[0].items",+ "ItemProcessor": {+ "StartAt": "ItemStep",+ "States": {"ItemStep": {"Type": "Pass", "End": true}}+ },+ "End": true+ }+ }+ }+ """, "{\"items\": [1, 2]}", null, history);++ assertEquals("SUCCEEDED", execution.getStatus(), "history: " + typesOf(history));+ assertEquals(+ List.of("ParallelStateEntered", "ParallelStateExited",+ "MapStateEntered", "MapStateExited", "ExecutionSucceeded"),+ typesOf(history));+ assertChain(history);+ }+private static List<String> typesOf(List<HistoryEvent> history) {return history.stream().map(HistoryEvent::getType).toList();}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationSchedulerTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationSchedulerTest.javaindex cc2f72691d..2598c03121 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationSchedulerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationSchedulerTest.java@@ -10,6 +10,7 @@ import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;import java.util.concurrent.atomic.AtomicInteger;import static org.junit.jupiter.api.Assertions.assertEquals;@@ -35,7 +36,7 @@ class MapIterationSchedulerTest {List<Integer> results = MapIterationScheduler.execute(5, 1, index -> () -> {starts.add(index);return index;- });+ }, Long.MAX_VALUE);assertEquals(List.of(0, 1, 2, 3, 4), starts);assertEquals(starts, results);@@ -64,7 +65,7 @@ class MapIterationSchedulerTest {} finally {active.decrementAndGet();}- }));+ }, Long.MAX_VALUE));assertTrue(firstWaveStarted.await(2, TimeUnit.SECONDS), "initial worker window did not start");assertEquals(maxConcurrency, started.get(), "queued items must not be submitted early");@@ -99,7 +100,7 @@ class MapIterationSchedulerTest {fail("slow iteration did not start");}throw new IllegalStateException("later iteration failed");- }));+ }, Long.MAX_VALUE));ExecutionException failure = assertThrows(ExecutionException.class,() -> execution.get(2, TimeUnit.SECONDS));@@ -123,8 +124,105 @@ class MapIterationSchedulerTest {laterItemsCompleted.countDown();}return index;- }));+ }, Long.MAX_VALUE));assertEquals(List.of(0, 1, 2), results);}++ @Test+ void stopsStartingSerialIterationsOncePastTheDeadline() {+ AtomicInteger started = new AtomicInteger();++ assertThrows(TimeoutException.class, () -> MapIterationScheduler.execute(3, 1, index -> () -> {+ started.incrementAndGet();+ return index;+ }, System.nanoTime() - 1));++ assertEquals(0, started.get());+ }++ @Test+ void endsTheSerialWaitForASingleItemOncePastTheDeadline() throws Exception {+ CountDownLatch neverRelease = new CountDownLatch(1);+ CountDownLatch blockedInterrupted = new CountDownLatch(1);++ // Not try-with-resources: against the still-buggy serial path the iteration below never+ // returns and is never interrupted, so ExecutorService#close() would block on+ // awaitTermination forever. shutdownNow() in the finally block does not wait.+ ExecutorService driver = Executors.newSingleThreadExecutor();+ try {+ Future<List<Integer>> execution = driver.submit(() -> MapIterationScheduler.execute(+ 1, 3, index -> () -> {+ try {+ neverRelease.await();+ return index;+ } catch (InterruptedException e) {+ blockedInterrupted.countDown();+ throw e;+ }+ }, System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(200)));++ ExecutionException failure = assertThrows(ExecutionException.class,+ () -> execution.get(2, TimeUnit.SECONDS));+ assertEquals(TimeoutException.class, failure.getCause().getClass());+ assertTrue(blockedInterrupted.await(2, TimeUnit.SECONDS),+ "the single iteration should be interrupted once the deadline passes");+ } finally {+ driver.shutdownNow();+ }+ }++ @Test+ void endsTheSerialWaitForMaxConcurrencyOneOncePastTheDeadline() throws Exception {+ CountDownLatch neverRelease = new CountDownLatch(1);+ CountDownLatch blockedInterrupted = new CountDownLatch(1);++ // Not try-with-resources: see endsTheSerialWaitForASingleItemOncePastTheDeadline above.+ ExecutorService driver = Executors.newSingleThreadExecutor();+ try {+ Future<List<Integer>> execution = driver.submit(() -> MapIterationScheduler.execute(+ 2, 1, index -> () -> {+ try {+ neverRelease.await();+ return index;+ } catch (InterruptedException e) {+ blockedInterrupted.countDown();+ throw e;+ }+ }, System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(200)));++ ExecutionException failure = assertThrows(ExecutionException.class,+ () -> execution.get(2, TimeUnit.SECONDS));+ assertEquals(TimeoutException.class, failure.getCause().getClass());+ assertTrue(blockedInterrupted.await(2, TimeUnit.SECONDS),+ "the blocked iteration should be interrupted once the deadline passes");+ } finally {+ driver.shutdownNow();+ }+ }++ @Test+ void endsTheConcurrentWaitOncePastTheDeadline() throws Exception {+ CountDownLatch neverRelease = new CountDownLatch(1);+ CountDownLatch blockedInterrupted = new CountDownLatch(2);++ try (ExecutorService driver = Executors.newSingleThreadExecutor()) {+ Future<List<Integer>> execution = driver.submit(() -> MapIterationScheduler.execute(+ 2, 2, index -> () -> {+ try {+ neverRelease.await();+ return index;+ } catch (InterruptedException e) {+ blockedInterrupted.countDown();+ throw e;+ }+ }, System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(200)));++ ExecutionException failure = assertThrows(ExecutionException.class,+ () -> execution.get(5, TimeUnit.SECONDS));+ assertEquals(TimeoutException.class, failure.getCause().getClass());+ assertTrue(blockedInterrupted.await(2, TimeUnit.SECONDS),+ "the iterations still running should be interrupted once the deadline passes");+ }+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsAwsSdkTaskIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsAwsSdkTaskIntegrationTest.javaindex 3baf95ae21..ceee268ad9 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsAwsSdkTaskIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsAwsSdkTaskIntegrationTest.java@@ -284,10 +284,59 @@ class StepFunctionsAwsSdkTaskIntegrationTest {assertEquals("Sfn.InvalidTokenException", describe.jsonPath().getString("error"));}+ @Test+ @Order(9)+ void sendTaskSuccessOnATokenWhoseResourceInvocationThrewFailsAsInvalid() throws Exception {+ // "Leak" registers a task token, then fails invoking its own (unsupported) resource before it+ // ever waits on that token. The registration must not outlive the failure: the token is+ // recovered from the TaskScheduled event it wrote on its way to failing, and a second+ // execution's SendTaskSuccess on that same token must find nothing pending for it.+ var leakArn = createStateMachine("aws-sdk-task-leaked-token", """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Leak",+ "States": {+ "Leak": {+ "Type": "Task",+ "Resource": "arn:aws:states:::unsupported:doSomething.waitForTaskToken",+ "Arguments": {"token": "{% $states.context.Task.Token %}"},+ "End": true+ }+ }+ }+ """);++ var leaked = waitForTerminalState(startExecution(leakArn, "{}"));+ assertEquals("FAILED", leaked.jsonPath().getString("status"));+ assertEquals("States.TaskFailed", leaked.jsonPath().getString("error"));+ var taskToken = scheduledTaskToken(leaked.jsonPath().getString("executionArn"));++ var senderArn = createStateMachine("aws-sdk-send-success-leaked-token", """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Resume",+ "States": {+ "Resume": {+ "Type": "Task",+ "Resource": "arn:aws:states:::aws-sdk:sfn:sendTaskSuccess",+ "Arguments": {"TaskToken": "{% $states.input.token %}", "Output": {}},+ "End": true+ }+ }+ }+ """);++ var describe = waitForTerminalState(startExecution(senderArn, tokenInput(taskToken)));++ assertEquals("FAILED", describe.jsonPath().getString("status"),+ "Leak's token must be discarded once its resource invocation throws, not left pending");+ assertEquals("Sfn.InvalidTokenException", describe.jsonPath().getString("error"));+ }+// ──────────────────────────── scheduler:createSchedule / updateSchedule ────────────────────────────@Test- @Order(9)+ @Order(10)void createScheduleReturnsTheScheduleArnAndTheScheduleIsReadable() throws Exception {var result = mapper.readTree(succeedingOutputOf(createStateMachine("aws-sdk-create-schedule", scheduleTask("createSchedule",@@ -303,7 +352,7 @@ class StepFunctionsAwsSdkTaskIntegrationTest {}@Test- @Order(10)+ @Order(11)void updateScheduleKeepsTheArnAndChangesTheExpression() throws Exception {var result = mapper.readTree(succeedingOutputOf(createStateMachine("aws-sdk-update-schedule", scheduleTask("updateSchedule",@@ -317,7 +366,7 @@ class StepFunctionsAwsSdkTaskIntegrationTest {}@Test- @Order(11)+ @Order(12)void createScheduleOnANameAlreadyTakenFailsWithConflict() {var describe = waitForTerminalState(startExecution(createStateMachine("aws-sdk-create-schedule-twice", scheduleTask("createSchedule",@@ -328,7 +377,7 @@ class StepFunctionsAwsSdkTaskIntegrationTest {}@Test- @Order(12)+ @Order(13)void updateScheduleOnAScheduleThatDoesNotExistFailsWithResourceNotFound() {var describe = waitForTerminalState(startExecution(createStateMachine("aws-sdk-update-missing-schedule", scheduleTask("updateSchedule",@@ -339,7 +388,7 @@ class StepFunctionsAwsSdkTaskIntegrationTest {}@Test- @Order(13)+ @Order(14)void createScheduleWithIncompleteEventBridgeParametersIsCatchableAsAValidationException() throws Exception {var smArn = createStateMachine("aws-sdk-create-schedule-invalid-target", """{@@ -379,7 +428,7 @@ class StepFunctionsAwsSdkTaskIntegrationTest {}@Test- @Order(14)+ @Order(15)void createScheduleWithAMalformedListIsCatchableAsASerializationException() throws Exception {// A malformed EcsParameters list is refused before the conversion runs, so it reaches Catch// as the SDK exception AWS sends rather than as States.Runtime.@@ -569,6 +618,29 @@ class StepFunctionsAwsSdkTaskIntegrationTest {.when().post("/");}+ private static Response getExecutionHistory(String execArn) {+ return given()+ .header("X-Amz-Target", "AWSStepFunctions.GetExecutionHistory")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"executionArn": "%s"}+ """.formatted(execArn))+ .when().post("/");+ }++ /** The token a Task registered for itself, recovered from its own TaskScheduled event. */+ private static String scheduledTaskToken(String execArn) throws Exception {+ var events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");+ for (var event : events) {+ if ("TaskScheduled".equals(event.path("type").asText())) {+ var parameters = event.path("taskScheduledEventDetails").path("parameters").asText();+ return mapper.readTree(parameters).path("token").asText();+ }+ }+ fail("no TaskScheduled event found in history: " + events);+ return null;+ }+private static Response waitForTerminalState(String execArn) {for (var i = 0; i < 150; i++) {var resp = describeExecution(execArn);diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsStopExecutionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsStopExecutionIntegrationTest.javanew file mode 100644index 0000000000..846f002cef--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsStopExecutionIntegrationTest.java@@ -0,0 +1,194 @@+package io.github.hectorvent.floci.services.stepfunctions;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import jakarta.inject.Inject;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertAll;+import static org.junit.jupiter.api.Assertions.assertEquals;++/**+ * Guards that {@code StopExecution} stops the execution instead of relabelling it: the status,+ * error and cause it writes are what {@code DescribeExecution} returns, and the worker thread that+ * is still inside the {@code Wait} does not overwrite them when it gets there.+ */+@QuarkusTest+class StepFunctionsStopExecutionIntegrationTest {++ private static final String SFN_CONTENT_TYPE = "application/x-amz-json-1.0";+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/test-role";++ /** Long enough that the stop lands mid-flight, short enough that the re-read is cheap. */+ private static final int WAIT_SECONDS = 3;++ private static final String WAITING_MACHINE = """+ {"StartAt": "W", "States": {"W": {"Type": "Wait", "Seconds": %d, "End": true}}}+ """.formatted(WAIT_SECONDS);++ @Inject+ StepFunctionsService stepFunctionsService;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ /**+ * The executions survive a restart in persistent storage mode; their histories do not, because+ * the history lives in memory. An execution that reloads as RUNNING with no history behind it is+ * still stoppable, and StopExecution reports the abort it performed rather than an error. The+ * same pairing happens during a reset, between the moment the in-memory state is dropped and+ * the moment the stored executions are.+ */+ @Test+ void executionWithNoHistoryInMemoryIsStillStoppable() throws Exception {+ String executionArn = startWaitingExecution("stop-without-history");+ Thread.sleep(300);++ stepFunctionsService.clear();++ stopExecution(executionArn, "Manual", "probe");++ Response described = describeExecution(executionArn);+ assertEquals("ABORTED", described.jsonPath().getString("status"));+ assertEquals("Manual", described.jsonPath().getString("error"));+ assertEquals("probe", described.jsonPath().getString("cause"));+ }++ @Test+ void stoppedExecutionReportsTheCallersErrorAndCause() throws Exception {+ String executionArn = startWaitingExecution("stop-reports-error");+ Thread.sleep(300);++ stopExecution(executionArn, "Manual", "probe");++ Response described = describeExecution(executionArn);+ assertEquals("ABORTED", described.jsonPath().getString("status"));+ assertEquals("Manual", described.jsonPath().getString("error"));+ assertEquals("probe", described.jsonPath().getString("cause"));+ }++ @Test+ void stoppedExecutionStaysAbortedAfterTheWaitElapses() throws Exception {+ String executionArn = startWaitingExecution("stop-survives-wait");+ Thread.sleep(300);++ stopExecution(executionArn, "Manual", "probe");+ double stopDate = describeExecution(executionArn).jsonPath().getDouble("stopDate");++ Thread.sleep((WAIT_SECONDS + 2) * 1000L);++ Response described = describeExecution(executionArn);+ assertEquals("ABORTED", described.jsonPath().getString("status"));+ assertEquals("Manual", described.jsonPath().getString("error"));+ assertEquals("probe", described.jsonPath().getString("cause"));+ assertEquals(stopDate, described.jsonPath().getDouble("stopDate"));+ }++ @Test+ void abortedExecutionEndsItsHistoryAtExecutionAborted() throws Exception {+ String executionArn = startWaitingExecution("stop-ends-history");+ Thread.sleep(300);++ stopExecution(executionArn, "Manual", "probe");++ // The worker is still inside the Wait. Let it come out and try to record the state it left.+ Thread.sleep((WAIT_SECONDS + 2) * 1000L);++ List<Map<String, Object>> events = getExecutionHistory(executionArn).jsonPath().getList("events");+ List<String> types = events.stream().map(event -> (String) event.get("type")).toList();+ List<Object> ids = events.stream().map(event -> event.get("id")).toList();++ assertAll(+ () -> assertEquals("ExecutionAborted", types.get(types.size() - 1),+ "history continued past the terminal event: " + types),+ () -> assertEquals(ids.size(), new HashSet<>(ids).size(),+ "history repeats an event id: " + ids + " for " + types));+ }++ private String startWaitingExecution(String namePrefix) {+ String name = namePrefix + "-" + UUID.randomUUID();+ String stateMachineArn = createStateMachine(name, WAITING_MACHINE);+ Response response = given()+ .header("X-Amz-Target", "AWSStepFunctions.StartExecution")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"stateMachineArn": "%s", "input": "{}"}+ """.formatted(stateMachineArn))+ .when()+ .post("/");+ response.then().statusCode(200);+ return response.jsonPath().getString("executionArn");+ }++ private String createStateMachine(String name, String definition) {+ Response response = given()+ .header("X-Amz-Target", "AWSStepFunctions.CreateStateMachine")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"name": "%s", "definition": %s, "roleArn": "%s"}+ """.formatted(name, quote(definition), ROLE_ARN))+ .when()+ .post("/");+ response.then().statusCode(200);+ return response.jsonPath().getString("stateMachineArn");+ }++ private void stopExecution(String executionArn, String error, String cause) {+ given()+ .header("X-Amz-Target", "AWSStepFunctions.StopExecution")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"executionArn": "%s", "error": "%s", "cause": "%s"}+ """.formatted(executionArn, error, cause))+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ private Response describeExecution(String executionArn) {+ Response response = given()+ .header("X-Amz-Target", "AWSStepFunctions.DescribeExecution")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"executionArn": "%s"}+ """.formatted(executionArn))+ .when()+ .post("/");+ response.then().statusCode(200);+ return response;+ }++ private Response getExecutionHistory(String executionArn) {+ Response response = given()+ .header("X-Amz-Target", "AWSStepFunctions.GetExecutionHistory")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"executionArn": "%s", "includeExecutionData": true}+ """.formatted(executionArn))+ .when()+ .post("/");+ response.then().statusCode(200);+ return response;+ }++ private static String quote(String raw) {+ return "\"" + raw+ .replace("\\", "\\\\")+ .replace("\"", "\\\"")+ .replace("\n", "\\n")+ .replace("\r", "\\r")+ .replace("\t", "\\t")+ + "\"";+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsTimeoutSecondsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsTimeoutSecondsIntegrationTest.javanew file mode 100644index 0000000000..746be51b18--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsTimeoutSecondsIntegrationTest.java@@ -0,0 +1,553 @@+package io.github.hectorvent.floci.services.stepfunctions;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.util.ArrayList;+import java.util.List;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assertions.fail;++/**+ * The two {@code TimeoutSeconds} fields of ASL, each enforced on its own clock: the state+ * machine's total budget and a {@code Task}'s wait for its task token, the second one bounded a+ * second time by {@code HeartbeatSeconds}.+ *+ * <p>The bounds here are one to ten seconds where the report that measured them against us-east-1+ * used three and five; the shape of the terminal execution is what the assertions pin, not the+ * duration.+ */+@QuarkusTest+class StepFunctionsTimeoutSecondsIntegrationTest {++ private static final String SFN_CONTENT_TYPE = "application/x-amz-json-1.0";+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/test-role";+ private static final ObjectMapper mapper = new ObjectMapper();++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void stateMachineTimeoutSecondsCutsAWaitLongerThanTheRemainingBudget() throws Exception {+ String smArn = createStateMachine("top-level-timeout-wait", """+ {+ "StartAt": "Linger",+ "TimeoutSeconds": 1,+ "States": {+ "Linger": {"Type": "Wait", "Seconds": 3, "End": true}+ }+ }+ """);++ Response timedOut = waitForTerminalExecution(startExecution(smArn, "{}"));++ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"));+ JsonNode body = mapper.readTree(timedOut.body().asString());+ assertTrue(body.has("stopDate"), "stopDate must be set on a timed out execution");+ assertFalse(body.has("error"), "TIMED_OUT carries no error key: " + body);+ assertFalse(body.has("cause"), "TIMED_OUT carries no cause key: " + body);+ }++ @Test+ void timedOutExecutionEmitsExecutionTimedOutAndNoStateExitedForTheCutState() throws Exception {+ String smArn = createStateMachine("top-level-timeout-history", """+ {+ "StartAt": "Linger",+ "TimeoutSeconds": 1,+ "States": {+ "Linger": {"Type": "Wait", "Seconds": 3, "End": true}+ }+ }+ """);++ String execArn = startExecution(smArn, "{}");+ waitForTerminalExecution(execArn);+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");++ assertEquals(List.of("ExecutionStarted", "WaitStateEntered", "ExecutionTimedOut"),+ eventTypes(events), "history was " + events);+ JsonNode timedOut = events.get(events.size() - 1);+ assertEquals(0, timedOut.path("previousEventId").asLong());+ assertEquals("States.Timeout",+ timedOut.path("executionTimedOutEventDetails").path("error").asText());+ assertFalse(timedOut.path("executionTimedOutEventDetails").has("cause"),+ "ExecutionTimedOut carries only the error: " + timedOut);+ }++ @Test+ void stateMachineTimeoutSecondsCutsTheWaitForAParallelBranch() throws Exception {+ // A branch is never cut mid-state, so its Wait sleeps out in full on its own thread. What+ // the budget cuts is the join: the execution ends while the branch is still running, and+ // the Parallel state gets no Exited event and no state after it is entered.+ String smArn = createStateMachine("top-level-timeout-loop", """+ {+ "StartAt": "Fork",+ "TimeoutSeconds": 1,+ "States": {+ "Fork": {+ "Type": "Parallel",+ "Branches": [{+ "StartAt": "Linger",+ "States": {"Linger": {"Type": "Wait", "Seconds": 3, "End": true}}+ }],+ "Next": "Unreached"+ },+ "Unreached": {"Type": "Pass", "End": true}+ }+ }+ """);++ String execArn = startExecution(smArn, "{}");+ long startedAt = System.currentTimeMillis();+ Response timedOut = waitForTerminalExecution(execArn);+ long elapsedMillis = System.currentTimeMillis() - startedAt;++ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"));+ assertTrue(elapsedMillis < 3_000,+ "the branch's 3-second Wait outlasted the 1-second budget: " + elapsedMillis + " ms");+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");+ assertEquals(List.of("ExecutionStarted", "ParallelStateEntered", "ExecutionTimedOut"),+ eventTypes(events), "history was " + events);+ }++ @Test+ void stateMachineTimeoutSecondsCutsTheWaitForATaskToken() throws Exception {+ // Measured against us-east-1: a state machine whose budget is shorter than the Task's own+ // TimeoutSeconds ends TIMED_OUT the instant the budget runs out, and writes nothing about+ // the task it cut. No ActivityTimedOut, no TaskFailed, no ExecutionFailed.+ String activityArn = createActivity("top-level-timeout-token");+ String smArn = createStateMachine("top-level-timeout-token", """+ {+ "StartAt": "Await",+ "TimeoutSeconds": 1,+ "States": {+ "Await": {+ "Type": "Task",+ "Resource": "%s",+ "TimeoutSeconds": 60,+ "End": true+ }+ }+ }+ """.formatted(activityArn));++ String execArn = startExecution(smArn, "{}");+ long startedAt = System.currentTimeMillis();+ Response timedOut = waitForTerminalExecution(execArn);+ long elapsedMillis = System.currentTimeMillis() - startedAt;++ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"), timedOut.body().asString());+ assertTrue(elapsedMillis < 5_000,+ "the task's 60-second wait outlasted the 1-second budget: " + elapsedMillis + " ms");+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");+ assertEquals(List.of("ExecutionStarted", "TaskStateEntered", "ActivityScheduled",+ "ActivityStarted", "ExecutionTimedOut"), eventTypes(events), "history was " + events);+ }++ @Test+ void stateMachineTimeoutSecondsCutsTheWaitForANestedExecution() throws Exception {+ // The child sleeps far past the parent's budget. The parent ends TIMED_OUT where its budget+ // runs out; the child keeps running, since nothing in ASL propagates the parent's timeout.+ String childArn = createStateMachine("nested-child", """+ {+ "StartAt": "Linger",+ "States": {"Linger": {"Type": "Wait", "Seconds": 10, "End": true}}+ }+ """);+ String parentArn = createStateMachine("nested-parent", """+ {+ "StartAt": "RunChild",+ "TimeoutSeconds": 1,+ "States": {+ "RunChild": {+ "Type": "Task",+ "Resource": "arn:aws:states:::states:startExecution.sync",+ "Parameters": {"StateMachineArn": "%s", "Input": {}},+ "End": true+ }+ }+ }+ """.formatted(childArn));++ String execArn = startExecution(parentArn, "{}");+ long startedAt = System.currentTimeMillis();+ Response timedOut = waitForTerminalExecution(execArn);+ long elapsedMillis = System.currentTimeMillis() - startedAt;++ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"), timedOut.body().asString());+ assertTrue(elapsedMillis < 5_000,+ "the child's 10-second Wait outlasted the 1-second budget: " + elapsedMillis + " ms");+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");+ assertEquals("ExecutionTimedOut", events.get(events.size() - 1).path("type").asText(),+ "history was " + events);+ }++ @Test+ void stateMachineTimeoutSecondsCutsTheWaitForMapIterations() throws Exception {+ // Two iterations run concurrently and each sleeps past the budget, so what the budget cuts+ // is the Map state's wait for them rather than any single iteration.+ String smArn = createStateMachine("top-level-timeout-map", """+ {+ "StartAt": "OverItems",+ "TimeoutSeconds": 1,+ "States": {+ "OverItems": {+ "Type": "Map",+ "ItemsPath": "$.items",+ "ItemProcessor": {+ "StartAt": "Linger",+ "States": {"Linger": {"Type": "Wait", "Seconds": 10, "End": true}}+ },+ "End": true+ }+ }+ }+ """);++ String execArn = startExecution(smArn, "{\"items\": [1, 2]}");+ long startedAt = System.currentTimeMillis();+ Response timedOut = waitForTerminalExecution(execArn);+ long elapsedMillis = System.currentTimeMillis() - startedAt;++ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"), timedOut.body().asString());+ assertTrue(elapsedMillis < 5_000,+ "the iterations' 10-second Wait outlasted the 1-second budget: " + elapsedMillis + " ms");+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");+ assertEquals(List.of("ExecutionStarted", "MapStateEntered", "ExecutionTimedOut"),+ eventTypes(events), "history was " + events);+ }++ @Test+ void stateMachineTimeoutSecondsCutsTheWaitForASingleMapIteration() throws Exception {+ // A single item runs the Map state's serial path rather than the concurrent one; the+ // budget must cut it the same way it cuts a concurrent iteration above.+ String smArn = createStateMachine("top-level-timeout-map-single-item", """+ {+ "StartAt": "OverItems",+ "TimeoutSeconds": 1,+ "States": {+ "OverItems": {+ "Type": "Map",+ "ItemsPath": "$.items",+ "ItemProcessor": {+ "StartAt": "Linger",+ "States": {"Linger": {"Type": "Wait", "Seconds": 10, "End": true}}+ },+ "End": true+ }+ }+ }+ """);++ String execArn = startExecution(smArn, "{\"items\": [1]}");+ long startedAt = System.currentTimeMillis();+ Response timedOut = waitForTerminalExecution(execArn);+ long elapsedMillis = System.currentTimeMillis() - startedAt;++ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"), timedOut.body().asString());+ assertTrue(elapsedMillis < 5_000,+ "the single iteration's 10-second Wait outlasted the 1-second budget: " + elapsedMillis + " ms");+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");+ assertEquals(List.of("ExecutionStarted", "MapStateEntered", "ExecutionTimedOut"),+ eventTypes(events), "history was " + events);+ }++ @Test+ void stateMachineTimeoutSecondsCutsARetryBackoffLongerThanTheRemainingBudget() throws Exception {+ // The Lambda does not exist, so every attempt fails immediately and the only thing this+ // state spends time on is the backoff between attempts. AWS ends the execution when the+ // budget runs out, mid-backoff; before this the backoff ran in full and the state failed+ // on its own error afterwards, so the execution ended FAILED past its budget.+ String smArn = createStateMachine("top-level-timeout-retry", """+ {+ "StartAt": "Flaky",+ "TimeoutSeconds": 1,+ "States": {+ "Flaky": {+ "Type": "Task",+ "Resource": "arn:aws:states:::lambda:invoke",+ "Parameters": {"FunctionName": "absent-function"},+ "Retry": [{"ErrorEquals": ["States.ALL"], "IntervalSeconds": 10, "MaxAttempts": 1}],+ "End": true+ }+ }+ }+ """);++ String execArn = startExecution(smArn, "{}");+ long startedAt = System.currentTimeMillis();+ Response timedOut = waitForTerminalExecution(execArn);+ long elapsedMillis = System.currentTimeMillis() - startedAt;++ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"), timedOut.body().asString());+ assertTrue(elapsedMillis < 5_000,+ "the 10-second backoff outlasted the 1-second budget: " + elapsedMillis + " ms");+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");+ assertEquals("ExecutionTimedOut", events.get(events.size() - 1).path("type").asText(),+ "history was " + events);+ }++ @Test+ void taskTimeoutSecondsBoundsTheWholeWaitForTheTaskToken() throws Exception {+ String activityArn = createActivity("timeout-seconds-total");+ String smArn = createStateMachine("task-timeout-seconds", activityTask(activityArn, 1, 0));++ Response failed = waitForTerminalExecution(startExecution(smArn, "{}"));++ assertEquals("FAILED", failed.jsonPath().getString("status"));+ assertEquals("States.Timeout", failed.jsonPath().getString("error"));+ JsonNode body = mapper.readTree(failed.body().asString());+ assertFalse(body.has("cause"),+ "a task that ran out of time carries no cause key: " + body);+ }++ @Test+ void taskHeartbeatSecondsBoundsTheGapBetweenHeartbeats() throws Exception {+ String activityArn = createActivity("heartbeat-seconds-gap");+ String smArn = createStateMachine("task-heartbeat-gap", activityTask(activityArn, 10, 1));+ String execArn = startExecution(smArn, "{}");++ // A worker picks the task up and then goes silent, so the gap HeartbeatSeconds allows runs+ // out long before the state's TimeoutSeconds does.+ getActivityTask(activityArn);+ Response failed = waitForTerminalExecution(execArn);++ assertEquals("FAILED", failed.jsonPath().getString("status"));+ assertEquals("States.Timeout", failed.jsonPath().getString("error"));+ JsonNode body = mapper.readTree(failed.body().asString());+ assertFalse(body.has("cause"),+ "a task that missed its heartbeats carries no cause key: " + body);++ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");+ assertEquals(List.of("ExecutionStarted", "TaskStateEntered", "ActivityScheduled",+ "ActivityStarted", "ActivityTimedOut", "ExecutionFailed"),+ eventTypes(events), "history was " + events);+ JsonNode timedOut = events.get(events.size() - 2);+ assertEquals("States.Timeout",+ timedOut.path("activityTimedOutEventDetails").path("error").asText());+ assertFalse(timedOut.path("activityTimedOutEventDetails").has("cause"),+ "ActivityTimedOut carries only the error: " + timedOut);+ }++ @Test+ void aCatchMatchesAHeartbeatExpiryUnderEitherErrorName() throws Exception {+ for (String errorEquals : List.of("States.HeartbeatTimeout", "States.Timeout")) {+ String activityArn = createActivity("heartbeat-seconds-catch");+ String smArn = createStateMachine("task-heartbeat-catch",+ caughtActivityTask(activityArn, errorEquals));+ String execArn = startExecution(smArn, "{}");++ getActivityTask(activityArn);+ Response caught = waitForTerminalExecution(execArn);++ assertEquals("SUCCEEDED", caught.jsonPath().getString("status"),+ "Catch on " + errorEquals + " left the execution " + caught.body().asString());+ assertEquals("States.Timeout",+ mapper.readTree(caught.jsonPath().getString("output")).path("Error").asText(),+ "Catch on " + errorEquals + " reported the wrong error");+ }+ }++ @Test+ void heartbeatsKeepATaskAliveUntilItsTimeoutSecondsRunOut() throws Exception {+ String activityArn = createActivity("heartbeat-seconds-reset");+ String smArn = createStateMachine("task-heartbeat-reset", activityTask(activityArn, 12, 3));+ String execArn = startExecution(smArn, "{}");++ String taskToken = getActivityTask(activityArn);+ // Five seconds of work reported every 500 ms: nearly twice the heartbeat gap, so the task+ // survives only if each heartbeat pushes that gap's deadline forward.+ for (int beat = 0; beat < 10; beat++) {+ Thread.sleep(500);+ sendTaskHeartbeat(taskToken);+ }+ sendTaskSuccess(taskToken, "{\"answered\":true}");++ Response succeeded = waitForTerminalExecution(execArn);+ assertEquals("SUCCEEDED", succeeded.jsonPath().getString("status"),+ "execution was " + succeeded.body().asString());+ assertTrue(mapper.readTree(succeeded.jsonPath().getString("output")).path("answered").asBoolean());+ }++ private static String activityTask(String activityArn, int timeoutSeconds, int heartbeatSeconds) {+ String heartbeat = heartbeatSeconds > 0 ? ",\"HeartbeatSeconds\": " + heartbeatSeconds : "";+ return """+ {+ "StartAt": "Await",+ "States": {+ "Await": {+ "Type": "Task",+ "Resource": "%s",+ "TimeoutSeconds": %d%s,+ "End": true+ }+ }+ }+ """.formatted(activityArn, timeoutSeconds, heartbeat);+ }++ private static String caughtActivityTask(String activityArn, String errorEquals) {+ return """+ {+ "StartAt": "Await",+ "States": {+ "Await": {+ "Type": "Task",+ "Resource": "%s",+ "TimeoutSeconds": 10,+ "HeartbeatSeconds": 1,+ "Catch": [{"ErrorEquals": ["%s"], "Next": "Recovered"}],+ "End": true+ },+ "Recovered": {"Type": "Pass", "End": true}+ }+ }+ """.formatted(activityArn, errorEquals);+ }++ private static List<String> eventTypes(JsonNode events) {+ List<String> types = new ArrayList<>();+ events.forEach(event -> types.add(event.path("type").asText()));+ return types;+ }++ private static String createActivity(String name) {+ Response resp = given()+ .header("X-Amz-Target", "AWSStepFunctions.CreateActivity")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"name": "%s-%d"}+ """.formatted(name, System.currentTimeMillis()))+ .when()+ .post("/");+ resp.then().statusCode(200);+ return resp.jsonPath().getString("activityArn");+ }++ private static String getActivityTask(String activityArn) {+ Response resp = given()+ .header("X-Amz-Target", "AWSStepFunctions.GetActivityTask")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"activityArn": "%s", "workerName": "timeout-seconds-worker"}+ """.formatted(activityArn))+ .when()+ .post("/");+ resp.then().statusCode(200);+ String taskToken = resp.jsonPath().getString("taskToken");+ assertFalse(taskToken == null || taskToken.isBlank(), "no activity task was queued");+ return taskToken;+ }++ private static void sendTaskHeartbeat(String taskToken) {+ given()+ .header("X-Amz-Target", "AWSStepFunctions.SendTaskHeartbeat")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"taskToken": %s}+ """.formatted(quote(taskToken)))+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ private static void sendTaskSuccess(String taskToken, String output) {+ given()+ .header("X-Amz-Target", "AWSStepFunctions.SendTaskSuccess")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"taskToken": %s, "output": %s}+ """.formatted(quote(taskToken), quote(output)))+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ private static String createStateMachine(String name, String definition) {+ Response resp = given()+ .header("X-Amz-Target", "AWSStepFunctions.CreateStateMachine")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"name": "%s-%d", "definition": %s, "roleArn": "%s"}+ """.formatted(name, System.currentTimeMillis(), quote(definition), ROLE_ARN))+ .when()+ .post("/");+ resp.then().statusCode(200);+ return resp.jsonPath().getString("stateMachineArn");+ }++ private static String startExecution(String smArn, String input) {+ Response resp = given()+ .header("X-Amz-Target", "AWSStepFunctions.StartExecution")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"stateMachineArn": "%s", "input": %s}+ """.formatted(smArn, quote(input)))+ .when()+ .post("/");+ resp.then().statusCode(200);+ return resp.jsonPath().getString("executionArn");+ }++ private static Response describeExecution(String execArn) {+ return given()+ .header("X-Amz-Target", "AWSStepFunctions.DescribeExecution")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"executionArn": "%s"}+ """.formatted(execArn))+ .when()+ .post("/");+ }++ private static Response getExecutionHistory(String execArn) {+ return given()+ .header("X-Amz-Target", "AWSStepFunctions.GetExecutionHistory")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"executionArn": "%s"}+ """.formatted(execArn))+ .when()+ .post("/");+ }++ /**+ * Polls for 20 seconds: twice the longest bound any definition here declares, and far short of+ * the 300-second default a task that ignores TimeoutSeconds falls back on.+ */+ private static Response waitForTerminalExecution(String execArn) throws InterruptedException {+ for (int poll = 0; poll < 200; poll++) {+ Response resp = describeExecution(execArn);+ String status = resp.jsonPath().getString("status");+ if (!"RUNNING".equals(status)) {+ return resp;+ }+ Thread.sleep(100);+ }+ fail("Execution " + execArn + " was still RUNNING after 20 seconds");+ return null;+ }++ private static String quote(String raw) {+ return "\"" + raw+ .replace("\\", "\\\\")+ .replace("\"", "\\\"")+ .replace("\n", "\\n")+ .replace("\r", "\\r")+ .replace("\t", "\\t")+ + "\"";+ }+}