Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(stepfunctions): bound a running execution the four ways AWS does (#2782)

Facundo Paredes92887696074b
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.md
index 0f5a366c5c..f06d0e0e6f 100644
--- a/docs/services/step-functions.md
+++ b/docs/services/step-functions.md
@@ -111,6 +111,37 @@ uniformly between zero and the computed delay, as on AWS. One deviation. The del
between attempts is capped at 30 seconds, the same cap Floci applies to `Wait` states,
so emulated runs stay fast.
+## Timeouts
+
+ASL carries two `TimeoutSeconds` fields and Floci enforces both, in the two terminal shapes
+AWS uses. The state machine's own field bounds every state; a `Task`'s own field only bounds
+one that waits for a task token — an activity, or a `.waitForTaskToken` integration. A Lambda
+or other SDK task that returns directly is not bound by it.
+
+The state machine's own `TimeoutSeconds` is the whole execution's budget. It is checked before
+every state and inside a `Wait`, so a `Wait` longer than what is left is cut rather than slept
+out. The execution ends `TIMED_OUT` with `stopDate` set and no `error` and no `cause` at all;
+`States.Timeout` is named only in the single `ExecutionTimedOut` event, whose `previousEventId`
+is `0`. The state that was cut gets no `*StateExited` event. A `Parallel` or `Map` branch runs
+on its own thread and is not cut mid-state: the budget is enforced again as soon as the branch
+returns.
+
+A `Task` that waits for a task token — an activity, or a `.waitForTaskToken` integration — has
+two independent bounds. `TimeoutSeconds` is the whole wait; `HeartbeatSeconds` is the longest gap
+allowed between two `SendTaskHeartbeat` calls, and every heartbeat pushes that gap forward, so a
+worker that reports as often as the definition asks runs until `TimeoutSeconds` runs out. Either
+clock ends the state the same way: an `ActivityTimedOut` event — `TaskTimedOut` for a
+`.waitForTaskToken` integration — carrying `States.Timeout` and no cause, then an execution that
+reads `FAILED` with that same error. A `Catch` on a heartbeat expiry matches under
+`States.HeartbeatTimeout` and under `States.Timeout` alike. Neither timeout carries a cause:
+`DescribeExecution` and the `ExecutionFailed` event both leave the key out, where every other
+failure reports one. A `Task` that declares no `TimeoutSeconds` waits 300 seconds, where AWS
+waits a year.
+
+One deviation. AWS starts the `TimeoutSeconds` clock when a worker picks the task up, the instant
+it emits `ActivityStarted`. Floci emits `ActivityStarted` at schedule time, so both clocks start
+when the task is scheduled.
+
## JSONata nulls
An expression that evaluates to JSON `null` produces a value, not a missing one. It keeps its key
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index d8aad8aedb..997bb5e5a1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -108,6 +108,19 @@ public class AslExecutor {
private static final Logger LOG = Logger.getLogger(AslExecutor.class);
private static final int MAX_WAIT_SECONDS = 30;
+ // How long a Task waits for its token when the state declares no TimeoutSeconds. AWS lets it
+ // run for a year; the emulator would rather free the worker thread.
+ private static final int DEFAULT_TASK_TOKEN_TIMEOUT_SECONDS = 300;
+
+ /**
+ * AWS ends an execution once its history reaches this many events. The count is neither reset
+ * nor offset: the event that ends the execution is number 25,000 itself, so the last event the
+ * state machine produced is 24,999.
+ */
+ private static final int MAX_HISTORY_EVENTS = 25_000;
+ private static final String HISTORY_EVENT_LIMIT_CAUSE =
+ "The execution reached the maximum number of history events (" + MAX_HISTORY_EVENTS + ").";
+
private static final int INLINE_MAP_MAX_CONCURRENCY = 40;
private static final int DISTRIBUTED_MAP_MAX_CONCURRENCY = 10_000;
@@ -354,9 +367,10 @@ public class AslExecutor {
private void doExecute(StateMachine sm, Execution exec, List<HistoryEvent> history,
BiConsumer<Execution, List<HistoryEvent>> onUpdate) {
- // Declared outside the try so the terminal handlers below can keep numbering the history
- // where the execution left it.
- AtomicLong eventId = new AtomicLong(history.size());
+ // Shared with every Parallel branch and every inline Map iteration of this execution: the
+ // 25,000-event limit is the execution's, not the thread's. It starts where the history
+ // already is, because ExecutionStarted is an event of this execution too.
+ AtomicLong producedEventCount = new AtomicLong(history.size());
var firstState = true;
try {
JsonNode definition = objectMapper.readTree(sm.getDefinition());
@@ -364,20 +378,30 @@ public class AslExecutor {
String startAt = definition.path("StartAt").asText();
String topLevelQueryLanguage = definition.path("QueryLanguage").asText("JSONPath");
JsonNode currentInput = parseInput(exec.getInput());
+ // The state machine's total budget, computed once so every state measures against the
+ // same instant. Long.MAX_VALUE stands for a definition with no TimeoutSeconds.
+ int timeoutSeconds = definition.path("TimeoutSeconds").asInt(0);
+ long executionDeadlineNanos = timeoutSeconds > 0
+ ? System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds)
+ : Long.MAX_VALUE;
JsonNode execContext = buildContext(exec, sm);
// Execution-scoped JSONata variables (the Assign field). Mutated in place as states
// assign, so later states observe earlier assignments.
ObjectNode variables = objectMapper.createObjectNode();
String currentStateName = startAt;
- while (currentStateName != null) {
+ while (currentStateName != null && !abortedByCaller(exec)) {
+ if (System.nanoTime() >= executionDeadlineNanos) {
+ throw new ExecutionTimedOutException();
+ }
JsonNode stateDef = states.path(currentStateName);
if (stateDef.isMissingNode()) {
throw new RuntimeException("State not found: " + currentStateName);
}
String type = stateDef.path("Type").asText();
- addEvent(history, eventId, stateEnteredEventType(type), firstState ? 0L : eventId.get(),
+ publishStateEnteredEvent(history, producedEventCount, stateEnteredEventType(type),
+ firstState ? 0L : history.size(),
Map.of("name", currentStateName, "input", currentInput.toString(),
"inputDetails", Map.of("truncated", false)));
firstState = false;
@@ -388,8 +412,9 @@ public class AslExecutor {
var jsonata = isJsonata(stateDef, topLevelQueryLanguage);
try {
var stateResult = executeStateWithRetry(currentStateName, type, stateDef, currentInput,
- history, eventId, sm, jsonata, topLevelQueryLanguage, execContext, variables);
- addEvent(history, eventId, stateExitedEventType(type),
+ history, producedEventCount, sm, jsonata, topLevelQueryLanguage, execContext,
+ variables, executionDeadlineNanos);
+ publishEvent(history, producedEventCount, stateExitedEventType(type),
Map.of("name", currentStateName, "output", stateResult.output().toString(),
"outputDetails", Map.of("truncated", false)));
@@ -411,35 +436,36 @@ public class AslExecutor {
failure = catchClauseFailure;
}
if (caught != null) {
- addEvent(history, eventId, stateExitedEventType(type),
+ publishEvent(history, producedEventCount, stateExitedEventType(type),
Map.of("name", currentStateName, "output", caught.output().toString(),
"outputDetails", Map.of("truncated", false)));
currentInput = caught.output();
currentStateName = caught.nextState();
continue;
}
- failExecution(exec, history, eventId, failure);
+ failExecution(exec, history, failure);
onUpdate.accept(exec, history);
return;
}
}
- // Status is the publication point, so it is set last. describeExecution hands out this
- // same live Execution, so a client polling for SUCCEEDED between setStatus and setOutput
- // would read a terminal execution with a null output, which real Step Functions never
- // returns. The same ordering applies to every terminal path below.
- exec.setOutput(currentInput.toString());
- exec.setStopDate(System.currentTimeMillis() / 1000.0);
- exec.setStatus("SUCCEEDED");
- addEvent(history, eventId, "ExecutionSucceeded",
- Map.of("output", currentInput.toString(), "outputDetails", Map.of("truncated", false)));
+ succeedExecution(exec, history, currentInput);
onUpdate.accept(exec, history);
+ } catch (ExecutionTimedOutException e) {
+ timeOutExecution(exec, history);
+ onUpdate.accept(exec, history);
+ } catch (FailStateException e) {
+ // A state's own failure is handled inside the loop, where its Catch clauses apply. What
+ // reaches here is a failure raised while recording a state's entered event, outside the
+ // per-state try: the execution hit the history-event limit.
+ failExecution(exec, history, e);
+ onUpdate.accept(exec, history);
} catch (Exception e) {
LOG.warnv("ASL execution failed for {0}: {1}", exec.getExecutionArn(), e.getMessage());
// This path previously set only the status, leaving error and cause null forever on an
// execution DescribeExecution reports as FAILED.
- failExecution(exec, history, eventId, "States.Runtime",
+ failExecution(exec, history, "States.Runtime",
e.getMessage() != null ? e.getMessage() : "Unknown error");
onUpdate.accept(exec, history);
} catch (Error e) {
@@ -449,7 +475,7 @@ public class AslExecutor {
// and the rethrow keeps the Error itself from being swallowed here. The cause carries
// toString() rather than getMessage(), because an Error's message is often null and
// the type name is the whole diagnostic.
- failExecution(exec, history, eventId, "States.Runtime", e.toString());
+ failExecution(exec, history, "States.Runtime", e.toString());
onUpdate.accept(exec, history);
throw e;
} finally {
@@ -470,16 +496,17 @@ public class AslExecutor {
* caller's Catch handling, preserving Retry-before-Catch order.
*/
private StateResult executeStateWithRetry(String name, String type, JsonNode stateDef, JsonNode input,
- List<HistoryEvent> history, AtomicLong eventId, StateMachine sm,
- boolean jsonata, String topLevelQueryLanguage, JsonNode context,
- ObjectNode variables) throws Exception {
+ List<HistoryEvent> history, AtomicLong producedEventCount,
+ StateMachine sm, boolean jsonata, String topLevelQueryLanguage,
+ JsonNode context, ObjectNode variables,
+ long executionDeadlineNanos) throws Exception {
var retriers = stateDef.path("Retry");
var attemptsPerRetrier = new HashMap<Integer, Integer>();
var attempt = 0;
while (true) {
try {
- return executeState(name, type, stateDef, input, history, eventId, sm, jsonata,
- topLevelQueryLanguage, context, variables, attempt);
+ return executeState(name, type, stateDef, input, history, producedEventCount, sm, jsonata,
+ topLevelQueryLanguage, context, variables, attempt, executionDeadlineNanos);
} catch (FailStateException e) {
var retrierIndex = findMatchingRetrier(retriers, e);
if (retrierIndex < 0) {
@@ -490,7 +517,7 @@ public class AslExecutor {
if (attemptsUsed > retrier.path("MaxAttempts").asInt(3)) {
throw e;
}
- sleepBeforeRetry(retrier, attemptsUsed);
+ sleepBeforeRetry(retrier, attemptsUsed, executionDeadlineNanos);
attempt++;
updateRetryCount(context, attempt);
}
@@ -501,20 +528,25 @@ public class AslExecutor {
if (!retriers.isArray()) {
return -1;
}
- var error = failure.error != null ? failure.error : "States.Runtime";
for (var i = 0; i < retriers.size(); i++) {
- if (catchMatches(retriers.get(i), error)) {
+ if (catchMatches(retriers.get(i), failure)) {
return i;
}
}
return -1;
}
- private void sleepBeforeRetry(JsonNode retrier, int attemptsUsed) throws InterruptedException {
+ /**
+ * Backs off before the next attempt. The backoff is a pause inside the state, so a retrier
+ * whose interval outlasts the state machine's {@code TimeoutSeconds} budget ends the execution
+ * where the budget runs out rather than attempting again past it, which is what AWS does. The
+ * deadline is read even when the delay is zero, so a state that already spent the budget stops
+ * instead of retrying instantly.
+ */
+ private void sleepBeforeRetry(JsonNode retrier, int attemptsUsed, long executionDeadlineNanos)
+ throws InterruptedException {
var delaySeconds = retryDelaySeconds(retrier, attemptsUsed, ThreadLocalRandom.current().nextDouble());
- if (delaySeconds > 0) {
- Thread.sleep((long) (delaySeconds * 1000));
- }
+ sleepOrTimeOutExecution((long) (delaySeconds * 1_000_000_000L), executionDeadlineNanos);
}
/**
@@ -542,19 +574,22 @@ public class AslExecutor {
}
private StateResult executeState(String name, String type, JsonNode stateDef, JsonNode input,
- List<HistoryEvent> history, AtomicLong eventId, StateMachine sm,
- boolean jsonata, String topLevelQueryLanguage, JsonNode context,
- ObjectNode variables, int attempt) throws Exception {
+ List<HistoryEvent> history, AtomicLong producedEventCount,
+ StateMachine sm, boolean jsonata, String topLevelQueryLanguage,
+ JsonNode context, ObjectNode variables, int attempt,
+ long executionDeadlineNanos) throws Exception {
return switch (type) {
case "Pass" -> executePassState(stateDef, input, jsonata, context, variables);
- case "Task" -> executeTaskState(name, stateDef, input, history, eventId, sm, jsonata, context,
- variables, attempt);
+ case "Task" -> executeTaskState(name, stateDef, input, history, producedEventCount, sm,
+ jsonata, context, variables, attempt, executionDeadlineNanos);
case "Choice" -> executeChoiceState(stateDef, input, jsonata, context, variables);
- case "Wait" -> executeWaitState(stateDef, input, jsonata, context, variables);
+ case "Wait" -> executeWaitState(stateDef, input, jsonata, context, variables, executionDeadlineNanos);
case "Succeed" -> executeSucceedState(stateDef, input, jsonata, context, variables);
case "Fail" -> executeFail(stateDef, input, jsonata, context, variables);
- case "Parallel" -> executeParallelState(name, stateDef, input, sm, jsonata, topLevelQueryLanguage, context, variables);
- case "Map" -> executeMapState(name, stateDef, input, sm, jsonata, topLevelQueryLanguage, context, variables);
+ case "Parallel" -> executeParallelState(name, stateDef, input, producedEventCount, sm, jsonata,
+ topLevelQueryLanguage, context, variables, executionDeadlineNanos);
+ case "Map" -> executeMapState(name, stateDef, input, producedEventCount, sm, jsonata,
+ topLevelQueryLanguage, context, variables, executionDeadlineNanos);
default -> new StateResult(input, stateDef.path("Next").asText(null));
};
}
@@ -585,9 +620,10 @@ public class AslExecutor {
}
private StateResult executeTaskState(String stateName, JsonNode stateDef, JsonNode input,
- List<HistoryEvent> history, AtomicLong eventId, StateMachine sm,
- boolean jsonata, JsonNode context, ObjectNode variables,
- int attempt) throws Exception {
+ List<HistoryEvent> history, AtomicLong producedEventCount,
+ StateMachine sm, boolean jsonata, JsonNode context,
+ ObjectNode variables, int attempt,
+ long executionDeadlineNanos) throws Exception {
var resource = stateDef.path("Resource").asText();
var isWaitForToken = resource.endsWith(".waitForTaskToken");
var effectiveResource = isWaitForToken
@@ -623,25 +659,44 @@ public class AslExecutor {
}
var profile = taskEventProfile(resource, isActivity);
- addTaskScheduledEvent(history, eventId, profile, stateDef, effectiveInput, sm);
- addTaskStartedEvent(history, eventId, profile);
-
JsonNode taskResult;
try {
- taskResult = mockedSteps != null
- ? mockedTaskResult(mockedSteps, stateName, attempt)
- : invokeResource(effectiveResource, effectiveInput, sm, taskToken);
- if (tokenFuture != null) {
- taskResult = awaitToken(tokenFuture, stateDef);
+ addTaskScheduledEvent(history, producedEventCount, profile, stateDef, effectiveInput, sm);
+ addTaskStartedEvent(history, producedEventCount, profile);
+ try {
+ taskResult = mockedSteps != null
+ ? mockedTaskResult(mockedSteps, stateName, attempt)
+ : invokeResource(effectiveResource, effectiveInput, sm, taskToken, executionDeadlineNanos);
+ if (tokenFuture != null) {
+ taskResult = awaitToken(tokenFuture, stateDef, taskToken, executionDeadlineNanos);
+ }
+ } catch (ExecutionTimedOutException e) {
+ // The state machine's TimeoutSeconds budget ran out while this task was waiting. AWS
+ // ends the execution there and writes nothing about the state it cut: the history of
+ // a task still waiting on its token is ExecutionStarted, TaskStateEntered,
+ // ActivityScheduled, ExecutionTimedOut, with no TaskFailed and no TaskTimedOut.
+ throw e;
+ } catch (TaskTimedOutException e) {
+ addTaskTimedOutEvent(history, producedEventCount, profile);
+ throw e;
+ } catch (Exception e) {
+ var failure = e instanceof FailStateException f ? f : null;
+ addTaskFailedEvent(history, producedEventCount, profile,
+ failure != null && failure.error != null ? failure.error : "States.Runtime",
+ failure != null ? failure.cause : e.getMessage());
+ throw e;
}
} catch (Exception e) {
- var failure = e instanceof FailStateException f ? f : null;
- addTaskFailedEvent(history, eventId, profile,
- failure != null && failure.error != null ? failure.error : "States.Runtime",
- failure != null ? failure.cause : e.getMessage());
+ // A token registered above is normally discarded by awaitToken's own finally. Anything
+ // that throws before awaitToken runs — the scheduled/started events themselves, or the
+ // resource invocation — would otherwise leave it pending forever; the discard here is a
+ // no-op once awaitToken already ran it.
+ if (needsToken) {
+ sfnService.get().discardPendingToken(taskToken);
+ }
throw e;
}
- addTaskSucceededEvent(history, eventId, profile, taskResult);
+ addTaskSucceededEvent(history, producedEventCount, profile, taskResult);
if (jsonata) {
JsonNode output = applyJsonataOutput(stateDef, input, taskResult, context, variables);
@@ -685,17 +740,51 @@ public class AslExecutor {
"No mocked response defined for attempt " + attempt + " of state '" + stateName + "'");
}
- private JsonNode awaitToken(CompletableFuture<JsonNode> future, JsonNode stateDef) throws Exception {
- int timeout = stateDef.path("HeartbeatSeconds").asInt(0);
- if (timeout <= 0) {
- timeout = 300;
+ /**
+ * Waits for the worker to answer the task token under the two independent bounds AWS enforces:
+ * {@code TimeoutSeconds} is the whole wait, and {@code HeartbeatSeconds} is the longest gap
+ * allowed between two SendTaskHeartbeat calls, each of which pushes that gap forward. Either
+ * clock ends the state as a {@link TaskTimedOutException}: the error is {@code States.Timeout}
+ * and there is no cause.
+ *
+ * <p>Both clocks start when the task is scheduled. AWS starts TimeoutSeconds when a worker
+ * picks the task up, which is the instant it emits ActivityStarted; Floci emits that event at
+ * schedule time, so there is no later instant to anchor on here.
+ */
+ private JsonNode awaitToken(CompletableFuture<JsonNode> future, JsonNode stateDef, String taskToken,
+ long executionDeadlineNanos) throws Exception {
+ int timeoutSeconds = stateDef.path("TimeoutSeconds").asInt(0);
+ if (timeoutSeconds <= 0) {
+ timeoutSeconds = DEFAULT_TASK_TOKEN_TIMEOUT_SECONDS;
}
+ int heartbeatSeconds = stateDef.path("HeartbeatSeconds").asInt(0);
+ long timeoutDeadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds);
try {
- return future.get(timeout, TimeUnit.SECONDS);
- } catch (java.util.concurrent.TimeoutException e) {
- future.cancel(true);
- throw new FailStateException("States.HeartbeatTimeout",
- "Task timed out after " + timeout + " seconds");
+ while (true) {
+ long wakeAtNanos = Math.min(executionDeadlineNanos, Math.min(timeoutDeadlineNanos,
+ heartbeatDeadlineNanos(taskToken, heartbeatSeconds)));
+ try {
+ return future.get(wakeAtNanos - System.nanoTime(), TimeUnit.NANOSECONDS);
+ } catch (java.util.concurrent.TimeoutException e) {
+ // The execution's budget is read first: when it is the clock that ran out, the
+ // execution ends as TIMED_OUT and the task's own timeout never applies.
+ if (System.nanoTime() >= executionDeadlineNanos) {
+ future.cancel(true);
+ throw new ExecutionTimedOutException();
+ }
+ if (System.nanoTime() >= timeoutDeadlineNanos) {
+ future.cancel(true);
+ throw new TaskTimedOutException("States.Timeout");
+ }
+ // Read the gap again rather than trusting the one this thread parked on: a
+ // heartbeat that landed meanwhile has already moved it past now, and the task
+ // goes back to waiting on the later deadline.
+ if (System.nanoTime() >= heartbeatDeadlineNanos(taskToken, heartbeatSeconds)) {
+ future.cancel(true);
+ throw new TaskTimedOutException("States.HeartbeatTimeout");
+ }
+ }
+ }
} catch (ExecutionException e) {
Throwable cause = e.getCause();
if (cause instanceof FailStateException fse) {
@@ -703,9 +792,21 @@ public class AslExecutor {
}
throw new FailStateException("States.TaskFailed",
cause != null ? cause.getMessage() : "Task failed");
+ } finally {
+ sfnService.get().discardPendingToken(taskToken);
}
}
+ /**
+ * When the worker's silence becomes too long: its last heartbeat plus the state's
+ * {@code HeartbeatSeconds}, or never for a state that declares none.
+ */
+ private long heartbeatDeadlineNanos(String taskToken, int heartbeatSeconds) {
+ return heartbeatSeconds > 0
+ ? sfnService.get().lastTaskHeartbeatNanos(taskToken) + TimeUnit.SECONDS.toNanos(heartbeatSeconds)
+ : Long.MAX_VALUE;
+ }
+
/**
* Extracts the Lambda function name from a reference that may be a bare name, a name with a
* version/alias qualifier (e.g. "name:$LATEST"), or a full/partial function ARN
@@ -745,7 +846,8 @@ public class AslExecutor {
CustomResourceLiveness.tokenIn(payload).ifPresent(customResourceLiveness::touch);
}
- private JsonNode invokeResource(String resource, JsonNode input, StateMachine sm, String taskToken) throws Exception {
+ private JsonNode invokeResource(String resource, JsonNode input, StateMachine sm, String taskToken,
+ long executionDeadlineNanos) throws Exception {
// Support Lambda resources: direct ARN or optimized integration
String functionName = null;
JsonNode lambdaPayload = input;
@@ -869,7 +971,7 @@ public class AslExecutor {
? ".waitForTaskToken"
: resource.substring("arn:aws:states:::ecs:runTask".length());
String region = extractRegionFromArn(sm.getStateMachineArn());
- return invokeEcsRunTask(mode, input, region);
+ return invokeEcsRunTask(mode, input, region, executionDeadlineNanos);
}
// AWS SDK service integrations: Step Functions
@@ -896,7 +998,7 @@ public class AslExecutor {
if (resource.startsWith("arn:aws:states:::states:startExecution")) {
String mode = resource.substring("arn:aws:states:::states:startExecution".length());
String region = extractRegionFromArn(sm.getStateMachineArn());
- return invokeNestedStateMachine(mode, input, region);
+ return invokeNestedStateMachine(mode, input, region, executionDeadlineNanos);
}
// Activity resource: arn:aws:states:{region}:{account}:activity:{name}
@@ -1229,7 +1331,8 @@ public class AslExecutor {
return envelope;
}
- private JsonNode invokeNestedStateMachine(String mode, JsonNode input, String region) throws Exception {
+ private JsonNode invokeNestedStateMachine(String mode, JsonNode input, String region,
+ long executionDeadlineNanos) throws Exception {
String smArn = input.path("StateMachineArn").asText(null);
if (smArn == null || smArn.isBlank()) {
throw new FailStateException("States.TaskFailed",
@@ -1250,9 +1353,10 @@ public class AslExecutor {
return result;
}
- // .sync or .sync:2 — poll until terminal
+ // .sync or .sync:2 — poll until terminal, or until the parent execution's TimeoutSeconds
+ // budget runs out, which ends the parent as TIMED_OUT and leaves the child running.
for (int i = 0; i < 600; i++) {
- Thread.sleep(100);
+ sleepOrTimeOutExecution(TimeUnit.MILLISECONDS.toNanos(100), executionDeadlineNanos);
io.github.hectorvent.floci.services.stepfunctions.model.Execution current =
sfnService.get().describeExecution(execArn);
String status = current.getStatus();
@@ -1302,7 +1406,8 @@ public class AslExecutor {
* STOPPED, or ".waitForTaskToken" to launch and let the token future carry the result
* (both ".sync" and ".waitForTaskToken" fail the state on a placement failure).
*/
- private JsonNode invokeEcsRunTask(String mode, JsonNode input, String region) throws Exception {
+ private JsonNode invokeEcsRunTask(String mode, JsonNode input, String region,
+ long executionDeadlineNanos) throws Exception {
String taskDefinition = input.path("TaskDefinition").asText(null);
if (taskDefinition == null || taskDefinition.isBlank()) {
throw new FailStateException("States.TaskFailed",
@@ -1374,7 +1479,8 @@ public class AslExecutor {
// fail the state, otherwise tasks beyond the first would run unmonitored.
List<String> taskArns = launched.stream().map(EcsTask::getTaskArn).toList();
for (int i = 0; i < ECS_SYNC_POLL_ATTEMPTS; i++) {
- Thread.sleep(ECS_SYNC_POLL_INTERVAL_MS);
+ sleepOrTimeOutExecution(TimeUnit.MILLISECONDS.toNanos(ECS_SYNC_POLL_INTERVAL_MS),
+ executionDeadlineNanos);
List<EcsTask> described = ecsService.describeTasks(cluster, taskArns, region);
boolean allStopped = described.size() == taskArns.size()
&& described.stream().allMatch(t -> "STOPPED".equals(t.getLastStatus()));
@@ -1845,7 +1951,8 @@ public class AslExecutor {
}
private StateResult executeWaitState(JsonNode stateDef, JsonNode input, boolean jsonata, JsonNode context,
- ObjectNode variables) throws InterruptedException {
+ ObjectNode variables, long executionDeadlineNanos)
+ throws InterruptedException {
int seconds = 0;
if (jsonata) {
if (stateDef.has("Seconds")) {
@@ -1869,7 +1976,7 @@ public class AslExecutor {
}
// Timestamp and TimestampPath: wait until that time or now, whichever is sooner
if (seconds > 0) {
- TimeUnit.SECONDS.sleep(seconds);
+ sleepOrTimeOutExecution(TimeUnit.SECONDS.toNanos(seconds), executionDeadlineNanos);
}
if (jsonata) {
JsonNode output = applyJsonataOutput(stateDef, input, null, context, variables);
@@ -1878,6 +1985,23 @@ public class AslExecutor {
return new StateResult(input, stateDef.path("Next").asText(null));
}
+ /**
+ * Sleeps out a pause the definition asked for, ending the execution instead when the state
+ * machine's {@code TimeoutSeconds} budget runs out first. The two pauses long enough to
+ * outlast that budget are a Wait and a Retry's backoff, and both leave the state they cut
+ * without its Exited event, the same way AWS does.
+ */
+ private void sleepOrTimeOutExecution(long pauseNanos, long executionDeadlineNanos)
+ throws InterruptedException {
+ long remainingNanos = executionDeadlineNanos - System.nanoTime();
+ if (pauseNanos < remainingNanos) {
+ TimeUnit.NANOSECONDS.sleep(pauseNanos);
+ return;
+ }
+ TimeUnit.NANOSECONDS.sleep(Math.max(remainingNanos, 0));
+ throw new ExecutionTimedOutException();
+ }
+
private StateResult executeSucceedState(JsonNode stateDef, JsonNode input, boolean jsonata, JsonNode context,
ObjectNode variables) {
if (jsonata) {
@@ -1890,7 +2014,9 @@ public class AslExecutor {
private StateResult executeFail(JsonNode stateDef, JsonNode input, boolean jsonata, JsonNode context,
ObjectNode variables) {
String error = stateDef.path("Error").asText(null);
- String cause = stateDef.path("Cause").asText(null);
+ // A Fail state that declares no Cause reports an empty one, not a missing key. A task that
+ // ran out of one of its clocks is the only failure that omits the key.
+ String cause = stateDef.path("Cause").asText("");
if (jsonata) {
JsonNode statesVar = buildStatesVar(input, null, context);
if (error != null && JsonataEvaluator.isExpression(error)) {
@@ -1903,9 +2029,11 @@ public class AslExecutor {
throw new FailStateException(error, cause);
}
- private StateResult executeParallelState(String name, JsonNode stateDef, JsonNode input, StateMachine sm,
- boolean jsonata, String topLevelQueryLanguage, JsonNode context,
- ObjectNode variables) throws Exception {
+ private StateResult executeParallelState(String name, JsonNode stateDef, JsonNode input,
+ AtomicLong producedEventCount, StateMachine sm, boolean jsonata,
+ String topLevelQueryLanguage, JsonNode context,
+ ObjectNode variables, long executionDeadlineNanos)
+ throws Exception {
JsonNode branches = stateDef.path("Branches");
List<Future<JsonNode>> futures = new ArrayList<>();
@@ -1924,28 +2052,29 @@ public class AslExecutor {
// scope is thread-bound, so without this a branch's Task integrations would resolve to
// the default account rather than the execution's.
futures.add(executor.submit(() -> callUnderExecutionAccount(sm,
- () -> executeBranch(startAt, branchStates, capturedInput, sm, topLevelQueryLanguage,
- branchContext, branchVariables))));
+ () -> executeBranch(startAt, branchStates, capturedInput, producedEventCount, sm,
+ topLevelQueryLanguage, branchContext, branchVariables))));
}
int timeoutSeconds = stateDef.path("TimeoutSeconds").asInt(0);
- long deadlineNanos = timeoutSeconds > 0
+ long stateDeadlineNanos = timeoutSeconds > 0
? System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds)
: Long.MAX_VALUE;
+ // Two clocks can end this wait, and the join stops at whichever comes first: the state's
+ // own TimeoutSeconds, and the state machine's budget for the whole execution.
+ long joinDeadlineNanos = Math.min(stateDeadlineNanos, executionDeadlineNanos);
ArrayNode results = objectMapper.createArrayNode();
try {
for (Future<JsonNode> future : futures) {
- long remainingNanos = deadlineNanos - System.nanoTime();
+ long remainingNanos = joinDeadlineNanos - System.nanoTime();
if (remainingNanos <= 0) {
- throw new FailStateException("States.Timeout",
- "Parallel state timed out after " + timeoutSeconds + " seconds");
+ throw parallelJoinExpired(stateDeadlineNanos, timeoutSeconds);
}
try {
results.add(future.get(remainingNanos, TimeUnit.NANOSECONDS));
} catch (java.util.concurrent.TimeoutException e) {
- throw new FailStateException("States.Timeout",
- "Parallel state timed out after " + timeoutSeconds + " seconds");
+ throw parallelJoinExpired(stateDeadlineNanos, timeoutSeconds);
}
}
} catch (InterruptedException e) {
@@ -1986,9 +2115,24 @@ public class AslExecutor {
return new StateResult(output, stateDef.path("Next").asText(null));
}
- private StateResult executeMapState(String name, JsonNode stateDef, JsonNode input, StateMachine sm,
- boolean jsonata, String topLevelQueryLanguage, JsonNode context,
- ObjectNode variables) throws Exception {
+ /**
+ * Names the clock that ended a Parallel's join. The state's own {@code TimeoutSeconds} fails
+ * the state, so its Retry and Catch still apply; the state machine's budget ends the whole
+ * execution and no Catch sees it.
+ */
+ private RuntimeException parallelJoinExpired(long stateDeadlineNanos, int timeoutSeconds) {
+ if (System.nanoTime() < stateDeadlineNanos) {
+ return new ExecutionTimedOutException();
+ }
+ return new FailStateException("States.Timeout",
+ "Parallel state timed out after " + timeoutSeconds + " seconds");
+ }
+
+ private StateResult executeMapState(String name, JsonNode stateDef, JsonNode input,
+ AtomicLong producedEventCount, StateMachine sm, boolean jsonata,
+ String topLevelQueryLanguage, JsonNode context,
+ ObjectNode variables, long executionDeadlineNanos)
+ throws Exception {
String processorMode = stateDef.path("ItemProcessor").path("ProcessorConfig")
.path("Mode").asText("INLINE");
boolean distributed = "DISTRIBUTED".equals(processorMode);
@@ -2055,8 +2199,13 @@ public class AslExecutor {
if (hasResultWriter) {
childInputsByIndex[i] = iterInput;
}
- JsonNode branchOutput = executeBranch(startAt, iteratorStates, iterInput, sm,
- topLevelQueryLanguage, iterContext, variables.deepCopy());
+ // A Distributed Map runs each item as a child execution, and a child execution has a
+ // history of its own: the item's events count against its own limit, not the parent's.
+ // An inline Map's iterations are part of this execution and count here.
+ AtomicLong childExecutionEventCount = distributed ? new AtomicLong() : producedEventCount;
+ JsonNode branchOutput = executeBranch(startAt, iteratorStates, iterInput,
+ childExecutionEventCount, sm, topLevelQueryLanguage, iterContext,
+ variables.deepCopy());
if (hasResultWriter) {
childTimingsByIndex[i] = new long[]{startMs, System.currentTimeMillis()};
}
@@ -2064,9 +2213,17 @@ public class AslExecutor {
};
if (itemCount > 0) {
- List<JsonNode> itemOutputs = MapIterationScheduler.execute(
- itemCount, Math.max(1, effectiveConcurrency),
- i -> () -> callUnderExecutionAccount(sm, makeTask.apply(i)));
+ List<JsonNode> itemOutputs;
+ try {
+ itemOutputs = MapIterationScheduler.execute(
+ itemCount, Math.max(1, effectiveConcurrency),
+ i -> () -> callUnderExecutionAccount(sm, makeTask.apply(i)),
+ executionDeadlineNanos);
+ } catch (java.util.concurrent.TimeoutException e) {
+ // The only deadline the scheduler is given is the state machine's budget, so its
+ // expiry ends the execution rather than failing the Map state.
+ throw new ExecutionTimedOutException();
+ }
results.addAll(itemOutputs);
}
@@ -2499,10 +2656,17 @@ public class AslExecutor {
return limited;
}
- private JsonNode executeBranch(String startAt, JsonNode states, JsonNode input, StateMachine sm,
- String topLevelQueryLanguage, JsonNode context, ObjectNode variables) throws Exception {
- List<HistoryEvent> ignored = new ArrayList<>();
- AtomicLong eventId = new AtomicLong(0);
+ /**
+ * Runs the states of one Parallel branch or one Map iteration. floci does not publish their
+ * events, but they are events of the execution all the same, so each one is counted against its
+ * history-event limit: a branch that never reaches a terminal state ends the whole execution at
+ * event 25,000, exactly as one in the top-level flow does. A null history is what tells the
+ * states below they are running inside a branch.
+ */
+ private JsonNode executeBranch(String startAt, JsonNode states, JsonNode input,
+ AtomicLong producedEventCount, StateMachine sm,
+ String topLevelQueryLanguage, JsonNode context,
+ ObjectNode variables) throws Exception {
JsonNode currentInput = input;
String currentState = startAt;
@@ -2517,10 +2681,15 @@ public class AslExecutor {
String type = stateDef.path("Type").asText();
boolean stateJsonata = isJsonata(stateDef, topLevelQueryLanguage);
updateStateContext(context, currentState);
+ countTowardsHistoryEventLimit(producedEventCount);
StateResult result;
try {
- result = executeStateWithRetry(currentState, type, stateDef, currentInput, ignored, eventId, sm,
- stateJsonata, topLevelQueryLanguage, context, variables);
+ // A Parallel or Map branch runs on its own thread and is not cut mid-state by the
+ // execution's TimeoutSeconds: the state loop that resumes once the branch returns
+ // is where the budget is enforced.
+ result = executeStateWithRetry(currentState, type, stateDef, currentInput,
+ null, producedEventCount, sm, stateJsonata, topLevelQueryLanguage, context,
+ variables, Long.MAX_VALUE);
} catch (FailStateException e) {
StateResult caught = handleCatch(stateDef, currentInput, e, stateJsonata, context, variables);
if (caught == null) {
@@ -2528,6 +2697,7 @@ public class AslExecutor {
}
result = caught;
}
+ countTowardsHistoryEventLimit(producedEventCount);
currentInput = result.output();
currentState = result.nextState();
if ("Succeed".equals(type) || stateDef.path("End").asBoolean(false)) {
@@ -3458,18 +3628,76 @@ public class AslExecutor {
// ──────────────────────────── History helpers ────────────────────────────
- private void addEvent(List<HistoryEvent> history, AtomicLong counter, String type, Map<String, Object> details) {
- addEvent(history, counter, type, counter.get(), details);
+ /**
+ * Counts one event towards the limit AWS puts on an execution's history, leaving the last slot
+ * free: it belongs to the event that ends the execution. The count is taken before it is
+ * judged, so however many branches and Map iterations are producing events at once, exactly one
+ * of them takes event 24,999 and every other one finds the limit reached.
+ *
+ * <p>Reaching it raises {@code States.Runtime} at the state that produced the event, and that
+ * ends the whole execution: {@link #catchMatches} refuses {@code States.Runtime} before it
+ * reads {@code ErrorEquals}, so a Retry and a Catch the state declares for it both stand down.
+ */
+ static void countTowardsHistoryEventLimit(AtomicLong producedEventCount) {
+ if (producedEventCount.incrementAndGet() >= MAX_HISTORY_EVENTS) {
+ throw new FailStateException("States.Runtime", HISTORY_EVENT_LIMIT_CAUSE);
+ }
+ }
+
+ /**
+ * Records an event the state machine produced: counted against the history-event limit, then
+ * published.
+ *
+ * <p>{@code history} is null inside a Parallel branch or a Map iteration. Their states are
+ * states of this execution and their events count against its limit, but floci does not publish
+ * them, so there is nothing to build for them beyond the count.
+ */
+ private void publishEvent(List<HistoryEvent> history, AtomicLong producedEventCount, String type,
+ Map<String, Object> details) {
+ countTowardsHistoryEventLimit(producedEventCount);
+ if (history == null) {
+ return;
+ }
+ appendEvent(history, type, history.size(), details);
+ }
+
+ /**
+ * Records a state's Entered event with the previousEventId the top-level flow works out: AWS
+ * leaves the Entered event of the state an execution starts in unchained, at previousEventId 0,
+ * rather than pointing it at the ExecutionStarted event before it. Only the top-level flow
+ * publishes these, so its history is never null.
+ */
+ private void publishStateEnteredEvent(List<HistoryEvent> history, AtomicLong producedEventCount,
+ String type, long previousEventId,
+ Map<String, Object> details) {
+ countTowardsHistoryEventLimit(producedEventCount);
+ appendEvent(history, type, previousEventId, details);
}
- private void addEvent(List<HistoryEvent> history, AtomicLong counter, String type, long previousEventId,
- Map<String, Object> details) {
- var event = new HistoryEvent();
- event.setId(counter.incrementAndGet());
- event.setPreviousEventId(previousEventId);
- event.setType(type);
- event.setDetails(details);
- history.add(event);
+ /**
+ * Records the event that ends the execution. It does not count towards the history-event limit:
+ * an execution always gets to say how it ended, in the slot {@link #publishEvent} leaves free.
+ */
+ private void publishTerminalEvent(List<HistoryEvent> history, String type, Map<String, Object> details) {
+ appendEvent(history, type, history.size(), details);
+ }
+
+ /**
+ * Appends an event and numbers it from the end of the history: the published history is the one
+ * authority for an event's id, so an event's id is its position in the list. Held under the
+ * history's own monitor, because StopExecution appends the terminal event of an aborted
+ * execution from another thread and seals the history against anything after it.
+ */
+ private void appendEvent(List<HistoryEvent> history, String type, long previousEventId,
+ Map<String, Object> details) {
+ synchronized (history) {
+ var event = new HistoryEvent();
+ event.setId(history.size() + 1L);
+ event.setPreviousEventId(previousEventId);
+ event.setType(type);
+ event.setDetails(details);
+ history.add(event);
+ }
}
private record TaskEventProfile(String prefix, String resourceType, String resource) {}
@@ -3492,8 +3720,9 @@ public class AslExecutor {
return new TaskEventProfile("Task", resource, resource);
}
- private void addTaskScheduledEvent(List<HistoryEvent> history, AtomicLong eventId, TaskEventProfile profile,
- JsonNode stateDef, JsonNode effectiveInput, StateMachine sm) {
+ private void addTaskScheduledEvent(List<HistoryEvent> history, AtomicLong producedEventCount,
+ TaskEventProfile profile, JsonNode stateDef, JsonNode effectiveInput,
+ StateMachine sm) {
var details = new LinkedHashMap<String, Object>();
if (profile.resourceType() != null) {
details.put("resourceType", profile.resourceType());
@@ -3512,33 +3741,34 @@ public class AslExecutor {
if (stateDef.path("HeartbeatSeconds").isNumber()) {
details.put("heartbeatInSeconds", stateDef.path("HeartbeatSeconds").asLong());
}
- addEvent(history, eventId, profile.prefix() + "Scheduled", details);
+ publishEvent(history, producedEventCount, profile.prefix() + "Scheduled", details);
}
- private void addTaskStartedEvent(List<HistoryEvent> history, AtomicLong eventId, TaskEventProfile profile) {
+ private void addTaskStartedEvent(List<HistoryEvent> history, AtomicLong producedEventCount,
+ TaskEventProfile profile) {
if ("Task".equals(profile.prefix())) {
- addEvent(history, eventId, profile.prefix() + "Started",
+ publishEvent(history, producedEventCount, profile.prefix() + "Started",
Map.of("resourceType", profile.resourceType(), "resource", profile.resource()));
} else {
- addEvent(history, eventId, profile.prefix() + "Started", null);
+ publishEvent(history, producedEventCount, profile.prefix() + "Started", null);
}
}
- private void addTaskSucceededEvent(List<HistoryEvent> history, AtomicLong eventId, TaskEventProfile profile,
- JsonNode taskResult) {
+ private void addTaskSucceededEvent(List<HistoryEvent> history, AtomicLong producedEventCount,
+ TaskEventProfile profile, JsonNode taskResult) {
var output = taskResult.toString();
if ("Task".equals(profile.prefix())) {
- addEvent(history, eventId, profile.prefix() + "Succeeded",
+ publishEvent(history, producedEventCount, profile.prefix() + "Succeeded",
Map.of("resourceType", profile.resourceType(), "resource", profile.resource(),
"output", output, "outputDetails", Map.of("truncated", false)));
} else {
- addEvent(history, eventId, profile.prefix() + "Succeeded",
+ publishEvent(history, producedEventCount, profile.prefix() + "Succeeded",
Map.of("output", output, "outputDetails", Map.of("truncated", false)));
}
}
- private void addTaskFailedEvent(List<HistoryEvent> history, AtomicLong eventId, TaskEventProfile profile,
- String error, String cause) {
+ private void addTaskFailedEvent(List<HistoryEvent> history, AtomicLong producedEventCount,
+ TaskEventProfile profile, String error, String cause) {
var details = new LinkedHashMap<String, Object>();
if ("Task".equals(profile.prefix())) {
details.put("resourceType", profile.resourceType());
@@ -3550,27 +3780,103 @@ public class AslExecutor {
if (cause != null) {
details.put("cause", cause);
}
- addEvent(history, eventId, profile.prefix() + "Failed", details);
+ publishEvent(history, producedEventCount, profile.prefix() + "Failed", details);
}
- private void failExecution(Execution exec, List<HistoryEvent> history, AtomicLong eventId, FailStateException e) {
- failExecution(exec, history, eventId, e.error != null ? e.error : "States.Runtime",
- e.cause != null ? e.cause : "");
+ /**
+ * The event a Task leaves when one of its clocks runs out. It names {@code States.Timeout} for
+ * both {@code TimeoutSeconds} and {@code HeartbeatSeconds}, and carries no cause.
+ */
+ private void addTaskTimedOutEvent(List<HistoryEvent> history, AtomicLong producedEventCount,
+ TaskEventProfile profile) {
+ var details = new LinkedHashMap<String, Object>();
+ if ("Task".equals(profile.prefix())) {
+ details.put("resourceType", profile.resourceType());
+ details.put("resource", profile.resource());
+ }
+ details.put("error", "States.Timeout");
+ publishEvent(history, producedEventCount, profile.prefix() + "TimedOut", details);
+ }
+
+ private void failExecution(Execution exec, List<HistoryEvent> history, FailStateException e) {
+ failExecution(exec, history, e.error != null ? e.error : "States.Runtime", e.cause);
}
/**
* The single terminal-failure write: every way an execution can fail leaves the same
* {@code error}, {@code cause} and {@code ExecutionFailed} event behind, so a client cannot
* tell a Fail state from a state that threw from a runtime Error by what it reads back.
+ *
+ * <p>A null {@code cause} is the failure saying it has none, and both DescribeExecution and the
+ * ExecutionFailed event leave the key out rather than reporting it empty. Only a task that ran
+ * out of its TimeoutSeconds or HeartbeatSeconds budget arrives here without one.
*/
- private void failExecution(Execution exec, List<HistoryEvent> history, AtomicLong eventId,
- String error, String cause) {
- exec.setError(error);
- exec.setCause(cause);
- exec.setStopDate(System.currentTimeMillis() / 1000.0);
- exec.setStatus("FAILED");
- addEvent(history, eventId, "ExecutionFailed",
- Map.of("error", error, "cause", cause));
+ private void failExecution(Execution exec, List<HistoryEvent> history, String error, String cause) {
+ synchronized (exec) {
+ if (abortedByCaller(exec)) {
+ return;
+ }
+ exec.setError(error);
+ exec.setCause(cause);
+ exec.setStopDate(System.currentTimeMillis() / 1000.0);
+ exec.setStatus("FAILED");
+ }
+ var details = new LinkedHashMap<String, Object>();
+ details.put("error", error);
+ if (cause != null) {
+ details.put("cause", cause);
+ }
+ publishTerminalEvent(history, "ExecutionFailed", details);
+ }
+
+ /**
+ * The third terminal write. A timed out execution carries neither error nor cause:
+ * DescribeExecution leaves both keys out, and States.Timeout is named only inside the
+ * ExecutionTimedOut event, which points at the start of the execution rather than at the state
+ * it cut. The event is appended rather than published, because it is what ends the execution
+ * and the history-event limit leaves the last slot free for exactly that.
+ */
+ private void timeOutExecution(Execution exec, List<HistoryEvent> history) {
+ synchronized (exec) {
+ if (abortedByCaller(exec)) {
+ return;
+ }
+ exec.setStopDate(System.currentTimeMillis() / 1000.0);
+ exec.setStatus("TIMED_OUT");
+ }
+ appendEvent(history, "ExecutionTimedOut", 0L, Map.of("error", "States.Timeout"));
+ }
+
+ /**
+ * The single terminal-success write, the mirror of {@link #failExecution}.
+ *
+ * <p>Status is the publication point, so it is set last. describeExecution hands out this same
+ * live Execution, so a client polling for SUCCEEDED between setStatus and setOutput would read
+ * a terminal execution with a null output, which real Step Functions never returns.
+ */
+ private void succeedExecution(Execution exec, List<HistoryEvent> history, JsonNode output) {
+ synchronized (exec) {
+ if (abortedByCaller(exec)) {
+ return;
+ }
+ exec.setOutput(output.toString());
+ exec.setStopDate(System.currentTimeMillis() / 1000.0);
+ exec.setStatus("SUCCEEDED");
+ }
+ publishTerminalEvent(history, "ExecutionSucceeded",
+ Map.of("output", output.toString(), "outputDetails", Map.of("truncated", false)));
+ }
+
+ /**
+ * True once StopExecution published ABORTED on this execution. The state loop reads it between
+ * states and every terminal write here reads it before publishing, so the worker's own status
+ * loses the race against a stop that arrived while it was still stepping: what a caller has
+ * already read back from DescribeExecution is what stands.
+ */
+ private static boolean abortedByCaller(Execution exec) {
+ synchronized (exec) {
+ return "ABORTED".equals(exec.getStatus());
+ }
}
private StateResult handleCatch(JsonNode stateDef, JsonNode input, FailStateException failure,
@@ -3583,7 +3889,7 @@ public class AslExecutor {
String cause = failure.cause != null ? failure.cause : "";
for (int i = 0; i < catchers.size(); i++) {
JsonNode catcher = catchers.get(i);
- if (!catchMatches(catcher, error)) {
+ if (!catchMatches(catcher, failure)) {
continue;
}
String next = catcher.path("Next").asText(null);
@@ -3607,11 +3913,12 @@ public class AslExecutor {
return null;
}
- private boolean catchMatches(JsonNode catcher, String error) {
+ private boolean catchMatches(JsonNode catcher, FailStateException failure) {
var errors = catcher.path("ErrorEquals");
if (!errors.isArray()) {
return false;
}
+ var error = failure.error != null ? failure.error : "States.Runtime";
// States.Runtime is never retried or caught, even when named explicitly in
// ErrorEquals. Verified against real AWS: the execution fails immediately.
if ("States.Runtime".equals(error)) {
@@ -3619,7 +3926,7 @@ public class AslExecutor {
}
for (JsonNode candidate : errors) {
var expected = candidate.asText();
- if (expected.equals(error)) {
+ if (failure.isNamedBy(expected)) {
return true;
}
if ("States.TaskFailed".equals(expected)
@@ -3663,6 +3970,17 @@ public class AslExecutor {
record StateResult(JsonNode output, String nextState) {}
+ /**
+ * Thrown when the state machine's top-level {@code TimeoutSeconds} budget runs out. It is not a
+ * {@link FailStateException} on purpose: a Catch clause never sees it, no Retry re-runs the
+ * state it cut, and the execution ends TIMED_OUT rather than FAILED.
+ */
+ static class ExecutionTimedOutException extends RuntimeException {
+ ExecutionTimedOutException() {
+ super("States.Timeout");
+ }
+ }
+
static class FailStateException extends RuntimeException {
final String error;
final String cause;
@@ -3672,5 +3990,34 @@ public class AslExecutor {
this.error = error;
this.cause = cause;
}
+
+ /**
+ * Whether an {@code ErrorEquals} entry spelling {@code errorName} names this failure. A
+ * failure answers to the error it reports, and a task timeout answers to the name of the
+ * clock that ran out as well.
+ */
+ boolean isNamedBy(String errorName) {
+ return errorName.equals(error);
+ }
+ }
+
+ /**
+ * Thrown when a Task ran out of one of the two clocks bounding its wait for a task token. Both
+ * report {@code States.Timeout} with no cause and emit a {@code TimedOut} history event; a
+ * {@code HeartbeatSeconds} expiry is also caught by an {@code ErrorEquals} naming
+ * {@code States.HeartbeatTimeout}.
+ */
+ static class TaskTimedOutException extends FailStateException {
+ private final String expiredClockError;
+
+ TaskTimedOutException(String expiredClockError) {
+ super("States.Timeout", null);
+ this.expiredClockError = expiredClockError;
+ }
+
+ @Override
+ boolean isNamedBy(String errorName) {
+ return super.isNamedBy(errorName) || errorName.equals(expiredClockError);
+ }
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationScheduler.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationScheduler.java
index d8cb258798..77b6a8450f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationScheduler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationScheduler.java
@@ -12,6 +12,8 @@ import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.function.IntFunction;
/** Runs Map iterations with a bounded in-flight window while preserving input order. */
@@ -20,8 +22,15 @@ final class MapIterationScheduler {
private MapIterationScheduler() {
}
+ /**
+ * @param deadlineNanos a {@link System#nanoTime()} reading the iterations must all finish
+ * before, or {@link Long#MAX_VALUE} for no deadline. Reaching it throws
+ * {@link TimeoutException}, leaving the caller to say what the expiry
+ * means for the state it belongs to.
+ */
static <T> List<T> execute(int itemCount, int maxConcurrency,
- IntFunction<Callable<T>> taskFactory) throws Exception {
+ IntFunction<Callable<T>> taskFactory, long deadlineNanos)
+ throws Exception {
if (itemCount < 0) {
throw new IllegalArgumentException("itemCount must not be negative");
}
@@ -31,12 +40,8 @@ final class MapIterationScheduler {
if (itemCount == 0) {
return List.of();
}
- if (itemCount == 1 || maxConcurrency == 1) {
- List<T> results = new ArrayList<>(itemCount);
- for (int i = 0; i < itemCount; i++) {
- results.add(taskFactory.apply(i).call());
- }
- return results;
+ if (System.nanoTime() >= deadlineNanos) {
+ throw new TimeoutException();
}
int inFlightLimit = Math.min(itemCount, maxConcurrency);
@@ -53,7 +58,12 @@ final class MapIterationScheduler {
inFlight.add(submit(completions, taskFactory, nextIndex++));
}
while (completed < itemCount) {
- Future<IndexedResult<T>> future = completions.take();
+ Future<IndexedResult<T>> future =
+ completions.poll(deadlineNanos - System.nanoTime(), TimeUnit.NANOSECONDS);
+ if (future == null) {
+ cancel(inFlight);
+ throw new TimeoutException();
+ }
inFlight.remove(future);
IndexedResult<T> result = future.get();
results.set(result.index(), result.value());
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java
index ed56bf86cb..fe51d8bd4c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java
@@ -410,7 +410,9 @@ public class StepFunctionsJsonHandler {
private Response handleSendTaskFailure(JsonNode request) {
service.sendTaskFailure(
request.path("taskToken").asText(),
- request.path("cause").asText(null),
+ // A SendTaskFailure that names no cause fails the task with an empty one, not with
+ // a missing key.
+ request.path("cause").asText(""),
request.path("error").asText(null)
);
return Response.ok(objectMapper.createObjectNode()).build();
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
index 479b7666ff..b878a74f04 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
@@ -46,9 +46,12 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
private final StorageBackend<String, Execution> executionStore;
private final StorageBackend<String, Activity> activityStore;
private final StorageBackend<String, MapRun> mapRunStore;
- private final Map<String, List<HistoryEvent>> historyCache = new ConcurrentHashMap<>();
+ private final Map<String, ExecutionHistory> historyCache = new ConcurrentHashMap<>();
private final Map<String, BlockingQueue<ActivityTask>> activityQueues = new ConcurrentHashMap<>();
private final Map<String, CompletableFuture<JsonNode>> pendingTaskTokens = new ConcurrentHashMap<>();
+ // When each pending token last showed progress, so a Task's HeartbeatSeconds can bound the gap
+ // between heartbeats instead of the whole wait.
+ private final Map<String, Long> taskHeartbeatNanos = new ConcurrentHashMap<>();
private final RegionResolver regionResolver;
private final AslExecutor aslExecutor;
private final ObjectMapper objectMapper;
@@ -104,6 +107,7 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
activityQueues.clear();
pendingTaskTokens.values().forEach(f -> f.completeExceptionally(new RuntimeException("StepFunctionsService cleared")));
pendingTaskTokens.clear();
+ taskHeartbeatNanos.clear();
}
@Override
@@ -524,7 +528,7 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
executionStore.put(arn, exec);
- var history = new ArrayList<HistoryEvent>();
+ var history = new ExecutionHistory();
var startEvent = new HistoryEvent();
startEvent.setId(1L);
startEvent.setPreviousEventId(0L);
@@ -537,9 +541,10 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
LOG.infov("Started execution: {0}", arn);
+ // The executor appends to this same history, so the cache entry put above is already the
+ // finished history by the time this runs.
aslExecutor.executeAsync(sm, exec, history, mockedTestCase, (updatedExec, updatedHistory) -> {
executionStore.put(updatedExec.getExecutionArn(), updatedExec);
- historyCache.put(updatedExec.getExecutionArn(), updatedHistory);
LOG.infov("Execution {0} completed with status {1}", updatedExec.getExecutionArn(), updatedExec.getStatus());
});
@@ -660,30 +665,80 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
.map(e -> e.getStateMachineArn().equals(stateMachineArn)).orElse(false));
}
+ /**
+ * Aborts a running execution. The caller's {@code error} and {@code cause} land on the
+ * Execution itself, not only on the history event, so DescribeExecution reports them.
+ *
+ * <p>The worker thread may still be inside a state when this runs. It shares this Execution
+ * instance and reads the status it publishes here, so the writes are made under the same
+ * monitor the worker's terminal write takes: ABORTED is the status that stands.
+ *
+ * <p>ExecutionAborted seals the history for the same reason: the worker still has the state it
+ * is inside left to record, and those events belong to an execution the caller has already
+ * been told is finished.
+ */
public void stopExecution(String arn, String cause, String error) {
Execution exec = describeExecution(arn);
- if (!"RUNNING".equals(exec.getStatus())) {
- return;
+ synchronized (exec) {
+ if (!"RUNNING".equals(exec.getStatus())) {
+ return;
+ }
+ exec.setError(error);
+ exec.setCause(cause);
+ exec.setStopDate(System.currentTimeMillis() / 1000.0);
+ exec.setStatus("ABORTED");
}
- exec.setStopDate(System.currentTimeMillis() / 1000.0);
- exec.setStatus("ABORTED");
executionStore.put(arn, exec);
- List<HistoryEvent> history = historyCache.getOrDefault(arn, new ArrayList<>());
- HistoryEvent event = new HistoryEvent();
- event.setId(history.size() + 1L);
- event.setPreviousEventId((long) history.size());
- event.setType("ExecutionAborted");
Map<String, Object> details = new HashMap<>();
if (error != null) details.put("error", error);
if (cause != null) details.put("cause", cause);
- event.setDetails(details);
- history.add(event);
+ // An execution can outlive its history: the executions are stored, the histories are held in
+ // memory only, so a restart in persistent mode brings a RUNNING execution back with nothing
+ // behind it. The abort still gets recorded, against a history that starts here.
+ historyCache.computeIfAbsent(arn, key -> new ExecutionHistory())
+ .sealWith("ExecutionAborted", details);
}
public List<HistoryEvent> getExecutionHistory(String arn) {
describeExecution(arn);
- return historyCache.getOrDefault(arn, Collections.emptyList());
+ ExecutionHistory history = historyCache.get(arn);
+ return history != null ? history : Collections.emptyList();
+ }
+
+ /**
+ * The history of one execution, appended to by the worker thread running the state machine and
+ * by the thread serving StopExecution. Sealing it with the terminal event is what makes that
+ * event the last one: the worker is still inside a state when the abort lands, and the events
+ * it has left to record belong to an execution the caller has already been told is finished.
+ *
+ * <p>The seal is enforced in {@link #add}, the one way the worker appends. {@code addAll} does
+ * not route through it, so a bulk append would write past the seal; nothing does one today.
+ */
+ static final class ExecutionHistory extends ArrayList<HistoryEvent> {
+
+ private static final long serialVersionUID = 1L;
+
+ private boolean sealed;
+
+ @Override
+ public synchronized boolean add(HistoryEvent event) {
+ if (sealed) {
+ return false;
+ }
+ return super.add(event);
+ }
+
+ /** Appends the terminal event, numbered from the end of the history, and takes no more. */
+ synchronized void sealWith(String terminalEventType, Map<String, Object> details) {
+ HistoryEvent terminalEvent = new HistoryEvent();
+ terminalEvent.setId(size() + 1L);
+ terminalEvent.setPreviousEventId((long) size());
+ terminalEvent.setType(terminalEventType);
+ terminalEvent.setDetails(details);
+ add(terminalEvent);
+ sealed = true;
+ }
}
// ──────────────────────────── Activities ────────────────────────────
@@ -744,9 +799,32 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
public CompletableFuture<JsonNode> registerPendingToken(String token) {
CompletableFuture<JsonNode> future = new CompletableFuture<>();
pendingTaskTokens.put(token, future);
+ taskHeartbeatNanos.put(token, System.nanoTime());
return future;
}
+ /**
+ * When the token last showed progress: the moment its task was scheduled, then every
+ * SendTaskHeartbeat that named it. A token that is no longer pending reads as having just
+ * reported, so a task whose result already arrived is never failed on a heartbeat gap.
+ */
+ public long lastTaskHeartbeatNanos(String taskToken) {
+ Long reportedAt = taskToken != null ? taskHeartbeatNanos.get(taskToken) : null;
+ return reportedAt != null ? reportedAt : System.nanoTime();
+ }
+
+ /**
+ * Forgets a token the waiting task has stopped listening for, so a later SendTaskSuccess,
+ * SendTaskFailure or SendTaskHeartbeat naming it reports no pending task.
+ */
+ public void discardPendingToken(String taskToken) {
+ if (taskToken == null) {
+ return;
+ }
+ pendingTaskTokens.remove(taskToken);
+ taskHeartbeatNanos.remove(taskToken);
+ }
+
// ──────────────────────────── Tasks ────────────────────────────
/**
@@ -782,8 +860,16 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
return true;
}
+ /**
+ * Resets the gap a Task's {@code HeartbeatSeconds} allows. A heartbeat for a token nobody is
+ * waiting for is logged and changes nothing, as in {@link #sendTaskSuccess(String, String)}.
+ */
public void sendTaskHeartbeat(String taskToken) {
- LOG.debugv("Task heartbeat for token {0}", taskToken);
+ if (taskToken == null || !pendingTaskTokens.containsKey(taskToken)) {
+ LOG.warnv("SendTaskHeartbeat: no pending task for token {0}", taskToken);
+ return;
+ }
+ taskHeartbeatNanos.put(taskToken, System.nanoTime());
}
// ──────────────────────────── Map runs ────────────────────────────
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHistoryEventLimitTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHistoryEventLimitTest.java
new file mode 100644
index 0000000000..49c3a225d0
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHistoryEventLimitTest.java
@@ -0,0 +1,377 @@
+package io.github.hectorvent.floci.services.stepfunctions;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbJsonHandler;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;
+import io.github.hectorvent.floci.services.lambda.LambdaExecutorService;
+import io.github.hectorvent.floci.services.lambda.LambdaFunctionStore;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;
+import io.github.hectorvent.floci.services.stepfunctions.model.Execution;
+import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;
+import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;
+import io.quarkus.test.junit.QuarkusTest;
+import io.vertx.mutiny.core.Vertx;
+import jakarta.enterprise.inject.Instance;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Guards the bound AWS puts on a running execution: a state machine that never reaches a terminal
+ * state is ended at 25,000 history events with {@code States.Runtime}. The cut is an event count,
+ * not a wall clock, and the counter is neither reset nor offset: the {@code ExecutionFailed} event
+ * is event 25,000 itself, so the last event the machine produced is 24,999.
+ *
+ * <p>A {@code Choice} that loops back through a {@code Pass} reaches it in a few hundred
+ * milliseconds, which is why nothing here waits on the clock.
+ */
+@QuarkusTest
+class AslExecutorHistoryEventLimitTest {
+
+ private static final String REGION = "us-east-2";
+ private static final String ACCOUNT = "000000000000";
+
+ /** A Choice that spins forever, four history events per lap and no terminal state. */
+ private static final String RUNAWAY_LOOP = """
+ {
+ "StartAt": "Loop",
+ "States": {
+ "Loop": {
+ "Type": "Choice",
+ "Choices": [{"Variable": "$.spin", "BooleanEquals": true, "Next": "Tick"}],
+ "Default": "Done"
+ },
+ "Tick": {"Type": "Pass", "Next": "Loop"},
+ "Done": {"Type": "Succeed"}
+ }
+ }
+ """;
+
+ /**
+ * A Parallel branch that laps 10,000 times, four history events per lap. The lap count is high
+ * enough that the branch alone produces far more than 25,000 events, and finite so the run ends
+ * either way: bounded, at event 25,000; unbounded, by finishing every lap.
+ */
+ private static final String RUNAWAY_PARALLEL_BRANCH = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Seed",
+ "States": {
+ "Seed": {"Type": "Pass", "Assign": {"laps": 0}, "Next": "Fan"},
+ "Fan": {
+ "Type": "Parallel",
+ "Branches": [{
+ "StartAt": "Spin",
+ "States": {
+ "Spin": {
+ "Type": "Choice",
+ "Choices": [{"Condition": "{% $laps < 10000 %}", "Next": "Tick"}],
+ "Default": "Stop"
+ },
+ "Tick": {"Type": "Pass", "Assign": {"laps": "{% $laps + 1 %}"}, "Next": "Spin"},
+ "Stop": {"Type": "Succeed"}
+ }
+ }],
+ "End": true
+ }
+ }
+ }
+ """;
+
+ /**
+ * The same runaway branch under a Parallel that declares a Retry and a Catch for States.ALL,
+ * the two clauses AWS allows on a Parallel. Reaching the limit is States.Runtime, which
+ * catchMatches refuses before it reads ErrorEquals, so neither clause runs.
+ */
+ private static final String RUNAWAY_PARALLEL_BRANCH_UNDER_RETRY_AND_CATCH = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Seed",
+ "States": {
+ "Seed": {"Type": "Pass", "Assign": {"laps": 0}, "Next": "Fan"},
+ "Fan": {
+ "Type": "Parallel",
+ "Retry": [{"ErrorEquals": ["States.ALL"], "IntervalSeconds": 30, "MaxAttempts": 3}],
+ "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "Recovered"}],
+ "Branches": [{
+ "StartAt": "Spin",
+ "States": {
+ "Spin": {
+ "Type": "Choice",
+ "Choices": [{"Condition": "{% $laps < 10000 %}", "Next": "Tick"}],
+ "Default": "Stop"
+ },
+ "Tick": {"Type": "Pass", "Assign": {"laps": "{% $laps + 1 %}"}, "Next": "Spin"},
+ "Stop": {"Type": "Succeed"}
+ }
+ }],
+ "Next": "Recovered"
+ },
+ "Recovered": {"Type": "Succeed"}
+ }
+ }
+ """;
+
+ /**
+ * More items than the 25,000-event limit has room for: each one enters and exits a single Pass,
+ * so 13,000 items are 26,000 events if the parent is charged for them.
+ */
+ private static final int DISTRIBUTED_MAP_ITEMS = 13_000;
+
+ /** One item per iteration, run one at a time so the run is the same on every machine. */
+ private static final String DISTRIBUTED_MAP_OVER_MANY_ITEMS = """
+ {
+ "StartAt": "Fan",
+ "States": {
+ "Fan": {
+ "Type": "Map",
+ "ItemsPath": "$.items",
+ "MaxConcurrency": 1,
+ "ItemProcessor": {
+ "ProcessorConfig": {"Mode": "DISTRIBUTED", "ExecutionType": "STANDARD"},
+ "StartAt": "Handle",
+ "States": {"Handle": {"Type": "Pass", "End": true}}
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ /**
+ * The same items run inline, 40 at a time. An inline Map's iterations are part of this
+ * execution, so together they run it out of history events while 40 of them are in flight.
+ */
+ private static final String INLINE_MAP_OVER_MANY_ITEMS = """
+ {
+ "StartAt": "Fan",
+ "States": {
+ "Fan": {
+ "Type": "Map",
+ "ItemsPath": "$.items",
+ "MaxConcurrency": 40,
+ "ItemProcessor": {
+ "StartAt": "Handle",
+ "States": {"Handle": {"Type": "Pass", "End": true}}
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private final List<HistoryEvent> history = new ArrayList<>();
+ private AslExecutor executor;
+
+ @Inject
+ Vertx vertx;
+
+ @BeforeEach
+ void setUp() {
+ executor = new AslExecutor(
+ mock(LambdaExecutorService.class),
+ mock(LambdaFunctionStore.class),
+ mock(DynamoDbService.class),
+ mock(DynamoDbJsonHandler.class),
+ mock(SqsJsonHandler.class),
+ mock(io.github.hectorvent.floci.services.cloudformation.CloudFormationQueryHandler.class),
+ mock(io.github.hectorvent.floci.services.ec2.Ec2Service.class),
+ mock(S3Service.class),
+ mock(io.github.hectorvent.floci.services.ecs.EcsService.class),
+ mock(io.github.hectorvent.floci.services.ecs.EcsJsonHandler.class),
+ mock(io.github.hectorvent.floci.services.eventbridge.EventBridgeHandler.class),
+ mock(io.github.hectorvent.floci.services.scheduler.SchedulerService.class),
+ mock(io.github.hectorvent.floci.services.scheduler.SchedulerController.class),
+ objectMapper,
+ new JsonataEvaluator(objectMapper),
+ mock(Instance.class), mock(EmulatorConfig.class), vertx, null);
+ }
+
+ @Test
+ void runawayLoopFailsWithTheHistoryEventLimitError() {
+ Execution execution = run(RUNAWAY_LOOP);
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.Runtime", execution.getError());
+ assertEquals("The execution reached the maximum number of history events (25000).",
+ execution.getCause());
+ assertNotNull(execution.getStopDate());
+ }
+
+ @Test
+ void executionFailedIsTheTwentyFiveThousandthEvent() {
+ run(RUNAWAY_LOOP);
+
+ assertEquals(25000, history.size());
+ HistoryEvent last = history.get(history.size() - 1);
+ assertEquals("ExecutionFailed", last.getType());
+ assertEquals(25000L, last.getId());
+ // Not reset and not offset: 24,999 is the last event the machine itself produced.
+ HistoryEvent lastFromTheMachine = history.get(history.size() - 2);
+ assertEquals(24999L, lastFromTheMachine.getId());
+ assertNotEquals("ExecutionFailed", lastFromTheMachine.getType());
+ }
+
+ @Test
+ void eventsProducedInsideAParallelBranchCountTowardsTheLimit() {
+ Execution execution = run(RUNAWAY_PARALLEL_BRANCH);
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.Runtime", execution.getError());
+ assertEquals("The execution reached the maximum number of history events (25000).",
+ execution.getCause());
+ }
+
+ /**
+ * The branch produced the events that ran the execution out of them, and floci publishes none
+ * of them. What the caller reads back is the four events the top-level flow did produce, ending
+ * in the failure: an unbroken chain, not four events numbered as if 25,000 had happened.
+ */
+ @Test
+ void aParallelBranchLeavesThePublishedHistoryUnbroken() {
+ run(RUNAWAY_PARALLEL_BRANCH);
+
+ assertEquals(List.of("PassStateEntered", "PassStateExited", "ParallelStateEntered",
+ "ExecutionFailed"),
+ history.stream().map(HistoryEvent::getType).toList());
+ for (int index = 0; index < history.size(); index++) {
+ assertEquals(index + 1L, history.get(index).getId(), "unexpected id at index " + index);
+ assertEquals((long) index, history.get(index).getPreviousEventId(),
+ "unexpected previousEventId at index " + index);
+ }
+ }
+
+ /**
+ * A Distributed Map runs each item as a child execution, and a child execution has a history of
+ * its own. The parent is charged for the Map run, not for what the items do inside it, so a
+ * Distributed Map over more items than the limit has room for still succeeds.
+ */
+ @Test
+ void aDistributedMapItemDoesNotSpendTheParentsHistoryEvents() {
+ Execution execution = run(DISTRIBUTED_MAP_OVER_MANY_ITEMS, manyItemsInput());
+
+ assertEquals("SUCCEEDED", execution.getStatus(),
+ "error=" + execution.getError() + " cause=" + execution.getCause());
+ }
+
+ /**
+ * The same items run inline are the execution's own, whatever the concurrency: 40 iterations
+ * counting against one limit run it out of history events, where a Distributed Map's items do
+ * not.
+ */
+ @Test
+ void concurrentInlineMapIterationsSpendTheExecutionsHistoryEvents() {
+ Execution execution = run(INLINE_MAP_OVER_MANY_ITEMS, manyItemsInput());
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.Runtime", execution.getError());
+ assertEquals("The execution reached the maximum number of history events (25000).",
+ execution.getCause());
+ }
+
+ /**
+ * The event that reaches the limit is what ends the execution, so exactly one of the iterations
+ * racing for the last countable event may take it. Reading the count and then taking it lets
+ * every racer through the check and spends more events than the execution has.
+ */
+ @Test
+ void onlyOneOfTheThreadsRacingForTheLastCountableEventTakesIt() throws Exception {
+ int racers = 40;
+ // 24,998 events produced: one is left before the limit, and event 25,000 is the failure.
+ AtomicLong producedEventCount = new AtomicLong(24_998);
+ CountDownLatch ready = new CountDownLatch(racers);
+ CountDownLatch go = new CountDownLatch(1);
+ AtomicInteger counted = new AtomicInteger();
+
+ List<Thread> racerThreads = new ArrayList<>();
+ for (int racer = 0; racer < racers; racer++) {
+ Thread thread = new Thread(() -> {
+ ready.countDown();
+ try {
+ go.await();
+ AslExecutor.countTowardsHistoryEventLimit(producedEventCount);
+ counted.incrementAndGet();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } catch (AslExecutor.FailStateException expected) {
+ // The limit was already reached by another racer.
+ }
+ });
+ racerThreads.add(thread);
+ thread.start();
+ }
+ ready.await();
+ go.countDown();
+ for (Thread thread : racerThreads) {
+ thread.join();
+ }
+
+ assertEquals(1, counted.get(), "more than one racer took the last countable event");
+ }
+
+ /** {@code {"items": [0, 1, ... ]}}, one item per iteration of the Distributed Map. */
+ private static String manyItemsInput() {
+ StringBuilder input = new StringBuilder("{\"items\": [0");
+ for (int item = 1; item < DISTRIBUTED_MAP_ITEMS; item++) {
+ input.append(',').append(item);
+ }
+ return input.append("]}").toString();
+ }
+
+ @Test
+ void neitherRetryNorCatchInterceptsTheHistoryEventLimit() {
+ long startedAt = System.currentTimeMillis();
+
+ Execution execution = run(RUNAWAY_PARALLEL_BRANCH_UNDER_RETRY_AND_CATCH);
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.Runtime", execution.getError());
+ assertEquals("The execution reached the maximum number of history events (25000).",
+ execution.getCause());
+ assertEquals("ExecutionFailed", history.get(history.size() - 1).getType());
+ assertNotEquals("SucceedStateEntered", history.get(history.size() - 2).getType());
+ // Three attempts at IntervalSeconds 30 would put this past 90 seconds if the Retry ran.
+ assertTrue(System.currentTimeMillis() - startedAt < 30_000,
+ "the Retry's backoff ran");
+ }
+
+ private Execution run(String definition) {
+ return run(definition, "{\"spin\": true}");
+ }
+
+ private Execution run(String definition, String input) {
+ StateMachine stateMachine = new StateMachine();
+ stateMachine.setName("history-event-limit-test");
+ stateMachine.setStateMachineArn(
+ "arn:aws:states:%s:%s:stateMachine:history-event-limit-test".formatted(REGION, ACCOUNT));
+ stateMachine.setRoleArn("arn:aws:iam::%s:role/test-role".formatted(ACCOUNT));
+ stateMachine.setDefinition(definition);
+
+ Execution execution = new Execution();
+ execution.setName("history-event-limit-execution");
+ execution.setExecutionArn(
+ "arn:aws:states:%s:%s:execution:history-event-limit-test:history-event-limit-execution"
+ .formatted(REGION, ACCOUNT));
+ execution.setStateMachineArn(stateMachine.getStateMachineArn());
+ execution.setInput(input);
+
+ executor.executeSync(stateMachine, execution, history, (updated, events) -> {
+ });
+ return execution;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTaskHistoryEventsTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTaskHistoryEventsTest.java
index 4ca7bc7c8a..57c54717ad 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTaskHistoryEventsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTaskHistoryEventsTest.java
@@ -257,6 +257,48 @@ class AslExecutorTaskHistoryEventsTest {
assertNull(started.getDetails());
}
+ /**
+ * A Parallel branch and a Map iteration run states of their own whose events floci does not
+ * publish. The published history is still one unbroken chain: the ids of the events around them
+ * do not skip the numbers those states used.
+ */
+ @Test
+ void parallelAndMapPublishAnUnbrokenEventChain() throws Exception {
+ var history = new ArrayList<HistoryEvent>();
+ var execution = run("""
+ {
+ "StartAt": "Fan",
+ "States": {
+ "Fan": {
+ "Type": "Parallel",
+ "Branches": [{
+ "StartAt": "BranchStep",
+ "States": {"BranchStep": {"Type": "Pass", "Next": "BranchTail"},
+ "BranchTail": {"Type": "Pass", "End": true}}
+ }],
+ "Next": "Iterate"
+ },
+ "Iterate": {
+ "Type": "Map",
+ "ItemsPath": "$[0].items",
+ "ItemProcessor": {
+ "StartAt": "ItemStep",
+ "States": {"ItemStep": {"Type": "Pass", "End": true}}
+ },
+ "End": true
+ }
+ }
+ }
+ """, "{\"items\": [1, 2]}", null, history);
+
+ assertEquals("SUCCEEDED", execution.getStatus(), "history: " + typesOf(history));
+ assertEquals(
+ List.of("ParallelStateEntered", "ParallelStateExited",
+ "MapStateEntered", "MapStateExited", "ExecutionSucceeded"),
+ typesOf(history));
+ assertChain(history);
+ }
+
private static List<String> typesOf(List<HistoryEvent> history) {
return history.stream().map(HistoryEvent::getType).toList();
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationSchedulerTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationSchedulerTest.java
index cc2f72691d..2598c03121 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationSchedulerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/MapIterationSchedulerTest.java
@@ -10,6 +10,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -35,7 +36,7 @@ class MapIterationSchedulerTest {
List<Integer> results = MapIterationScheduler.execute(5, 1, index -> () -> {
starts.add(index);
return index;
- });
+ }, Long.MAX_VALUE);
assertEquals(List.of(0, 1, 2, 3, 4), starts);
assertEquals(starts, results);
@@ -64,7 +65,7 @@ class MapIterationSchedulerTest {
} finally {
active.decrementAndGet();
}
- }));
+ }, Long.MAX_VALUE));
assertTrue(firstWaveStarted.await(2, TimeUnit.SECONDS), "initial worker window did not start");
assertEquals(maxConcurrency, started.get(), "queued items must not be submitted early");
@@ -99,7 +100,7 @@ class MapIterationSchedulerTest {
fail("slow iteration did not start");
}
throw new IllegalStateException("later iteration failed");
- }));
+ }, Long.MAX_VALUE));
ExecutionException failure = assertThrows(ExecutionException.class,
() -> execution.get(2, TimeUnit.SECONDS));
@@ -123,8 +124,105 @@ class MapIterationSchedulerTest {
laterItemsCompleted.countDown();
}
return index;
- }));
+ }, Long.MAX_VALUE));
assertEquals(List.of(0, 1, 2), results);
}
+
+ @Test
+ void stopsStartingSerialIterationsOncePastTheDeadline() {
+ AtomicInteger started = new AtomicInteger();
+
+ assertThrows(TimeoutException.class, () -> MapIterationScheduler.execute(3, 1, index -> () -> {
+ started.incrementAndGet();
+ return index;
+ }, System.nanoTime() - 1));
+
+ assertEquals(0, started.get());
+ }
+
+ @Test
+ void endsTheSerialWaitForASingleItemOncePastTheDeadline() throws Exception {
+ CountDownLatch neverRelease = new CountDownLatch(1);
+ CountDownLatch blockedInterrupted = new CountDownLatch(1);
+
+ // Not try-with-resources: against the still-buggy serial path the iteration below never
+ // returns and is never interrupted, so ExecutorService#close() would block on
+ // awaitTermination forever. shutdownNow() in the finally block does not wait.
+ ExecutorService driver = Executors.newSingleThreadExecutor();
+ try {
+ Future<List<Integer>> execution = driver.submit(() -> MapIterationScheduler.execute(
+ 1, 3, index -> () -> {
+ try {
+ neverRelease.await();
+ return index;
+ } catch (InterruptedException e) {
+ blockedInterrupted.countDown();
+ throw e;
+ }
+ }, System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(200)));
+
+ ExecutionException failure = assertThrows(ExecutionException.class,
+ () -> execution.get(2, TimeUnit.SECONDS));
+ assertEquals(TimeoutException.class, failure.getCause().getClass());
+ assertTrue(blockedInterrupted.await(2, TimeUnit.SECONDS),
+ "the single iteration should be interrupted once the deadline passes");
+ } finally {
+ driver.shutdownNow();
+ }
+ }
+
+ @Test
+ void endsTheSerialWaitForMaxConcurrencyOneOncePastTheDeadline() throws Exception {
+ CountDownLatch neverRelease = new CountDownLatch(1);
+ CountDownLatch blockedInterrupted = new CountDownLatch(1);
+
+ // Not try-with-resources: see endsTheSerialWaitForASingleItemOncePastTheDeadline above.
+ ExecutorService driver = Executors.newSingleThreadExecutor();
+ try {
+ Future<List<Integer>> execution = driver.submit(() -> MapIterationScheduler.execute(
+ 2, 1, index -> () -> {
+ try {
+ neverRelease.await();
+ return index;
+ } catch (InterruptedException e) {
+ blockedInterrupted.countDown();
+ throw e;
+ }
+ }, System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(200)));
+
+ ExecutionException failure = assertThrows(ExecutionException.class,
+ () -> execution.get(2, TimeUnit.SECONDS));
+ assertEquals(TimeoutException.class, failure.getCause().getClass());
+ assertTrue(blockedInterrupted.await(2, TimeUnit.SECONDS),
+ "the blocked iteration should be interrupted once the deadline passes");
+ } finally {
+ driver.shutdownNow();
+ }
+ }
+
+ @Test
+ void endsTheConcurrentWaitOncePastTheDeadline() throws Exception {
+ CountDownLatch neverRelease = new CountDownLatch(1);
+ CountDownLatch blockedInterrupted = new CountDownLatch(2);
+
+ try (ExecutorService driver = Executors.newSingleThreadExecutor()) {
+ Future<List<Integer>> execution = driver.submit(() -> MapIterationScheduler.execute(
+ 2, 2, index -> () -> {
+ try {
+ neverRelease.await();
+ return index;
+ } catch (InterruptedException e) {
+ blockedInterrupted.countDown();
+ throw e;
+ }
+ }, System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(200)));
+
+ ExecutionException failure = assertThrows(ExecutionException.class,
+ () -> execution.get(5, TimeUnit.SECONDS));
+ assertEquals(TimeoutException.class, failure.getCause().getClass());
+ assertTrue(blockedInterrupted.await(2, TimeUnit.SECONDS),
+ "the iterations still running should be interrupted once the deadline passes");
+ }
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsAwsSdkTaskIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsAwsSdkTaskIntegrationTest.java
index 3baf95ae21..ceee268ad9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsAwsSdkTaskIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsAwsSdkTaskIntegrationTest.java
@@ -284,10 +284,59 @@ class StepFunctionsAwsSdkTaskIntegrationTest {
assertEquals("Sfn.InvalidTokenException", describe.jsonPath().getString("error"));
}
+ @Test
+ @Order(9)
+ void sendTaskSuccessOnATokenWhoseResourceInvocationThrewFailsAsInvalid() throws Exception {
+ // "Leak" registers a task token, then fails invoking its own (unsupported) resource before it
+ // ever waits on that token. The registration must not outlive the failure: the token is
+ // recovered from the TaskScheduled event it wrote on its way to failing, and a second
+ // execution's SendTaskSuccess on that same token must find nothing pending for it.
+ var leakArn = createStateMachine("aws-sdk-task-leaked-token", """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Leak",
+ "States": {
+ "Leak": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::unsupported:doSomething.waitForTaskToken",
+ "Arguments": {"token": "{% $states.context.Task.Token %}"},
+ "End": true
+ }
+ }
+ }
+ """);
+
+ var leaked = waitForTerminalState(startExecution(leakArn, "{}"));
+ assertEquals("FAILED", leaked.jsonPath().getString("status"));
+ assertEquals("States.TaskFailed", leaked.jsonPath().getString("error"));
+ var taskToken = scheduledTaskToken(leaked.jsonPath().getString("executionArn"));
+
+ var senderArn = createStateMachine("aws-sdk-send-success-leaked-token", """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Resume",
+ "States": {
+ "Resume": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::aws-sdk:sfn:sendTaskSuccess",
+ "Arguments": {"TaskToken": "{% $states.input.token %}", "Output": {}},
+ "End": true
+ }
+ }
+ }
+ """);
+
+ var describe = waitForTerminalState(startExecution(senderArn, tokenInput(taskToken)));
+
+ assertEquals("FAILED", describe.jsonPath().getString("status"),
+ "Leak's token must be discarded once its resource invocation throws, not left pending");
+ assertEquals("Sfn.InvalidTokenException", describe.jsonPath().getString("error"));
+ }
+
// ──────────────────────────── scheduler:createSchedule / updateSchedule ────────────────────────────
@Test
- @Order(9)
+ @Order(10)
void createScheduleReturnsTheScheduleArnAndTheScheduleIsReadable() throws Exception {
var result = mapper.readTree(succeedingOutputOf(
createStateMachine("aws-sdk-create-schedule", scheduleTask("createSchedule",
@@ -303,7 +352,7 @@ class StepFunctionsAwsSdkTaskIntegrationTest {
}
@Test
- @Order(10)
+ @Order(11)
void updateScheduleKeepsTheArnAndChangesTheExpression() throws Exception {
var result = mapper.readTree(succeedingOutputOf(
createStateMachine("aws-sdk-update-schedule", scheduleTask("updateSchedule",
@@ -317,7 +366,7 @@ class StepFunctionsAwsSdkTaskIntegrationTest {
}
@Test
- @Order(11)
+ @Order(12)
void createScheduleOnANameAlreadyTakenFailsWithConflict() {
var describe = waitForTerminalState(startExecution(
createStateMachine("aws-sdk-create-schedule-twice", scheduleTask("createSchedule",
@@ -328,7 +377,7 @@ class StepFunctionsAwsSdkTaskIntegrationTest {
}
@Test
- @Order(12)
+ @Order(13)
void updateScheduleOnAScheduleThatDoesNotExistFailsWithResourceNotFound() {
var describe = waitForTerminalState(startExecution(
createStateMachine("aws-sdk-update-missing-schedule", scheduleTask("updateSchedule",
@@ -339,7 +388,7 @@ class StepFunctionsAwsSdkTaskIntegrationTest {
}
@Test
- @Order(13)
+ @Order(14)
void createScheduleWithIncompleteEventBridgeParametersIsCatchableAsAValidationException() throws Exception {
var smArn = createStateMachine("aws-sdk-create-schedule-invalid-target", """
{
@@ -379,7 +428,7 @@ class StepFunctionsAwsSdkTaskIntegrationTest {
}
@Test
- @Order(14)
+ @Order(15)
void createScheduleWithAMalformedListIsCatchableAsASerializationException() throws Exception {
// A malformed EcsParameters list is refused before the conversion runs, so it reaches Catch
// as the SDK exception AWS sends rather than as States.Runtime.
@@ -569,6 +618,29 @@ class StepFunctionsAwsSdkTaskIntegrationTest {
.when().post("/");
}
+ private static Response getExecutionHistory(String execArn) {
+ return given()
+ .header("X-Amz-Target", "AWSStepFunctions.GetExecutionHistory")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"executionArn": "%s"}
+ """.formatted(execArn))
+ .when().post("/");
+ }
+
+ /** The token a Task registered for itself, recovered from its own TaskScheduled event. */
+ private static String scheduledTaskToken(String execArn) throws Exception {
+ var events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");
+ for (var event : events) {
+ if ("TaskScheduled".equals(event.path("type").asText())) {
+ var parameters = event.path("taskScheduledEventDetails").path("parameters").asText();
+ return mapper.readTree(parameters).path("token").asText();
+ }
+ }
+ fail("no TaskScheduled event found in history: " + events);
+ return null;
+ }
+
private static Response waitForTerminalState(String execArn) {
for (var i = 0; i < 150; i++) {
var resp = describeExecution(execArn);
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsStopExecutionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsStopExecutionIntegrationTest.java
new file mode 100644
index 0000000000..846f002cef
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsStopExecutionIntegrationTest.java
@@ -0,0 +1,194 @@
+package io.github.hectorvent.floci.services.stepfunctions;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Guards that {@code StopExecution} stops the execution instead of relabelling it: the status,
+ * error and cause it writes are what {@code DescribeExecution} returns, and the worker thread that
+ * is still inside the {@code Wait} does not overwrite them when it gets there.
+ */
+@QuarkusTest
+class StepFunctionsStopExecutionIntegrationTest {
+
+ private static final String SFN_CONTENT_TYPE = "application/x-amz-json-1.0";
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/test-role";
+
+ /** Long enough that the stop lands mid-flight, short enough that the re-read is cheap. */
+ private static final int WAIT_SECONDS = 3;
+
+ private static final String WAITING_MACHINE = """
+ {"StartAt": "W", "States": {"W": {"Type": "Wait", "Seconds": %d, "End": true}}}
+ """.formatted(WAIT_SECONDS);
+
+ @Inject
+ StepFunctionsService stepFunctionsService;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ /**
+ * The executions survive a restart in persistent storage mode; their histories do not, because
+ * the history lives in memory. An execution that reloads as RUNNING with no history behind it is
+ * still stoppable, and StopExecution reports the abort it performed rather than an error. The
+ * same pairing happens during a reset, between the moment the in-memory state is dropped and
+ * the moment the stored executions are.
+ */
+ @Test
+ void executionWithNoHistoryInMemoryIsStillStoppable() throws Exception {
+ String executionArn = startWaitingExecution("stop-without-history");
+ Thread.sleep(300);
+
+ stepFunctionsService.clear();
+
+ stopExecution(executionArn, "Manual", "probe");
+
+ Response described = describeExecution(executionArn);
+ assertEquals("ABORTED", described.jsonPath().getString("status"));
+ assertEquals("Manual", described.jsonPath().getString("error"));
+ assertEquals("probe", described.jsonPath().getString("cause"));
+ }
+
+ @Test
+ void stoppedExecutionReportsTheCallersErrorAndCause() throws Exception {
+ String executionArn = startWaitingExecution("stop-reports-error");
+ Thread.sleep(300);
+
+ stopExecution(executionArn, "Manual", "probe");
+
+ Response described = describeExecution(executionArn);
+ assertEquals("ABORTED", described.jsonPath().getString("status"));
+ assertEquals("Manual", described.jsonPath().getString("error"));
+ assertEquals("probe", described.jsonPath().getString("cause"));
+ }
+
+ @Test
+ void stoppedExecutionStaysAbortedAfterTheWaitElapses() throws Exception {
+ String executionArn = startWaitingExecution("stop-survives-wait");
+ Thread.sleep(300);
+
+ stopExecution(executionArn, "Manual", "probe");
+ double stopDate = describeExecution(executionArn).jsonPath().getDouble("stopDate");
+
+ Thread.sleep((WAIT_SECONDS + 2) * 1000L);
+
+ Response described = describeExecution(executionArn);
+ assertEquals("ABORTED", described.jsonPath().getString("status"));
+ assertEquals("Manual", described.jsonPath().getString("error"));
+ assertEquals("probe", described.jsonPath().getString("cause"));
+ assertEquals(stopDate, described.jsonPath().getDouble("stopDate"));
+ }
+
+ @Test
+ void abortedExecutionEndsItsHistoryAtExecutionAborted() throws Exception {
+ String executionArn = startWaitingExecution("stop-ends-history");
+ Thread.sleep(300);
+
+ stopExecution(executionArn, "Manual", "probe");
+
+ // The worker is still inside the Wait. Let it come out and try to record the state it left.
+ Thread.sleep((WAIT_SECONDS + 2) * 1000L);
+
+ List<Map<String, Object>> events = getExecutionHistory(executionArn).jsonPath().getList("events");
+ List<String> types = events.stream().map(event -> (String) event.get("type")).toList();
+ List<Object> ids = events.stream().map(event -> event.get("id")).toList();
+
+ assertAll(
+ () -> assertEquals("ExecutionAborted", types.get(types.size() - 1),
+ "history continued past the terminal event: " + types),
+ () -> assertEquals(ids.size(), new HashSet<>(ids).size(),
+ "history repeats an event id: " + ids + " for " + types));
+ }
+
+ private String startWaitingExecution(String namePrefix) {
+ String name = namePrefix + "-" + UUID.randomUUID();
+ String stateMachineArn = createStateMachine(name, WAITING_MACHINE);
+ Response response = given()
+ .header("X-Amz-Target", "AWSStepFunctions.StartExecution")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"stateMachineArn": "%s", "input": "{}"}
+ """.formatted(stateMachineArn))
+ .when()
+ .post("/");
+ response.then().statusCode(200);
+ return response.jsonPath().getString("executionArn");
+ }
+
+ private String createStateMachine(String name, String definition) {
+ Response response = given()
+ .header("X-Amz-Target", "AWSStepFunctions.CreateStateMachine")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"name": "%s", "definition": %s, "roleArn": "%s"}
+ """.formatted(name, quote(definition), ROLE_ARN))
+ .when()
+ .post("/");
+ response.then().statusCode(200);
+ return response.jsonPath().getString("stateMachineArn");
+ }
+
+ private void stopExecution(String executionArn, String error, String cause) {
+ given()
+ .header("X-Amz-Target", "AWSStepFunctions.StopExecution")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"executionArn": "%s", "error": "%s", "cause": "%s"}
+ """.formatted(executionArn, error, cause))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
+
+ private Response describeExecution(String executionArn) {
+ Response response = given()
+ .header("X-Amz-Target", "AWSStepFunctions.DescribeExecution")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"executionArn": "%s"}
+ """.formatted(executionArn))
+ .when()
+ .post("/");
+ response.then().statusCode(200);
+ return response;
+ }
+
+ private Response getExecutionHistory(String executionArn) {
+ Response response = given()
+ .header("X-Amz-Target", "AWSStepFunctions.GetExecutionHistory")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"executionArn": "%s", "includeExecutionData": true}
+ """.formatted(executionArn))
+ .when()
+ .post("/");
+ response.then().statusCode(200);
+ return response;
+ }
+
+ private static String quote(String raw) {
+ return "\"" + raw
+ .replace("\\", "\\\\")
+ .replace("\"", "\\\"")
+ .replace("\n", "\\n")
+ .replace("\r", "\\r")
+ .replace("\t", "\\t")
+ + "\"";
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsTimeoutSecondsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsTimeoutSecondsIntegrationTest.java
new file mode 100644
index 0000000000..746be51b18
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsTimeoutSecondsIntegrationTest.java
@@ -0,0 +1,553 @@
+package io.github.hectorvent.floci.services.stepfunctions;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * The two {@code TimeoutSeconds} fields of ASL, each enforced on its own clock: the state
+ * machine's total budget and a {@code Task}'s wait for its task token, the second one bounded a
+ * second time by {@code HeartbeatSeconds}.
+ *
+ * <p>The bounds here are one to ten seconds where the report that measured them against us-east-1
+ * used three and five; the shape of the terminal execution is what the assertions pin, not the
+ * duration.
+ */
+@QuarkusTest
+class StepFunctionsTimeoutSecondsIntegrationTest {
+
+ private static final String SFN_CONTENT_TYPE = "application/x-amz-json-1.0";
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/test-role";
+ private static final ObjectMapper mapper = new ObjectMapper();
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void stateMachineTimeoutSecondsCutsAWaitLongerThanTheRemainingBudget() throws Exception {
+ String smArn = createStateMachine("top-level-timeout-wait", """
+ {
+ "StartAt": "Linger",
+ "TimeoutSeconds": 1,
+ "States": {
+ "Linger": {"Type": "Wait", "Seconds": 3, "End": true}
+ }
+ }
+ """);
+
+ Response timedOut = waitForTerminalExecution(startExecution(smArn, "{}"));
+
+ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"));
+ JsonNode body = mapper.readTree(timedOut.body().asString());
+ assertTrue(body.has("stopDate"), "stopDate must be set on a timed out execution");
+ assertFalse(body.has("error"), "TIMED_OUT carries no error key: " + body);
+ assertFalse(body.has("cause"), "TIMED_OUT carries no cause key: " + body);
+ }
+
+ @Test
+ void timedOutExecutionEmitsExecutionTimedOutAndNoStateExitedForTheCutState() throws Exception {
+ String smArn = createStateMachine("top-level-timeout-history", """
+ {
+ "StartAt": "Linger",
+ "TimeoutSeconds": 1,
+ "States": {
+ "Linger": {"Type": "Wait", "Seconds": 3, "End": true}
+ }
+ }
+ """);
+
+ String execArn = startExecution(smArn, "{}");
+ waitForTerminalExecution(execArn);
+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");
+
+ assertEquals(List.of("ExecutionStarted", "WaitStateEntered", "ExecutionTimedOut"),
+ eventTypes(events), "history was " + events);
+ JsonNode timedOut = events.get(events.size() - 1);
+ assertEquals(0, timedOut.path("previousEventId").asLong());
+ assertEquals("States.Timeout",
+ timedOut.path("executionTimedOutEventDetails").path("error").asText());
+ assertFalse(timedOut.path("executionTimedOutEventDetails").has("cause"),
+ "ExecutionTimedOut carries only the error: " + timedOut);
+ }
+
+ @Test
+ void stateMachineTimeoutSecondsCutsTheWaitForAParallelBranch() throws Exception {
+ // A branch is never cut mid-state, so its Wait sleeps out in full on its own thread. What
+ // the budget cuts is the join: the execution ends while the branch is still running, and
+ // the Parallel state gets no Exited event and no state after it is entered.
+ String smArn = createStateMachine("top-level-timeout-loop", """
+ {
+ "StartAt": "Fork",
+ "TimeoutSeconds": 1,
+ "States": {
+ "Fork": {
+ "Type": "Parallel",
+ "Branches": [{
+ "StartAt": "Linger",
+ "States": {"Linger": {"Type": "Wait", "Seconds": 3, "End": true}}
+ }],
+ "Next": "Unreached"
+ },
+ "Unreached": {"Type": "Pass", "End": true}
+ }
+ }
+ """);
+
+ String execArn = startExecution(smArn, "{}");
+ long startedAt = System.currentTimeMillis();
+ Response timedOut = waitForTerminalExecution(execArn);
+ long elapsedMillis = System.currentTimeMillis() - startedAt;
+
+ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"));
+ assertTrue(elapsedMillis < 3_000,
+ "the branch's 3-second Wait outlasted the 1-second budget: " + elapsedMillis + " ms");
+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");
+ assertEquals(List.of("ExecutionStarted", "ParallelStateEntered", "ExecutionTimedOut"),
+ eventTypes(events), "history was " + events);
+ }
+
+ @Test
+ void stateMachineTimeoutSecondsCutsTheWaitForATaskToken() throws Exception {
+ // Measured against us-east-1: a state machine whose budget is shorter than the Task's own
+ // TimeoutSeconds ends TIMED_OUT the instant the budget runs out, and writes nothing about
+ // the task it cut. No ActivityTimedOut, no TaskFailed, no ExecutionFailed.
+ String activityArn = createActivity("top-level-timeout-token");
+ String smArn = createStateMachine("top-level-timeout-token", """
+ {
+ "StartAt": "Await",
+ "TimeoutSeconds": 1,
+ "States": {
+ "Await": {
+ "Type": "Task",
+ "Resource": "%s",
+ "TimeoutSeconds": 60,
+ "End": true
+ }
+ }
+ }
+ """.formatted(activityArn));
+
+ String execArn = startExecution(smArn, "{}");
+ long startedAt = System.currentTimeMillis();
+ Response timedOut = waitForTerminalExecution(execArn);
+ long elapsedMillis = System.currentTimeMillis() - startedAt;
+
+ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"), timedOut.body().asString());
+ assertTrue(elapsedMillis < 5_000,
+ "the task's 60-second wait outlasted the 1-second budget: " + elapsedMillis + " ms");
+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");
+ assertEquals(List.of("ExecutionStarted", "TaskStateEntered", "ActivityScheduled",
+ "ActivityStarted", "ExecutionTimedOut"), eventTypes(events), "history was " + events);
+ }
+
+ @Test
+ void stateMachineTimeoutSecondsCutsTheWaitForANestedExecution() throws Exception {
+ // The child sleeps far past the parent's budget. The parent ends TIMED_OUT where its budget
+ // runs out; the child keeps running, since nothing in ASL propagates the parent's timeout.
+ String childArn = createStateMachine("nested-child", """
+ {
+ "StartAt": "Linger",
+ "States": {"Linger": {"Type": "Wait", "Seconds": 10, "End": true}}
+ }
+ """);
+ String parentArn = createStateMachine("nested-parent", """
+ {
+ "StartAt": "RunChild",
+ "TimeoutSeconds": 1,
+ "States": {
+ "RunChild": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::states:startExecution.sync",
+ "Parameters": {"StateMachineArn": "%s", "Input": {}},
+ "End": true
+ }
+ }
+ }
+ """.formatted(childArn));
+
+ String execArn = startExecution(parentArn, "{}");
+ long startedAt = System.currentTimeMillis();
+ Response timedOut = waitForTerminalExecution(execArn);
+ long elapsedMillis = System.currentTimeMillis() - startedAt;
+
+ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"), timedOut.body().asString());
+ assertTrue(elapsedMillis < 5_000,
+ "the child's 10-second Wait outlasted the 1-second budget: " + elapsedMillis + " ms");
+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");
+ assertEquals("ExecutionTimedOut", events.get(events.size() - 1).path("type").asText(),
+ "history was " + events);
+ }
+
+ @Test
+ void stateMachineTimeoutSecondsCutsTheWaitForMapIterations() throws Exception {
+ // Two iterations run concurrently and each sleeps past the budget, so what the budget cuts
+ // is the Map state's wait for them rather than any single iteration.
+ String smArn = createStateMachine("top-level-timeout-map", """
+ {
+ "StartAt": "OverItems",
+ "TimeoutSeconds": 1,
+ "States": {
+ "OverItems": {
+ "Type": "Map",
+ "ItemsPath": "$.items",
+ "ItemProcessor": {
+ "StartAt": "Linger",
+ "States": {"Linger": {"Type": "Wait", "Seconds": 10, "End": true}}
+ },
+ "End": true
+ }
+ }
+ }
+ """);
+
+ String execArn = startExecution(smArn, "{\"items\": [1, 2]}");
+ long startedAt = System.currentTimeMillis();
+ Response timedOut = waitForTerminalExecution(execArn);
+ long elapsedMillis = System.currentTimeMillis() - startedAt;
+
+ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"), timedOut.body().asString());
+ assertTrue(elapsedMillis < 5_000,
+ "the iterations' 10-second Wait outlasted the 1-second budget: " + elapsedMillis + " ms");
+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");
+ assertEquals(List.of("ExecutionStarted", "MapStateEntered", "ExecutionTimedOut"),
+ eventTypes(events), "history was " + events);
+ }
+
+ @Test
+ void stateMachineTimeoutSecondsCutsTheWaitForASingleMapIteration() throws Exception {
+ // A single item runs the Map state's serial path rather than the concurrent one; the
+ // budget must cut it the same way it cuts a concurrent iteration above.
+ String smArn = createStateMachine("top-level-timeout-map-single-item", """
+ {
+ "StartAt": "OverItems",
+ "TimeoutSeconds": 1,
+ "States": {
+ "OverItems": {
+ "Type": "Map",
+ "ItemsPath": "$.items",
+ "ItemProcessor": {
+ "StartAt": "Linger",
+ "States": {"Linger": {"Type": "Wait", "Seconds": 10, "End": true}}
+ },
+ "End": true
+ }
+ }
+ }
+ """);
+
+ String execArn = startExecution(smArn, "{\"items\": [1]}");
+ long startedAt = System.currentTimeMillis();
+ Response timedOut = waitForTerminalExecution(execArn);
+ long elapsedMillis = System.currentTimeMillis() - startedAt;
+
+ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"), timedOut.body().asString());
+ assertTrue(elapsedMillis < 5_000,
+ "the single iteration's 10-second Wait outlasted the 1-second budget: " + elapsedMillis + " ms");
+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");
+ assertEquals(List.of("ExecutionStarted", "MapStateEntered", "ExecutionTimedOut"),
+ eventTypes(events), "history was " + events);
+ }
+
+ @Test
+ void stateMachineTimeoutSecondsCutsARetryBackoffLongerThanTheRemainingBudget() throws Exception {
+ // The Lambda does not exist, so every attempt fails immediately and the only thing this
+ // state spends time on is the backoff between attempts. AWS ends the execution when the
+ // budget runs out, mid-backoff; before this the backoff ran in full and the state failed
+ // on its own error afterwards, so the execution ended FAILED past its budget.
+ String smArn = createStateMachine("top-level-timeout-retry", """
+ {
+ "StartAt": "Flaky",
+ "TimeoutSeconds": 1,
+ "States": {
+ "Flaky": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::lambda:invoke",
+ "Parameters": {"FunctionName": "absent-function"},
+ "Retry": [{"ErrorEquals": ["States.ALL"], "IntervalSeconds": 10, "MaxAttempts": 1}],
+ "End": true
+ }
+ }
+ }
+ """);
+
+ String execArn = startExecution(smArn, "{}");
+ long startedAt = System.currentTimeMillis();
+ Response timedOut = waitForTerminalExecution(execArn);
+ long elapsedMillis = System.currentTimeMillis() - startedAt;
+
+ assertEquals("TIMED_OUT", timedOut.jsonPath().getString("status"), timedOut.body().asString());
+ assertTrue(elapsedMillis < 5_000,
+ "the 10-second backoff outlasted the 1-second budget: " + elapsedMillis + " ms");
+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");
+ assertEquals("ExecutionTimedOut", events.get(events.size() - 1).path("type").asText(),
+ "history was " + events);
+ }
+
+ @Test
+ void taskTimeoutSecondsBoundsTheWholeWaitForTheTaskToken() throws Exception {
+ String activityArn = createActivity("timeout-seconds-total");
+ String smArn = createStateMachine("task-timeout-seconds", activityTask(activityArn, 1, 0));
+
+ Response failed = waitForTerminalExecution(startExecution(smArn, "{}"));
+
+ assertEquals("FAILED", failed.jsonPath().getString("status"));
+ assertEquals("States.Timeout", failed.jsonPath().getString("error"));
+ JsonNode body = mapper.readTree(failed.body().asString());
+ assertFalse(body.has("cause"),
+ "a task that ran out of time carries no cause key: " + body);
+ }
+
+ @Test
+ void taskHeartbeatSecondsBoundsTheGapBetweenHeartbeats() throws Exception {
+ String activityArn = createActivity("heartbeat-seconds-gap");
+ String smArn = createStateMachine("task-heartbeat-gap", activityTask(activityArn, 10, 1));
+ String execArn = startExecution(smArn, "{}");
+
+ // A worker picks the task up and then goes silent, so the gap HeartbeatSeconds allows runs
+ // out long before the state's TimeoutSeconds does.
+ getActivityTask(activityArn);
+ Response failed = waitForTerminalExecution(execArn);
+
+ assertEquals("FAILED", failed.jsonPath().getString("status"));
+ assertEquals("States.Timeout", failed.jsonPath().getString("error"));
+ JsonNode body = mapper.readTree(failed.body().asString());
+ assertFalse(body.has("cause"),
+ "a task that missed its heartbeats carries no cause key: " + body);
+
+ JsonNode events = mapper.readTree(getExecutionHistory(execArn).body().asString()).path("events");
+ assertEquals(List.of("ExecutionStarted", "TaskStateEntered", "ActivityScheduled",
+ "ActivityStarted", "ActivityTimedOut", "ExecutionFailed"),
+ eventTypes(events), "history was " + events);
+ JsonNode timedOut = events.get(events.size() - 2);
+ assertEquals("States.Timeout",
+ timedOut.path("activityTimedOutEventDetails").path("error").asText());
+ assertFalse(timedOut.path("activityTimedOutEventDetails").has("cause"),
+ "ActivityTimedOut carries only the error: " + timedOut);
+ }
+
+ @Test
+ void aCatchMatchesAHeartbeatExpiryUnderEitherErrorName() throws Exception {
+ for (String errorEquals : List.of("States.HeartbeatTimeout", "States.Timeout")) {
+ String activityArn = createActivity("heartbeat-seconds-catch");
+ String smArn = createStateMachine("task-heartbeat-catch",
+ caughtActivityTask(activityArn, errorEquals));
+ String execArn = startExecution(smArn, "{}");
+
+ getActivityTask(activityArn);
+ Response caught = waitForTerminalExecution(execArn);
+
+ assertEquals("SUCCEEDED", caught.jsonPath().getString("status"),
+ "Catch on " + errorEquals + " left the execution " + caught.body().asString());
+ assertEquals("States.Timeout",
+ mapper.readTree(caught.jsonPath().getString("output")).path("Error").asText(),
+ "Catch on " + errorEquals + " reported the wrong error");
+ }
+ }
+
+ @Test
+ void heartbeatsKeepATaskAliveUntilItsTimeoutSecondsRunOut() throws Exception {
+ String activityArn = createActivity("heartbeat-seconds-reset");
+ String smArn = createStateMachine("task-heartbeat-reset", activityTask(activityArn, 12, 3));
+ String execArn = startExecution(smArn, "{}");
+
+ String taskToken = getActivityTask(activityArn);
+ // Five seconds of work reported every 500 ms: nearly twice the heartbeat gap, so the task
+ // survives only if each heartbeat pushes that gap's deadline forward.
+ for (int beat = 0; beat < 10; beat++) {
+ Thread.sleep(500);
+ sendTaskHeartbeat(taskToken);
+ }
+ sendTaskSuccess(taskToken, "{\"answered\":true}");
+
+ Response succeeded = waitForTerminalExecution(execArn);
+ assertEquals("SUCCEEDED", succeeded.jsonPath().getString("status"),
+ "execution was " + succeeded.body().asString());
+ assertTrue(mapper.readTree(succeeded.jsonPath().getString("output")).path("answered").asBoolean());
+ }
+
+ private static String activityTask(String activityArn, int timeoutSeconds, int heartbeatSeconds) {
+ String heartbeat = heartbeatSeconds > 0 ? ",\"HeartbeatSeconds\": " + heartbeatSeconds : "";
+ return """
+ {
+ "StartAt": "Await",
+ "States": {
+ "Await": {
+ "Type": "Task",
+ "Resource": "%s",
+ "TimeoutSeconds": %d%s,
+ "End": true
+ }
+ }
+ }
+ """.formatted(activityArn, timeoutSeconds, heartbeat);
+ }
+
+ private static String caughtActivityTask(String activityArn, String errorEquals) {
+ return """
+ {
+ "StartAt": "Await",
+ "States": {
+ "Await": {
+ "Type": "Task",
+ "Resource": "%s",
+ "TimeoutSeconds": 10,
+ "HeartbeatSeconds": 1,
+ "Catch": [{"ErrorEquals": ["%s"], "Next": "Recovered"}],
+ "End": true
+ },
+ "Recovered": {"Type": "Pass", "End": true}
+ }
+ }
+ """.formatted(activityArn, errorEquals);
+ }
+
+ private static List<String> eventTypes(JsonNode events) {
+ List<String> types = new ArrayList<>();
+ events.forEach(event -> types.add(event.path("type").asText()));
+ return types;
+ }
+
+ private static String createActivity(String name) {
+ Response resp = given()
+ .header("X-Amz-Target", "AWSStepFunctions.CreateActivity")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"name": "%s-%d"}
+ """.formatted(name, System.currentTimeMillis()))
+ .when()
+ .post("/");
+ resp.then().statusCode(200);
+ return resp.jsonPath().getString("activityArn");
+ }
+
+ private static String getActivityTask(String activityArn) {
+ Response resp = given()
+ .header("X-Amz-Target", "AWSStepFunctions.GetActivityTask")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"activityArn": "%s", "workerName": "timeout-seconds-worker"}
+ """.formatted(activityArn))
+ .when()
+ .post("/");
+ resp.then().statusCode(200);
+ String taskToken = resp.jsonPath().getString("taskToken");
+ assertFalse(taskToken == null || taskToken.isBlank(), "no activity task was queued");
+ return taskToken;
+ }
+
+ private static void sendTaskHeartbeat(String taskToken) {
+ given()
+ .header("X-Amz-Target", "AWSStepFunctions.SendTaskHeartbeat")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"taskToken": %s}
+ """.formatted(quote(taskToken)))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
+
+ private static void sendTaskSuccess(String taskToken, String output) {
+ given()
+ .header("X-Amz-Target", "AWSStepFunctions.SendTaskSuccess")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"taskToken": %s, "output": %s}
+ """.formatted(quote(taskToken), quote(output)))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
+
+ private static String createStateMachine(String name, String definition) {
+ Response resp = given()
+ .header("X-Amz-Target", "AWSStepFunctions.CreateStateMachine")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"name": "%s-%d", "definition": %s, "roleArn": "%s"}
+ """.formatted(name, System.currentTimeMillis(), quote(definition), ROLE_ARN))
+ .when()
+ .post("/");
+ resp.then().statusCode(200);
+ return resp.jsonPath().getString("stateMachineArn");
+ }
+
+ private static String startExecution(String smArn, String input) {
+ Response resp = given()
+ .header("X-Amz-Target", "AWSStepFunctions.StartExecution")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"stateMachineArn": "%s", "input": %s}
+ """.formatted(smArn, quote(input)))
+ .when()
+ .post("/");
+ resp.then().statusCode(200);
+ return resp.jsonPath().getString("executionArn");
+ }
+
+ private static Response describeExecution(String execArn) {
+ return given()
+ .header("X-Amz-Target", "AWSStepFunctions.DescribeExecution")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"executionArn": "%s"}
+ """.formatted(execArn))
+ .when()
+ .post("/");
+ }
+
+ private static Response getExecutionHistory(String execArn) {
+ return given()
+ .header("X-Amz-Target", "AWSStepFunctions.GetExecutionHistory")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"executionArn": "%s"}
+ """.formatted(execArn))
+ .when()
+ .post("/");
+ }
+
+ /**
+ * Polls for 20 seconds: twice the longest bound any definition here declares, and far short of
+ * the 300-second default a task that ignores TimeoutSeconds falls back on.
+ */
+ private static Response waitForTerminalExecution(String execArn) throws InterruptedException {
+ for (int poll = 0; poll < 200; poll++) {
+ Response resp = describeExecution(execArn);
+ String status = resp.jsonPath().getString("status");
+ if (!"RUNNING".equals(status)) {
+ return resp;
+ }
+ Thread.sleep(100);
+ }
+ fail("Execution " + execArn + " was still RUNNING after 20 seconds");
+ return null;
+ }
+
+ private static String quote(String raw) {
+ return "\"" + raw
+ .replace("\\", "\\\\")
+ .replace("\"", "\\\"")
+ .replace("\n", "\\n")
+ .replace("\r", "\\r")
+ .replace("\t", "\\t")
+ + "\"";
+ }
+}