Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(sfn): support Retry policies in Task, Parallel, and Map states (#2455)

TATSUNO “Taz” Yasuhiro56e74d64c6a5
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.md
index 7f4842758f..ada52fd411 100644
--- a/docs/services/step-functions.md
+++ b/docs/services/step-functions.md
@@ -51,6 +51,21 @@ Results remain in input order even when iterations finish out of order. If an it
the Map state fails promptly, cancels its active sibling iterations, and does not start queued
iterations.
+## Retry policies
+
+`Task`, `Parallel`, and `Map` states honor their `Retry` field. `ErrorEquals` matching
+follows AWS semantics, including the `States.ALL` and `States.TaskFailed` wildcards.
+`States.Runtime` is never retried. AWS defaults apply when fields are omitted
+(`MaxAttempts` 3, `IntervalSeconds` 1, `BackoffRate` 2.0), `MaxDelaySeconds` is honored,
+and each retrier keeps its own attempt counter. `Retry` is evaluated before `Catch`, and
+`$$.State.RetryCount` increments per attempt. Attempt counts, defaults, and backoff
+timing were verified against real AWS Step Functions.
+
+`JitterStrategy` supports `NONE` (the default) and `FULL`. `FULL` draws the delay
+uniformly between zero and the computed delay, as on AWS. One deviation. The delay
+between attempts is capped at 30 seconds, the same cap Floci applies to `Wait` states,
+so emulated runs stay fast.
+
## Configuration
| Variable | Default | Description |
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index 3dc55e37d2..5ab24a1d21 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -75,6 +75,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.BiConsumer;
@@ -311,9 +312,9 @@ public class AslExecutor {
// Update per-state context fields
updateStateContext(execContext, currentStateName);
- boolean jsonata = isJsonata(stateDef, topLevelQueryLanguage);
+ var jsonata = isJsonata(stateDef, topLevelQueryLanguage);
try {
- StateResult stateResult = executeState(currentStateName, type, stateDef, currentInput,
+ var stateResult = executeStateWithRetry(currentStateName, type, stateDef, currentInput,
history, eventId, sm, jsonata, topLevelQueryLanguage, execContext, variables);
addEvent(history, eventId, stateExitedEventType(type), eventId.get() - 1,
Map.of("name", currentStateName, "output", stateResult.output().toString()));
@@ -373,6 +374,84 @@ public class AslExecutor {
}
}
+ /**
+ * Executes a state, honoring its {@code Retry} policy: a {@code FailStateException} matched by
+ * a retrier re-runs the state after the retrier's backoff until its {@code MaxAttempts} are
+ * used up. Errors that no retrier matches (or that exhaust their retrier) propagate to the
+ * caller's Catch handling, preserving Retry-before-Catch order.
+ */
+ private StateResult executeStateWithRetry(String name, String type, JsonNode stateDef, JsonNode input,
+ List<HistoryEvent> history, AtomicLong eventId, StateMachine sm,
+ boolean jsonata, String topLevelQueryLanguage, JsonNode context,
+ ObjectNode variables) throws Exception {
+ var retriers = stateDef.path("Retry");
+ var attemptsPerRetrier = new HashMap<Integer, Integer>();
+ var attempt = 0;
+ while (true) {
+ try {
+ return executeState(name, type, stateDef, input, history, eventId, sm, jsonata,
+ topLevelQueryLanguage, context, variables);
+ } catch (FailStateException e) {
+ var retrierIndex = findMatchingRetrier(retriers, e);
+ if (retrierIndex < 0) {
+ throw e;
+ }
+ var retrier = retriers.get(retrierIndex);
+ var attemptsUsed = attemptsPerRetrier.merge(retrierIndex, 1, Integer::sum);
+ if (attemptsUsed > retrier.path("MaxAttempts").asInt(3)) {
+ throw e;
+ }
+ sleepBeforeRetry(retrier, attemptsUsed);
+ attempt++;
+ updateRetryCount(context, attempt);
+ }
+ }
+ }
+
+ private int findMatchingRetrier(JsonNode retriers, FailStateException failure) {
+ if (!retriers.isArray()) {
+ return -1;
+ }
+ var error = failure.error != null ? failure.error : "States.Runtime";
+ for (var i = 0; i < retriers.size(); i++) {
+ if (catchMatches(retriers.get(i), error)) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ private void sleepBeforeRetry(JsonNode retrier, int attemptsUsed) throws InterruptedException {
+ var delaySeconds = retryDelaySeconds(retrier, attemptsUsed, ThreadLocalRandom.current().nextDouble());
+ if (delaySeconds > 0) {
+ Thread.sleep((long) (delaySeconds * 1000));
+ }
+ }
+
+ /**
+ * Computes the delay before a retry attempt. {@code random} is a value in [0, 1) used when
+ * the retrier declares {@code JitterStrategy: FULL}, which draws the delay uniformly between
+ * zero and the computed delay. Jitter applies after the caps, matching AWS.
+ */
+ static double retryDelaySeconds(JsonNode retrier, int attemptsUsed, double random) {
+ var interval = retrier.path("IntervalSeconds").asDouble(1.0);
+ var backoffRate = retrier.path("BackoffRate").asDouble(2.0);
+ var delaySeconds = interval * Math.pow(backoffRate, attemptsUsed - 1.0);
+ var maxDelay = retrier.path("MaxDelaySeconds").asDouble(MAX_WAIT_SECONDS);
+ // Like the Wait state, cap the delay at MAX_WAIT_SECONDS to keep emulated runs fast.
+ delaySeconds = Math.min(delaySeconds, Math.min(maxDelay, MAX_WAIT_SECONDS));
+ if ("FULL".equals(retrier.path("JitterStrategy").asText(null))) {
+ delaySeconds *= random;
+ }
+ return delaySeconds;
+ }
+
+ private void updateRetryCount(JsonNode context, int retryCount) {
+ if (context.get("State") instanceof ObjectNode state) {
+ state.put("RetryCount", retryCount);
+ }
+ }
+
private StateResult executeState(String name, String type, JsonNode stateDef, JsonNode input,
List<HistoryEvent> history, AtomicLong eventId, StateMachine sm,
boolean jsonata, String topLevelQueryLanguage, JsonNode context,
@@ -540,7 +619,10 @@ public class AslExecutor {
String region = extractRegionFromArn(sm.getStateMachineArn());
LambdaFunction fn = functionStore.get(region, functionName).orElse(null);
if (fn == null) {
- throw new RuntimeException("Lambda function not found: " + functionName);
+ // A missing function is a task failure on AWS, so it must stay reachable for
+ // Retry and Catch instead of surfacing as States.Runtime.
+ throw new FailStateException("Lambda.ResourceNotFoundException",
+ "Lambda function not found: " + functionName);
}
String payloadStr = objectMapper.writeValueAsString(lambdaPayload);
@@ -1443,13 +1525,16 @@ public class AslExecutor {
// Each branch gets an isolated copy of the current variables: assignments inside a
// branch are scoped to that branch and do not leak back to the parent after the state.
ObjectNode branchVariables = variables.deepCopy();
+ // Each branch also gets its own copy of the context object so State.RetryCount and
+ // Task.Token writes cannot race across concurrent branches.
+ var branchContext = ((ObjectNode) context).deepCopy();
// Run each branch on its own worker thread under the execution's account: the request
// scope is thread-bound, so without this a branch's Task integrations would resolve to
// the default account rather than the execution's.
futures.add(executor.submit(() -> callUnderExecutionAccount(sm,
- () -> executeBranch(startAt, branchStates, capturedInput, sm, topLevelQueryLanguage, context,
- branchVariables))));
+ () -> executeBranch(startAt, branchStates, capturedInput, sm, topLevelQueryLanguage,
+ branchContext, branchVariables))));
}
int timeoutSeconds = stateDef.path("TimeoutSeconds").asInt(0);
@@ -1476,6 +1561,16 @@ public class AslExecutor {
futures.forEach(future -> future.cancel(true));
Thread.currentThread().interrupt();
throw e;
+ } catch (ExecutionException e) {
+ futures.forEach(future -> future.cancel(true));
+ // Unwrap so a branch's FailStateException reaches the Parallel state's own
+ // Retry and Catch handling instead of surfacing as States.Runtime. An Error
+ // cause stays wrapped so the execution-level Exception handlers still
+ // publish a terminal FAILED update instead of leaving the execution RUNNING.
+ if (e.getCause() instanceof Exception exception) {
+ throw exception;
+ }
+ throw e;
} catch (Exception | Error e) {
futures.forEach(future -> future.cancel(true));
throw e;
@@ -1997,9 +2092,10 @@ public class AslExecutor {
}
String type = stateDef.path("Type").asText();
boolean stateJsonata = isJsonata(stateDef, topLevelQueryLanguage);
+ updateStateContext(context, currentState);
StateResult result;
try {
- result = executeState(currentState, type, stateDef, currentInput, ignored, eventId, sm,
+ result = executeStateWithRetry(currentState, type, stateDef, currentInput, ignored, eventId, sm,
stateJsonata, topLevelQueryLanguage, context, variables);
} catch (FailStateException e) {
StateResult caught = handleCatch(stateDef, currentInput, e, stateJsonata, context, variables);
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorRetryTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorRetryTest.java
new file mode 100644
index 0000000000..1ae1ad1818
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorRetryTest.java
@@ -0,0 +1,432 @@
+package io.github.hectorvent.floci.services.stepfunctions;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbJsonHandler;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;
+import io.github.hectorvent.floci.services.lambda.LambdaExecutorService;
+import io.github.hectorvent.floci.services.lambda.LambdaFunctionStore;
+import io.github.hectorvent.floci.services.lambda.model.InvocationType;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;
+import io.github.hectorvent.floci.services.stepfunctions.model.Execution;
+import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;
+import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;
+import io.quarkus.test.junit.QuarkusTest;
+import io.vertx.mutiny.core.Vertx;
+import jakarta.enterprise.inject.Instance;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Retry policy semantics of the ASL executor. Attempt counts, defaults, and backoff
+ * behavior were verified against real AWS Step Functions in us-east-1.
+ */
+@QuarkusTest
+class AslExecutorRetryTest {
+
+ private static final String REGION = "us-east-2";
+ private static final String ACCOUNT = "000000000000";
+ private static final String FLAKY_FUNCTION_NAME = "flaky-lambda";
+ private static final String FLAKY_FUNCTION_ARN =
+ "arn:aws:lambda:%s:%s:function:%s".formatted(REGION, ACCOUNT, FLAKY_FUNCTION_NAME);
+
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private LambdaExecutorService lambdaExecutor;
+ private LambdaFunctionStore functionStore;
+ private LambdaFunction flakyFunction;
+ private AslExecutor executor;
+
+ @Inject
+ Vertx vertx;
+
+ @BeforeEach
+ void setUp() {
+ lambdaExecutor = mock(LambdaExecutorService.class);
+ functionStore = mock(LambdaFunctionStore.class);
+ flakyFunction = new LambdaFunction();
+ flakyFunction.setFunctionName(FLAKY_FUNCTION_NAME);
+ flakyFunction.setFunctionArn(FLAKY_FUNCTION_ARN);
+ when(functionStore.get(REGION, FLAKY_FUNCTION_NAME)).thenReturn(Optional.of(flakyFunction));
+
+ executor = new AslExecutor(
+ lambdaExecutor,
+ functionStore,
+ mock(DynamoDbService.class),
+ mock(DynamoDbJsonHandler.class),
+ mock(SqsJsonHandler.class),
+ mock(io.github.hectorvent.floci.services.cloudformation.CloudFormationQueryHandler.class),
+ mock(io.github.hectorvent.floci.services.ec2.Ec2Service.class),
+ mock(S3Service.class),
+ mock(io.github.hectorvent.floci.services.ecs.EcsService.class),
+ mock(io.github.hectorvent.floci.services.ecs.EcsJsonHandler.class),
+ objectMapper,
+ new JsonataEvaluator(objectMapper),
+ mock(Instance.class), mock(EmulatorConfig.class), vertx);
+ }
+
+ @Test
+ void retriesUntilTheTaskSucceeds() throws Exception {
+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult(200, "Handled",
+ "{\"errorType\":\"Boom\"}".getBytes(StandardCharsets.UTF_8), null, "req-1"))
+ .thenReturn(new InvokeResult(200, null,
+ "{\"ok\":true}".getBytes(StandardCharsets.UTF_8), null, "req-2"));
+
+ var execution = run("""
+ {
+ "StartAt": "Flaky",
+ "States": {
+ "Flaky": {
+ "Type": "Task",
+ "Resource": "%s",
+ "End": true,
+ "Retry": [{
+ "ErrorEquals": ["Lambda.AWSLambdaException"],
+ "IntervalSeconds": 1,
+ "BackoffRate": 1.0,
+ "MaxAttempts": 2
+ }]
+ }
+ }
+ }
+ """.formatted(FLAKY_FUNCTION_ARN));
+
+ assertEquals("SUCCEEDED", execution.getStatus());
+ assertTrue(objectMapper.readTree(execution.getOutput()).path("ok").asBoolean());
+ verify(lambdaExecutor, times(2))
+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));
+ }
+
+ @Test
+ void exhaustedRetriesFailWithOriginalError() {
+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult(200, "Handled",
+ "{\"errorType\":\"Boom\"}".getBytes(StandardCharsets.UTF_8), null, "req-1"));
+
+ var execution = run("""
+ {
+ "StartAt": "Flaky",
+ "States": {
+ "Flaky": {
+ "Type": "Task",
+ "Resource": "%s",
+ "End": true,
+ "Retry": [{
+ "ErrorEquals": ["Lambda.AWSLambdaException"],
+ "IntervalSeconds": 1,
+ "BackoffRate": 1.0,
+ "MaxAttempts": 1
+ }]
+ }
+ }
+ }
+ """.formatted(FLAKY_FUNCTION_ARN));
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("Lambda.AWSLambdaException", execution.getError());
+ verify(lambdaExecutor, times(2))
+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));
+ }
+
+ @Test
+ void unmatchedErrorIsNotRetried() {
+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult(200, "Handled",
+ "{\"errorType\":\"Boom\"}".getBytes(StandardCharsets.UTF_8), null, "req-1"));
+
+ var execution = run("""
+ {
+ "StartAt": "Flaky",
+ "States": {
+ "Flaky": {
+ "Type": "Task",
+ "Resource": "%s",
+ "End": true,
+ "Retry": [{
+ "ErrorEquals": ["SomeOther.Error"],
+ "IntervalSeconds": 1,
+ "MaxAttempts": 3
+ }]
+ }
+ }
+ }
+ """.formatted(FLAKY_FUNCTION_ARN));
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("Lambda.AWSLambdaException", execution.getError());
+ verify(lambdaExecutor, times(1))
+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));
+ }
+
+ @Test
+ void exhaustedRetrierFallsThroughToCatch() throws Exception {
+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult(200, "Handled",
+ "{\"errorType\":\"Boom\"}".getBytes(StandardCharsets.UTF_8), null, "req-1"));
+
+ var execution = run("""
+ {
+ "StartAt": "Flaky",
+ "States": {
+ "Flaky": {
+ "Type": "Task",
+ "Resource": "%s",
+ "End": true,
+ "Retry": [{
+ "ErrorEquals": ["States.ALL"],
+ "IntervalSeconds": 1,
+ "BackoffRate": 1.0,
+ "MaxAttempts": 1
+ }],
+ "Catch": [{
+ "ErrorEquals": ["States.ALL"],
+ "Next": "HandleError"
+ }]
+ },
+ "HandleError": {
+ "Type": "Pass",
+ "End": true
+ }
+ }
+ }
+ """.formatted(FLAKY_FUNCTION_ARN));
+
+ assertEquals("SUCCEEDED", execution.getStatus());
+ assertEquals("Lambda.AWSLambdaException",
+ objectMapper.readTree(execution.getOutput()).path("Error").asText());
+ verify(lambdaExecutor, times(2))
+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));
+ }
+
+ @Test
+ void parallelStateRetriesFailingBranch() throws Exception {
+ failOnceThenSucceed();
+
+ var execution = run("""
+ {
+ "StartAt": "Par",
+ "States": {
+ "Par": {
+ "Type": "Parallel",
+ "Branches": [{
+ "StartAt": "Inner",
+ "States": {"Inner": {"Type": "Task", "Resource": "%s", "End": true}}
+ }],
+ "Retry": [{
+ "ErrorEquals": ["Lambda.AWSLambdaException"],
+ "IntervalSeconds": 1,
+ "BackoffRate": 1.0,
+ "MaxAttempts": 1
+ }],
+ "End": true
+ }
+ }
+ }
+ """.formatted(FLAKY_FUNCTION_ARN));
+
+ assertEquals("SUCCEEDED", execution.getStatus());
+ assertTrue(objectMapper.readTree(execution.getOutput()).path(0).path("ok").asBoolean());
+ verify(lambdaExecutor, times(2))
+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));
+ }
+
+ @Test
+ void parallelBranchFailureReachesCatch() throws Exception {
+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult(200, "Handled",
+ "{\"errorType\":\"Boom\"}".getBytes(StandardCharsets.UTF_8), null, "req-1"));
+
+ var execution = run("""
+ {
+ "StartAt": "Par",
+ "States": {
+ "Par": {
+ "Type": "Parallel",
+ "Branches": [{
+ "StartAt": "Inner",
+ "States": {"Inner": {"Type": "Task", "Resource": "%s", "End": true}}
+ }],
+ "Catch": [{
+ "ErrorEquals": ["States.ALL"],
+ "Next": "HandleError"
+ }],
+ "End": true
+ },
+ "HandleError": {"Type": "Pass", "End": true}
+ }
+ }
+ """.formatted(FLAKY_FUNCTION_ARN));
+
+ assertEquals("SUCCEEDED", execution.getStatus());
+ assertEquals("Lambda.AWSLambdaException",
+ objectMapper.readTree(execution.getOutput()).path("Error").asText());
+ verify(lambdaExecutor, times(1))
+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));
+ }
+
+ @Test
+ void taskInsideParallelBranchRetriesIndependently() throws Exception {
+ failOnceThenSucceed();
+
+ var execution = run("""
+ {
+ "StartAt": "Par",
+ "States": {
+ "Par": {
+ "Type": "Parallel",
+ "Branches": [{
+ "StartAt": "Inner",
+ "States": {
+ "Inner": {
+ "Type": "Task",
+ "Resource": "%s",
+ "End": true,
+ "Retry": [{
+ "ErrorEquals": ["Lambda.AWSLambdaException"],
+ "IntervalSeconds": 1,
+ "BackoffRate": 1.0,
+ "MaxAttempts": 1
+ }]
+ }
+ }
+ }],
+ "End": true
+ }
+ }
+ }
+ """.formatted(FLAKY_FUNCTION_ARN));
+
+ assertEquals("SUCCEEDED", execution.getStatus());
+ assertTrue(objectMapper.readTree(execution.getOutput()).path(0).path("ok").asBoolean());
+ verify(lambdaExecutor, times(2))
+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));
+ }
+
+ @Test
+ void mapStateRetriesFailingIteration() throws Exception {
+ failOnceThenSucceed();
+
+ var execution = run("""
+ {
+ "StartAt": "Loop",
+ "States": {
+ "Loop": {
+ "Type": "Map",
+ "ItemsPath": "$.items",
+ "ItemProcessor": {
+ "ProcessorConfig": {"Mode": "INLINE"},
+ "StartAt": "Inner",
+ "States": {"Inner": {"Type": "Task", "Resource": "%s", "End": true}}
+ },
+ "Retry": [{
+ "ErrorEquals": ["Lambda.AWSLambdaException"],
+ "IntervalSeconds": 1,
+ "BackoffRate": 1.0,
+ "MaxAttempts": 1
+ }],
+ "End": true
+ }
+ }
+ }
+ """.formatted(FLAKY_FUNCTION_ARN), "{\"items\": [1]}");
+
+ assertEquals("SUCCEEDED", execution.getStatus());
+ assertTrue(objectMapper.readTree(execution.getOutput()).path(0).path("ok").asBoolean());
+ verify(lambdaExecutor, times(2))
+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));
+ }
+
+ @Test
+ void branchErrorStillFailsTheExecution() {
+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenThrow(new AssertionError("boom"));
+
+ var execution = run("""
+ {
+ "StartAt": "Par",
+ "States": {
+ "Par": {
+ "Type": "Parallel",
+ "Branches": [{
+ "StartAt": "Inner",
+ "States": {"Inner": {"Type": "Task", "Resource": "%s", "End": true}}
+ }],
+ "End": true
+ }
+ }
+ }
+ """.formatted(FLAKY_FUNCTION_ARN));
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.Runtime", execution.getError());
+ }
+
+ @Test
+ void retryDelayFollowsBackoffAndCaps() throws Exception {
+ assertEquals(10.0, delay("{\"IntervalSeconds\": 10, \"BackoffRate\": 2.0}", 1, 0.5));
+ assertEquals(20.0, delay("{\"IntervalSeconds\": 10, \"BackoffRate\": 2.0}", 2, 0.5));
+ assertEquals(15.0, delay("{\"IntervalSeconds\": 10, \"BackoffRate\": 2.0, \"MaxDelaySeconds\": 15}", 2, 0.5));
+ assertEquals(30.0, delay("{\"IntervalSeconds\": 20, \"BackoffRate\": 2.0}", 2, 0.5));
+ }
+
+ @Test
+ void fullJitterScalesTheDelayByTheRandomFactor() throws Exception {
+ var retrier = "{\"IntervalSeconds\": 10, \"BackoffRate\": 1.0, \"JitterStrategy\": \"FULL\"}";
+ assertEquals(0.0, delay(retrier, 1, 0.0));
+ assertEquals(5.0, delay(retrier, 1, 0.5));
+ assertEquals(10.0, delay("{\"IntervalSeconds\": 10, \"BackoffRate\": 1.0, \"JitterStrategy\": \"NONE\"}", 1, 0.5));
+ }
+
+ private double delay(String retrierJson, int attemptsUsed, double random) throws Exception {
+ return AslExecutor.retryDelaySeconds(objectMapper.readTree(retrierJson), attemptsUsed, random);
+ }
+
+ private void failOnceThenSucceed() {
+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult(200, "Handled",
+ "{\"errorType\":\"Boom\"}".getBytes(StandardCharsets.UTF_8), null, "req-1"))
+ .thenReturn(new InvokeResult(200, null,
+ "{\"ok\":true}".getBytes(StandardCharsets.UTF_8), null, "req-2"));
+ }
+
+ private Execution run(String definition) {
+ return run(definition, "{}");
+ }
+
+ private Execution run(String definition, String input) {
+ var stateMachine = new StateMachine();
+ stateMachine.setName("retry-test");
+ stateMachine.setStateMachineArn("arn:aws:states:%s:%s:stateMachine:retry-test".formatted(REGION, ACCOUNT));
+ stateMachine.setRoleArn("arn:aws:iam::%s:role/test-role".formatted(ACCOUNT));
+ stateMachine.setDefinition(definition);
+
+ var execution = new Execution();
+ execution.setName("retry-test-execution");
+ execution.setExecutionArn(
+ "arn:aws:states:%s:%s:execution:retry-test:retry-test-execution".formatted(REGION, ACCOUNT));
+ execution.setStateMachineArn(stateMachine.getStateMachineArn());
+ execution.setInput(input);
+
+ var history = new ArrayList<HistoryEvent>();
+ executor.executeSync(stateMachine, execution, history, (updated, events) -> {
+ });
+ return execution;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsRetryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsRetryIntegrationTest.java
new file mode 100644
index 0000000000..ea51597443
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsRetryIntegrationTest.java
@@ -0,0 +1,234 @@
+package io.github.hectorvent.floci.services.stepfunctions;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * End-to-end Retry policy tests through the JSON 1.0 wire protocol.
+ *
+ * The fail-then-succeed case works without mocks by resolving the DynamoDB table name
+ * from {@code $$.State.RetryCount}. Attempt 0 targets a table that does not exist and
+ * fails. The retry re-resolves Parameters, targets the table that does exist, and
+ * succeeds. Real AWS re-resolves Parameters per attempt the same way (verified in
+ * us-east-1).
+ */
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class StepFunctionsRetryIntegrationTest {
+
+ private static final String SFN_CONTENT_TYPE = "application/x-amz-json-1.0";
+ private static final String DDB_CONTENT_TYPE = "application/x-amz-json-1.0";
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/test-role";
+ private static final String RETRY_TABLE = "sfn-retry-1";
+ private static final String MISSING_TABLE = "sfn-retry-none";
+ private static final ObjectMapper mapper = new ObjectMapper();
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(0)
+ void setup_createTargetTableForSecondAttempt() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.CreateTable")
+ .contentType(DDB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST"
+ }
+ """.formatted(RETRY_TABLE))
+ .when().post("/")
+ .then().statusCode(200);
+ }
+
+ @Test
+ @Order(1)
+ void taskIsRetriedAndSucceedsOnSecondAttempt() throws Exception {
+ var smArn = createStateMachine("sfn-retry-succeed-test", """
+ {
+ "StartAt": "Flaky",
+ "States": {
+ "Flaky": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::dynamodb:putItem",
+ "Parameters": {
+ "TableName.$": "States.Format('sfn-retry-{}', $$.State.RetryCount)",
+ "Item": {"pk": {"S": "attempt-win"}}
+ },
+ "Retry": [{
+ "ErrorEquals": ["DynamoDB.ResourceNotFoundException"],
+ "IntervalSeconds": 1,
+ "BackoffRate": 1.0,
+ "MaxAttempts": 2
+ }],
+ "End": true
+ }
+ }
+ }
+ """);
+ var execArn = startExecution(smArn, "{}");
+
+ var describe = waitForTerminalState(execArn);
+ assertEquals("SUCCEEDED", describe.jsonPath().getString("status"));
+
+ var getItem = given()
+ .header("X-Amz-Target", "DynamoDB_20120810.GetItem")
+ .contentType(DDB_CONTENT_TYPE)
+ .body("""
+ {"TableName": "%s", "Key": {"pk": {"S": "attempt-win"}}}
+ """.formatted(RETRY_TABLE))
+ .when().post("/");
+ getItem.then().statusCode(200);
+ assertEquals("attempt-win",
+ mapper.readTree(getItem.body().asString()).path("Item").path("pk").path("S").asText());
+ }
+
+ @Test
+ @Order(2)
+ void exhaustedRetriesFallThroughToCatch() throws Exception {
+ var smArn = createStateMachine("sfn-retry-catch-test", """
+ {
+ "StartAt": "Flaky",
+ "States": {
+ "Flaky": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::dynamodb:putItem",
+ "Parameters": {
+ "TableName": "%s",
+ "Item": {"pk": {"S": "never"}}
+ },
+ "Retry": [{
+ "ErrorEquals": ["DynamoDB.ResourceNotFoundException"],
+ "IntervalSeconds": 1,
+ "BackoffRate": 1.0,
+ "MaxAttempts": 1
+ }],
+ "Catch": [{
+ "ErrorEquals": ["States.ALL"],
+ "ResultPath": "$.err",
+ "Next": "Handled"
+ }],
+ "End": true
+ },
+ "Handled": {"Type": "Pass", "End": true}
+ }
+ }
+ """.formatted(MISSING_TABLE));
+ var execArn = startExecution(smArn, "{}");
+
+ var describe = waitForTerminalState(execArn);
+ assertEquals("SUCCEEDED", describe.jsonPath().getString("status"));
+ var output = mapper.readTree(describe.jsonPath().getString("output"));
+ assertEquals("DynamoDB.ResourceNotFoundException", output.path("err").path("Error").asText());
+ }
+
+ @Test
+ @Order(3)
+ void unmatchedErrorFailsWithoutRetry() {
+ var smArn = createStateMachine("sfn-retry-unmatched-test", """
+ {
+ "StartAt": "Flaky",
+ "States": {
+ "Flaky": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::dynamodb:putItem",
+ "Parameters": {
+ "TableName": "%s",
+ "Item": {"pk": {"S": "never"}}
+ },
+ "Retry": [{
+ "ErrorEquals": ["SomeOther.Error"],
+ "IntervalSeconds": 1,
+ "MaxAttempts": 3
+ }],
+ "End": true
+ }
+ }
+ }
+ """.formatted(MISSING_TABLE));
+ var execArn = startExecution(smArn, "{}");
+
+ var describe = waitForTerminalState(execArn);
+ assertEquals("FAILED", describe.jsonPath().getString("status"));
+ assertEquals("DynamoDB.ResourceNotFoundException", describe.jsonPath().getString("error"));
+ }
+
+ private static String createStateMachine(String name, String definition) {
+ var resp = given()
+ .header("X-Amz-Target", "AWSStepFunctions.CreateStateMachine")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {
+ "name": "%s",
+ "roleArn": "%s",
+ "definition": %s
+ }
+ """.formatted(name, ROLE_ARN, quote(definition)))
+ .when().post("/");
+ resp.then().statusCode(200);
+ return resp.jsonPath().getString("stateMachineArn");
+ }
+
+ private static String startExecution(String smArn, String input) {
+ var resp = given()
+ .header("X-Amz-Target", "AWSStepFunctions.StartExecution")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"stateMachineArn": "%s", "input": %s}
+ """.formatted(smArn, quote(input)))
+ .when().post("/");
+ resp.then().statusCode(200);
+ return resp.jsonPath().getString("executionArn");
+ }
+
+ private static Response waitForTerminalState(String execArn) {
+ for (var i = 0; i < 100; i++) {
+ var resp = given()
+ .header("X-Amz-Target", "AWSStepFunctions.DescribeExecution")
+ .contentType(SFN_CONTENT_TYPE)
+ .body("""
+ {"executionArn": "%s"}
+ """.formatted(execArn))
+ .when().post("/");
+ var status = resp.jsonPath().getString("status");
+ if (!"RUNNING".equals(status)) {
+ return resp;
+ }
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ fail("Interrupted while waiting for execution " + execArn);
+ }
+ }
+ fail("Execution did not complete within timeout: " + execArn);
+ return null;
+ }
+
+ private static String quote(String raw) {
+ return "\"" + raw
+ .replace("\\", "\\\\")
+ .replace("\"", "\\\"")
+ .replace("\n", "\\n")
+ .replace("\r", "\\r")
+ .replace("\t", "\\t")
+ + "\"";
+ }
+}