Light, fluffy, and always free - The AWS Local Emulator alternative
feat(sfn): support Retry policies in Task, Parallel, and Map states (#2455)
TATSUNO “Taz” Yasuhiro56e74d64c6a5
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.mdindex 7f4842758f..ada52fd411 100644--- a/docs/services/step-functions.md+++ b/docs/services/step-functions.md@@ -51,6 +51,21 @@ Results remain in input order even when iterations finish out of order. If an itthe Map state fails promptly, cancels its active sibling iterations, and does not start queuediterations.+## Retry policies++`Task`, `Parallel`, and `Map` states honor their `Retry` field. `ErrorEquals` matching+follows AWS semantics, including the `States.ALL` and `States.TaskFailed` wildcards.+`States.Runtime` is never retried. AWS defaults apply when fields are omitted+(`MaxAttempts` 3, `IntervalSeconds` 1, `BackoffRate` 2.0), `MaxDelaySeconds` is honored,+and each retrier keeps its own attempt counter. `Retry` is evaluated before `Catch`, and+`$$.State.RetryCount` increments per attempt. Attempt counts, defaults, and backoff+timing were verified against real AWS Step Functions.++`JitterStrategy` supports `NONE` (the default) and `FULL`. `FULL` draws the delay+uniformly between zero and the computed delay, as on AWS. One deviation. The delay+between attempts is capped at 30 seconds, the same cap Floci applies to `Wait` states,+so emulated runs stay fast.+## Configuration| Variable | Default | Description |diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.javaindex 3dc55e37d2..5ab24a1d21 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java@@ -75,6 +75,7 @@ import java.util.concurrent.ExecutionException;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;+import java.util.concurrent.ThreadLocalRandom;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicLong;import java.util.function.BiConsumer;@@ -311,9 +312,9 @@ public class AslExecutor {// Update per-state context fieldsupdateStateContext(execContext, currentStateName);- boolean jsonata = isJsonata(stateDef, topLevelQueryLanguage);+ var jsonata = isJsonata(stateDef, topLevelQueryLanguage);try {- StateResult stateResult = executeState(currentStateName, type, stateDef, currentInput,+ var stateResult = executeStateWithRetry(currentStateName, type, stateDef, currentInput,history, eventId, sm, jsonata, topLevelQueryLanguage, execContext, variables);addEvent(history, eventId, stateExitedEventType(type), eventId.get() - 1,Map.of("name", currentStateName, "output", stateResult.output().toString()));@@ -373,6 +374,84 @@ public class AslExecutor {}}+ /**+ * Executes a state, honoring its {@code Retry} policy: a {@code FailStateException} matched by+ * a retrier re-runs the state after the retrier's backoff until its {@code MaxAttempts} are+ * used up. Errors that no retrier matches (or that exhaust their retrier) propagate to the+ * caller's Catch handling, preserving Retry-before-Catch order.+ */+ private StateResult executeStateWithRetry(String name, String type, JsonNode stateDef, JsonNode input,+ List<HistoryEvent> history, AtomicLong eventId, StateMachine sm,+ boolean jsonata, String topLevelQueryLanguage, JsonNode context,+ ObjectNode variables) throws Exception {+ var retriers = stateDef.path("Retry");+ var attemptsPerRetrier = new HashMap<Integer, Integer>();+ var attempt = 0;+ while (true) {+ try {+ return executeState(name, type, stateDef, input, history, eventId, sm, jsonata,+ topLevelQueryLanguage, context, variables);+ } catch (FailStateException e) {+ var retrierIndex = findMatchingRetrier(retriers, e);+ if (retrierIndex < 0) {+ throw e;+ }+ var retrier = retriers.get(retrierIndex);+ var attemptsUsed = attemptsPerRetrier.merge(retrierIndex, 1, Integer::sum);+ if (attemptsUsed > retrier.path("MaxAttempts").asInt(3)) {+ throw e;+ }+ sleepBeforeRetry(retrier, attemptsUsed);+ attempt++;+ updateRetryCount(context, attempt);+ }+ }+ }++ private int findMatchingRetrier(JsonNode retriers, FailStateException failure) {+ if (!retriers.isArray()) {+ return -1;+ }+ var error = failure.error != null ? failure.error : "States.Runtime";+ for (var i = 0; i < retriers.size(); i++) {+ if (catchMatches(retriers.get(i), error)) {+ return i;+ }+ }+ return -1;+ }++ private void sleepBeforeRetry(JsonNode retrier, int attemptsUsed) throws InterruptedException {+ var delaySeconds = retryDelaySeconds(retrier, attemptsUsed, ThreadLocalRandom.current().nextDouble());+ if (delaySeconds > 0) {+ Thread.sleep((long) (delaySeconds * 1000));+ }+ }++ /**+ * Computes the delay before a retry attempt. {@code random} is a value in [0, 1) used when+ * the retrier declares {@code JitterStrategy: FULL}, which draws the delay uniformly between+ * zero and the computed delay. Jitter applies after the caps, matching AWS.+ */+ static double retryDelaySeconds(JsonNode retrier, int attemptsUsed, double random) {+ var interval = retrier.path("IntervalSeconds").asDouble(1.0);+ var backoffRate = retrier.path("BackoffRate").asDouble(2.0);+ var delaySeconds = interval * Math.pow(backoffRate, attemptsUsed - 1.0);+ var maxDelay = retrier.path("MaxDelaySeconds").asDouble(MAX_WAIT_SECONDS);+ // Like the Wait state, cap the delay at MAX_WAIT_SECONDS to keep emulated runs fast.+ delaySeconds = Math.min(delaySeconds, Math.min(maxDelay, MAX_WAIT_SECONDS));+ if ("FULL".equals(retrier.path("JitterStrategy").asText(null))) {+ delaySeconds *= random;+ }+ return delaySeconds;+ }++ private void updateRetryCount(JsonNode context, int retryCount) {+ if (context.get("State") instanceof ObjectNode state) {+ state.put("RetryCount", retryCount);+ }+ }+private StateResult executeState(String name, String type, JsonNode stateDef, JsonNode input,List<HistoryEvent> history, AtomicLong eventId, StateMachine sm,boolean jsonata, String topLevelQueryLanguage, JsonNode context,@@ -540,7 +619,10 @@ public class AslExecutor {String region = extractRegionFromArn(sm.getStateMachineArn());LambdaFunction fn = functionStore.get(region, functionName).orElse(null);if (fn == null) {- throw new RuntimeException("Lambda function not found: " + functionName);+ // A missing function is a task failure on AWS, so it must stay reachable for+ // Retry and Catch instead of surfacing as States.Runtime.+ throw new FailStateException("Lambda.ResourceNotFoundException",+ "Lambda function not found: " + functionName);}String payloadStr = objectMapper.writeValueAsString(lambdaPayload);@@ -1443,13 +1525,16 @@ public class AslExecutor {// Each branch gets an isolated copy of the current variables: assignments inside a// branch are scoped to that branch and do not leak back to the parent after the state.ObjectNode branchVariables = variables.deepCopy();+ // Each branch also gets its own copy of the context object so State.RetryCount and+ // Task.Token writes cannot race across concurrent branches.+ var branchContext = ((ObjectNode) context).deepCopy();// Run each branch on its own worker thread under the execution's account: the request// scope is thread-bound, so without this a branch's Task integrations would resolve to// the default account rather than the execution's.futures.add(executor.submit(() -> callUnderExecutionAccount(sm,- () -> executeBranch(startAt, branchStates, capturedInput, sm, topLevelQueryLanguage, context,- branchVariables))));+ () -> executeBranch(startAt, branchStates, capturedInput, sm, topLevelQueryLanguage,+ branchContext, branchVariables))));}int timeoutSeconds = stateDef.path("TimeoutSeconds").asInt(0);@@ -1476,6 +1561,16 @@ public class AslExecutor {futures.forEach(future -> future.cancel(true));Thread.currentThread().interrupt();throw e;+ } catch (ExecutionException e) {+ futures.forEach(future -> future.cancel(true));+ // Unwrap so a branch's FailStateException reaches the Parallel state's own+ // Retry and Catch handling instead of surfacing as States.Runtime. An Error+ // cause stays wrapped so the execution-level Exception handlers still+ // publish a terminal FAILED update instead of leaving the execution RUNNING.+ if (e.getCause() instanceof Exception exception) {+ throw exception;+ }+ throw e;} catch (Exception | Error e) {futures.forEach(future -> future.cancel(true));throw e;@@ -1997,9 +2092,10 @@ public class AslExecutor {}String type = stateDef.path("Type").asText();boolean stateJsonata = isJsonata(stateDef, topLevelQueryLanguage);+ updateStateContext(context, currentState);StateResult result;try {- result = executeState(currentState, type, stateDef, currentInput, ignored, eventId, sm,+ result = executeStateWithRetry(currentState, type, stateDef, currentInput, ignored, eventId, sm,stateJsonata, topLevelQueryLanguage, context, variables);} catch (FailStateException e) {StateResult caught = handleCatch(stateDef, currentInput, e, stateJsonata, context, variables);diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorRetryTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorRetryTest.javanew file mode 100644index 0000000000..1ae1ad1818--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorRetryTest.java@@ -0,0 +1,432 @@+package io.github.hectorvent.floci.services.stepfunctions;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbJsonHandler;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;+import io.github.hectorvent.floci.services.lambda.LambdaExecutorService;+import io.github.hectorvent.floci.services.lambda.LambdaFunctionStore;+import io.github.hectorvent.floci.services.lambda.model.InvocationType;+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;+import io.github.hectorvent.floci.services.stepfunctions.model.Execution;+import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;+import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;+import io.quarkus.test.junit.QuarkusTest;+import io.vertx.mutiny.core.Vertx;+import jakarta.enterprise.inject.Instance;+import jakarta.inject.Inject;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Optional;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++/**+ * Retry policy semantics of the ASL executor. Attempt counts, defaults, and backoff+ * behavior were verified against real AWS Step Functions in us-east-1.+ */+@QuarkusTest+class AslExecutorRetryTest {++ private static final String REGION = "us-east-2";+ private static final String ACCOUNT = "000000000000";+ private static final String FLAKY_FUNCTION_NAME = "flaky-lambda";+ private static final String FLAKY_FUNCTION_ARN =+ "arn:aws:lambda:%s:%s:function:%s".formatted(REGION, ACCOUNT, FLAKY_FUNCTION_NAME);++ private final ObjectMapper objectMapper = new ObjectMapper();+ private LambdaExecutorService lambdaExecutor;+ private LambdaFunctionStore functionStore;+ private LambdaFunction flakyFunction;+ private AslExecutor executor;++ @Inject+ Vertx vertx;++ @BeforeEach+ void setUp() {+ lambdaExecutor = mock(LambdaExecutorService.class);+ functionStore = mock(LambdaFunctionStore.class);+ flakyFunction = new LambdaFunction();+ flakyFunction.setFunctionName(FLAKY_FUNCTION_NAME);+ flakyFunction.setFunctionArn(FLAKY_FUNCTION_ARN);+ when(functionStore.get(REGION, FLAKY_FUNCTION_NAME)).thenReturn(Optional.of(flakyFunction));++ executor = new AslExecutor(+ lambdaExecutor,+ functionStore,+ mock(DynamoDbService.class),+ mock(DynamoDbJsonHandler.class),+ mock(SqsJsonHandler.class),+ mock(io.github.hectorvent.floci.services.cloudformation.CloudFormationQueryHandler.class),+ mock(io.github.hectorvent.floci.services.ec2.Ec2Service.class),+ mock(S3Service.class),+ mock(io.github.hectorvent.floci.services.ecs.EcsService.class),+ mock(io.github.hectorvent.floci.services.ecs.EcsJsonHandler.class),+ objectMapper,+ new JsonataEvaluator(objectMapper),+ mock(Instance.class), mock(EmulatorConfig.class), vertx);+ }++ @Test+ void retriesUntilTheTaskSucceeds() throws Exception {+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(new InvokeResult(200, "Handled",+ "{\"errorType\":\"Boom\"}".getBytes(StandardCharsets.UTF_8), null, "req-1"))+ .thenReturn(new InvokeResult(200, null,+ "{\"ok\":true}".getBytes(StandardCharsets.UTF_8), null, "req-2"));++ var execution = run("""+ {+ "StartAt": "Flaky",+ "States": {+ "Flaky": {+ "Type": "Task",+ "Resource": "%s",+ "End": true,+ "Retry": [{+ "ErrorEquals": ["Lambda.AWSLambdaException"],+ "IntervalSeconds": 1,+ "BackoffRate": 1.0,+ "MaxAttempts": 2+ }]+ }+ }+ }+ """.formatted(FLAKY_FUNCTION_ARN));++ assertEquals("SUCCEEDED", execution.getStatus());+ assertTrue(objectMapper.readTree(execution.getOutput()).path("ok").asBoolean());+ verify(lambdaExecutor, times(2))+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));+ }++ @Test+ void exhaustedRetriesFailWithOriginalError() {+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(new InvokeResult(200, "Handled",+ "{\"errorType\":\"Boom\"}".getBytes(StandardCharsets.UTF_8), null, "req-1"));++ var execution = run("""+ {+ "StartAt": "Flaky",+ "States": {+ "Flaky": {+ "Type": "Task",+ "Resource": "%s",+ "End": true,+ "Retry": [{+ "ErrorEquals": ["Lambda.AWSLambdaException"],+ "IntervalSeconds": 1,+ "BackoffRate": 1.0,+ "MaxAttempts": 1+ }]+ }+ }+ }+ """.formatted(FLAKY_FUNCTION_ARN));++ assertEquals("FAILED", execution.getStatus());+ assertEquals("Lambda.AWSLambdaException", execution.getError());+ verify(lambdaExecutor, times(2))+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));+ }++ @Test+ void unmatchedErrorIsNotRetried() {+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(new InvokeResult(200, "Handled",+ "{\"errorType\":\"Boom\"}".getBytes(StandardCharsets.UTF_8), null, "req-1"));++ var execution = run("""+ {+ "StartAt": "Flaky",+ "States": {+ "Flaky": {+ "Type": "Task",+ "Resource": "%s",+ "End": true,+ "Retry": [{+ "ErrorEquals": ["SomeOther.Error"],+ "IntervalSeconds": 1,+ "MaxAttempts": 3+ }]+ }+ }+ }+ """.formatted(FLAKY_FUNCTION_ARN));++ assertEquals("FAILED", execution.getStatus());+ assertEquals("Lambda.AWSLambdaException", execution.getError());+ verify(lambdaExecutor, times(1))+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));+ }++ @Test+ void exhaustedRetrierFallsThroughToCatch() throws Exception {+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(new InvokeResult(200, "Handled",+ "{\"errorType\":\"Boom\"}".getBytes(StandardCharsets.UTF_8), null, "req-1"));++ var execution = run("""+ {+ "StartAt": "Flaky",+ "States": {+ "Flaky": {+ "Type": "Task",+ "Resource": "%s",+ "End": true,+ "Retry": [{+ "ErrorEquals": ["States.ALL"],+ "IntervalSeconds": 1,+ "BackoffRate": 1.0,+ "MaxAttempts": 1+ }],+ "Catch": [{+ "ErrorEquals": ["States.ALL"],+ "Next": "HandleError"+ }]+ },+ "HandleError": {+ "Type": "Pass",+ "End": true+ }+ }+ }+ """.formatted(FLAKY_FUNCTION_ARN));++ assertEquals("SUCCEEDED", execution.getStatus());+ assertEquals("Lambda.AWSLambdaException",+ objectMapper.readTree(execution.getOutput()).path("Error").asText());+ verify(lambdaExecutor, times(2))+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));+ }++ @Test+ void parallelStateRetriesFailingBranch() throws Exception {+ failOnceThenSucceed();++ var execution = run("""+ {+ "StartAt": "Par",+ "States": {+ "Par": {+ "Type": "Parallel",+ "Branches": [{+ "StartAt": "Inner",+ "States": {"Inner": {"Type": "Task", "Resource": "%s", "End": true}}+ }],+ "Retry": [{+ "ErrorEquals": ["Lambda.AWSLambdaException"],+ "IntervalSeconds": 1,+ "BackoffRate": 1.0,+ "MaxAttempts": 1+ }],+ "End": true+ }+ }+ }+ """.formatted(FLAKY_FUNCTION_ARN));++ assertEquals("SUCCEEDED", execution.getStatus());+ assertTrue(objectMapper.readTree(execution.getOutput()).path(0).path("ok").asBoolean());+ verify(lambdaExecutor, times(2))+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));+ }++ @Test+ void parallelBranchFailureReachesCatch() throws Exception {+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(new InvokeResult(200, "Handled",+ "{\"errorType\":\"Boom\"}".getBytes(StandardCharsets.UTF_8), null, "req-1"));++ var execution = run("""+ {+ "StartAt": "Par",+ "States": {+ "Par": {+ "Type": "Parallel",+ "Branches": [{+ "StartAt": "Inner",+ "States": {"Inner": {"Type": "Task", "Resource": "%s", "End": true}}+ }],+ "Catch": [{+ "ErrorEquals": ["States.ALL"],+ "Next": "HandleError"+ }],+ "End": true+ },+ "HandleError": {"Type": "Pass", "End": true}+ }+ }+ """.formatted(FLAKY_FUNCTION_ARN));++ assertEquals("SUCCEEDED", execution.getStatus());+ assertEquals("Lambda.AWSLambdaException",+ objectMapper.readTree(execution.getOutput()).path("Error").asText());+ verify(lambdaExecutor, times(1))+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));+ }++ @Test+ void taskInsideParallelBranchRetriesIndependently() throws Exception {+ failOnceThenSucceed();++ var execution = run("""+ {+ "StartAt": "Par",+ "States": {+ "Par": {+ "Type": "Parallel",+ "Branches": [{+ "StartAt": "Inner",+ "States": {+ "Inner": {+ "Type": "Task",+ "Resource": "%s",+ "End": true,+ "Retry": [{+ "ErrorEquals": ["Lambda.AWSLambdaException"],+ "IntervalSeconds": 1,+ "BackoffRate": 1.0,+ "MaxAttempts": 1+ }]+ }+ }+ }],+ "End": true+ }+ }+ }+ """.formatted(FLAKY_FUNCTION_ARN));++ assertEquals("SUCCEEDED", execution.getStatus());+ assertTrue(objectMapper.readTree(execution.getOutput()).path(0).path("ok").asBoolean());+ verify(lambdaExecutor, times(2))+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));+ }++ @Test+ void mapStateRetriesFailingIteration() throws Exception {+ failOnceThenSucceed();++ var execution = run("""+ {+ "StartAt": "Loop",+ "States": {+ "Loop": {+ "Type": "Map",+ "ItemsPath": "$.items",+ "ItemProcessor": {+ "ProcessorConfig": {"Mode": "INLINE"},+ "StartAt": "Inner",+ "States": {"Inner": {"Type": "Task", "Resource": "%s", "End": true}}+ },+ "Retry": [{+ "ErrorEquals": ["Lambda.AWSLambdaException"],+ "IntervalSeconds": 1,+ "BackoffRate": 1.0,+ "MaxAttempts": 1+ }],+ "End": true+ }+ }+ }+ """.formatted(FLAKY_FUNCTION_ARN), "{\"items\": [1]}");++ assertEquals("SUCCEEDED", execution.getStatus());+ assertTrue(objectMapper.readTree(execution.getOutput()).path(0).path("ok").asBoolean());+ verify(lambdaExecutor, times(2))+ .invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse));+ }++ @Test+ void branchErrorStillFailsTheExecution() {+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenThrow(new AssertionError("boom"));++ var execution = run("""+ {+ "StartAt": "Par",+ "States": {+ "Par": {+ "Type": "Parallel",+ "Branches": [{+ "StartAt": "Inner",+ "States": {"Inner": {"Type": "Task", "Resource": "%s", "End": true}}+ }],+ "End": true+ }+ }+ }+ """.formatted(FLAKY_FUNCTION_ARN));++ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.Runtime", execution.getError());+ }++ @Test+ void retryDelayFollowsBackoffAndCaps() throws Exception {+ assertEquals(10.0, delay("{\"IntervalSeconds\": 10, \"BackoffRate\": 2.0}", 1, 0.5));+ assertEquals(20.0, delay("{\"IntervalSeconds\": 10, \"BackoffRate\": 2.0}", 2, 0.5));+ assertEquals(15.0, delay("{\"IntervalSeconds\": 10, \"BackoffRate\": 2.0, \"MaxDelaySeconds\": 15}", 2, 0.5));+ assertEquals(30.0, delay("{\"IntervalSeconds\": 20, \"BackoffRate\": 2.0}", 2, 0.5));+ }++ @Test+ void fullJitterScalesTheDelayByTheRandomFactor() throws Exception {+ var retrier = "{\"IntervalSeconds\": 10, \"BackoffRate\": 1.0, \"JitterStrategy\": \"FULL\"}";+ assertEquals(0.0, delay(retrier, 1, 0.0));+ assertEquals(5.0, delay(retrier, 1, 0.5));+ assertEquals(10.0, delay("{\"IntervalSeconds\": 10, \"BackoffRate\": 1.0, \"JitterStrategy\": \"NONE\"}", 1, 0.5));+ }++ private double delay(String retrierJson, int attemptsUsed, double random) throws Exception {+ return AslExecutor.retryDelaySeconds(objectMapper.readTree(retrierJson), attemptsUsed, random);+ }++ private void failOnceThenSucceed() {+ when(lambdaExecutor.invoke(eq(flakyFunction), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(new InvokeResult(200, "Handled",+ "{\"errorType\":\"Boom\"}".getBytes(StandardCharsets.UTF_8), null, "req-1"))+ .thenReturn(new InvokeResult(200, null,+ "{\"ok\":true}".getBytes(StandardCharsets.UTF_8), null, "req-2"));+ }++ private Execution run(String definition) {+ return run(definition, "{}");+ }++ private Execution run(String definition, String input) {+ var stateMachine = new StateMachine();+ stateMachine.setName("retry-test");+ stateMachine.setStateMachineArn("arn:aws:states:%s:%s:stateMachine:retry-test".formatted(REGION, ACCOUNT));+ stateMachine.setRoleArn("arn:aws:iam::%s:role/test-role".formatted(ACCOUNT));+ stateMachine.setDefinition(definition);++ var execution = new Execution();+ execution.setName("retry-test-execution");+ execution.setExecutionArn(+ "arn:aws:states:%s:%s:execution:retry-test:retry-test-execution".formatted(REGION, ACCOUNT));+ execution.setStateMachineArn(stateMachine.getStateMachineArn());+ execution.setInput(input);++ var history = new ArrayList<HistoryEvent>();+ executor.executeSync(stateMachine, execution, history, (updated, events) -> {+ });+ return execution;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsRetryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsRetryIntegrationTest.javanew file mode 100644index 0000000000..ea51597443--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsRetryIntegrationTest.java@@ -0,0 +1,234 @@+package io.github.hectorvent.floci.services.stepfunctions;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assertions.fail;++/**+ * End-to-end Retry policy tests through the JSON 1.0 wire protocol.+ *+ * The fail-then-succeed case works without mocks by resolving the DynamoDB table name+ * from {@code $$.State.RetryCount}. Attempt 0 targets a table that does not exist and+ * fails. The retry re-resolves Parameters, targets the table that does exist, and+ * succeeds. Real AWS re-resolves Parameters per attempt the same way (verified in+ * us-east-1).+ */+@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class StepFunctionsRetryIntegrationTest {++ private static final String SFN_CONTENT_TYPE = "application/x-amz-json-1.0";+ private static final String DDB_CONTENT_TYPE = "application/x-amz-json-1.0";+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/test-role";+ private static final String RETRY_TABLE = "sfn-retry-1";+ private static final String MISSING_TABLE = "sfn-retry-none";+ private static final ObjectMapper mapper = new ObjectMapper();++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ @Order(0)+ void setup_createTargetTableForSecondAttempt() {+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.CreateTable")+ .contentType(DDB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "%s",+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],+ "BillingMode": "PAY_PER_REQUEST"+ }+ """.formatted(RETRY_TABLE))+ .when().post("/")+ .then().statusCode(200);+ }++ @Test+ @Order(1)+ void taskIsRetriedAndSucceedsOnSecondAttempt() throws Exception {+ var smArn = createStateMachine("sfn-retry-succeed-test", """+ {+ "StartAt": "Flaky",+ "States": {+ "Flaky": {+ "Type": "Task",+ "Resource": "arn:aws:states:::dynamodb:putItem",+ "Parameters": {+ "TableName.$": "States.Format('sfn-retry-{}', $$.State.RetryCount)",+ "Item": {"pk": {"S": "attempt-win"}}+ },+ "Retry": [{+ "ErrorEquals": ["DynamoDB.ResourceNotFoundException"],+ "IntervalSeconds": 1,+ "BackoffRate": 1.0,+ "MaxAttempts": 2+ }],+ "End": true+ }+ }+ }+ """);+ var execArn = startExecution(smArn, "{}");++ var describe = waitForTerminalState(execArn);+ assertEquals("SUCCEEDED", describe.jsonPath().getString("status"));++ var getItem = given()+ .header("X-Amz-Target", "DynamoDB_20120810.GetItem")+ .contentType(DDB_CONTENT_TYPE)+ .body("""+ {"TableName": "%s", "Key": {"pk": {"S": "attempt-win"}}}+ """.formatted(RETRY_TABLE))+ .when().post("/");+ getItem.then().statusCode(200);+ assertEquals("attempt-win",+ mapper.readTree(getItem.body().asString()).path("Item").path("pk").path("S").asText());+ }++ @Test+ @Order(2)+ void exhaustedRetriesFallThroughToCatch() throws Exception {+ var smArn = createStateMachine("sfn-retry-catch-test", """+ {+ "StartAt": "Flaky",+ "States": {+ "Flaky": {+ "Type": "Task",+ "Resource": "arn:aws:states:::dynamodb:putItem",+ "Parameters": {+ "TableName": "%s",+ "Item": {"pk": {"S": "never"}}+ },+ "Retry": [{+ "ErrorEquals": ["DynamoDB.ResourceNotFoundException"],+ "IntervalSeconds": 1,+ "BackoffRate": 1.0,+ "MaxAttempts": 1+ }],+ "Catch": [{+ "ErrorEquals": ["States.ALL"],+ "ResultPath": "$.err",+ "Next": "Handled"+ }],+ "End": true+ },+ "Handled": {"Type": "Pass", "End": true}+ }+ }+ """.formatted(MISSING_TABLE));+ var execArn = startExecution(smArn, "{}");++ var describe = waitForTerminalState(execArn);+ assertEquals("SUCCEEDED", describe.jsonPath().getString("status"));+ var output = mapper.readTree(describe.jsonPath().getString("output"));+ assertEquals("DynamoDB.ResourceNotFoundException", output.path("err").path("Error").asText());+ }++ @Test+ @Order(3)+ void unmatchedErrorFailsWithoutRetry() {+ var smArn = createStateMachine("sfn-retry-unmatched-test", """+ {+ "StartAt": "Flaky",+ "States": {+ "Flaky": {+ "Type": "Task",+ "Resource": "arn:aws:states:::dynamodb:putItem",+ "Parameters": {+ "TableName": "%s",+ "Item": {"pk": {"S": "never"}}+ },+ "Retry": [{+ "ErrorEquals": ["SomeOther.Error"],+ "IntervalSeconds": 1,+ "MaxAttempts": 3+ }],+ "End": true+ }+ }+ }+ """.formatted(MISSING_TABLE));+ var execArn = startExecution(smArn, "{}");++ var describe = waitForTerminalState(execArn);+ assertEquals("FAILED", describe.jsonPath().getString("status"));+ assertEquals("DynamoDB.ResourceNotFoundException", describe.jsonPath().getString("error"));+ }++ private static String createStateMachine(String name, String definition) {+ var resp = given()+ .header("X-Amz-Target", "AWSStepFunctions.CreateStateMachine")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {+ "name": "%s",+ "roleArn": "%s",+ "definition": %s+ }+ """.formatted(name, ROLE_ARN, quote(definition)))+ .when().post("/");+ resp.then().statusCode(200);+ return resp.jsonPath().getString("stateMachineArn");+ }++ private static String startExecution(String smArn, String input) {+ var resp = given()+ .header("X-Amz-Target", "AWSStepFunctions.StartExecution")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"stateMachineArn": "%s", "input": %s}+ """.formatted(smArn, quote(input)))+ .when().post("/");+ resp.then().statusCode(200);+ return resp.jsonPath().getString("executionArn");+ }++ private static Response waitForTerminalState(String execArn) {+ for (var i = 0; i < 100; i++) {+ var resp = given()+ .header("X-Amz-Target", "AWSStepFunctions.DescribeExecution")+ .contentType(SFN_CONTENT_TYPE)+ .body("""+ {"executionArn": "%s"}+ """.formatted(execArn))+ .when().post("/");+ var status = resp.jsonPath().getString("status");+ if (!"RUNNING".equals(status)) {+ return resp;+ }+ try {+ Thread.sleep(100);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ fail("Interrupted while waiting for execution " + execArn);+ }+ }+ fail("Execution did not complete within timeout: " + execArn);+ return null;+ }++ private static String quote(String raw) {+ return "\"" + raw+ .replace("\\", "\\\\")+ .replace("\"", "\\\"")+ .replace("\n", "\\n")+ .replace("\r", "\\r")+ .replace("\t", "\\t")+ + "\"";+ }+}