Light, fluffy, and always free - The AWS Local Emulator alternative
fix(stepfunctions): persist execution history across restarts (#3291)
Niels Kootstra35159c5aaabe
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.mdindex fefadadd35..77a0abe100 100644--- a/docs/services/step-functions.md+++ b/docs/services/step-functions.md@@ -510,11 +510,12 @@ of every account are swept, each written back under its own account. Executionsreached a terminal status are left untouched, and so is the status and `stopDate` of one this sweepaborted on an earlier boot.-Execution histories are held in memory, not in storage. The events recorded before the restart are-gone, so the execution cannot be resumed, and `GetExecutionHistory` reports a single-`ExecutionAborted` event, with an empty `executionAbortedEventDetails`, only for the boot that-aborted it: after a further restart the execution is already terminal, no event is written, and the-history is empty while `DescribeExecution` still reports the status and `stopDate`.+Execution history is stored with the execution. While an execution is running, the current history+is checkpointed every 100 events and when the execution reaches a terminal state. A graceful+shutdown flushes the current execution state before the emulator stops. On restart, persisted+history is retained, and a previously running execution is marked `ABORTED` with one+`ExecutionAborted` event appended. After a further restart, the execution is already terminal, so+no additional event is written.## Configurationdiff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.javaindex 1fc25d05df..c859702370 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java@@ -37,11 +37,13 @@ import java.util.concurrent.LinkedBlockingQueue;import java.util.concurrent.TimeUnit;import java.util.regex.Matcher;import java.util.regex.Pattern;+import java.util.function.IntConsumer;@ApplicationScopedpublic class StepFunctionsService implements Resettable, ResourceProvider {private static final Logger LOG = Logger.getLogger(StepFunctionsService.class);+ private static final int HISTORY_PERSIST_CHECKPOINT = 100;private final StorageBackend<String, StateMachine> stateMachineStore;// Account-aware: the startup sweep has no request context and must reach every account.@@ -577,9 +579,11 @@ public class StepFunctionsService implements Resettable, ResourceProvider {exec.setName(execName);exec.setInput(input);exec.setStatus("RUNNING");- executionStore.put(arn, exec);-- history = new ExecutionHistory();+ history = new ExecutionHistory(eventCount -> {+ if (eventCount % HISTORY_PERSIST_CHECKPOINT == 0) {+ executionStore.put(arn, exec);+ }+ });var startEvent = new HistoryEvent();startEvent.setId(1L);startEvent.setPreviousEventId(0L);@@ -587,7 +591,9 @@ public class StepFunctionsService implements Resettable, ResourceProvider {startEvent.setDetails(Map.of("input", input != null ? input : "{}","roleArn", sm.getRoleArn() != null ? sm.getRoleArn() : "","inputDetails", Map.of("truncated", false)));+ exec.setHistory(history);history.add(startEvent);+ executionStore.put(arn, exec);historyCache.put(arn, history);}@@ -794,18 +800,23 @@ public class StepFunctionsService implements Resettable, ResourceProvider {if (cause != null) {details.put("cause", cause);}- // An execution can outlive its history: the executions are stored, the histories are held in- // memory only, so a restart in persistent mode brings a RUNNING execution back with nothing- // behind it. The abort still gets recorded, against a history that starts here.- historyCache.computeIfAbsent(arn, key -> new ExecutionHistory())- .sealWith("ExecutionAborted", details);+ // A restart can leave a RUNNING execution without its worker and token future. Preserve all+ // persisted events, then append the terminal event that explains the deterministic recovery.+ ExecutionHistory history = historyCache.computeIfAbsent(arn,+ key -> new ExecutionHistory(exec.getHistory(), () -> { }, false));+ exec.setHistory(history);+ history.sealWith("ExecutionAborted", details);return true;}public List<HistoryEvent> getExecutionHistory(String arn) {- describeExecution(arn);- ExecutionHistory history = historyCache.get(arn);- return history != null ? history : Collections.emptyList();+ Execution exec = describeExecution(arn);+ return historyCache.computeIfAbsent(arn,+ key -> new ExecutionHistory(exec.getHistory(), () -> { }, isTerminal(exec.getStatus())));+ }++ private static boolean isTerminal(String status) {+ return !"RUNNING".equals(status);}/**@@ -821,14 +832,37 @@ public class StepFunctionsService implements Resettable, ResourceProvider {private static final long serialVersionUID = 1L;+ private final IntConsumer onAppend;private boolean sealed;+ ExecutionHistory() {+ this(eventCount -> { });+ }++ ExecutionHistory(Runnable onAppend) {+ this(eventCount -> onAppend.run());+ }++ ExecutionHistory(IntConsumer onAppend) {+ this.onAppend = onAppend;+ }++ ExecutionHistory(List<HistoryEvent> events, Runnable onAppend, boolean sealed) {+ super(events != null ? events : List.of());+ this.onAppend = eventCount -> onAppend.run();+ this.sealed = sealed;+ }+@Overridepublic synchronized boolean add(HistoryEvent event) {if (sealed) {return false;}- return super.add(event);+ boolean added = super.add(event);+ if (added) {+ onAppend.accept(size());+ }+ return added;}/** Appends the terminal event, numbered from the end of the history, and takes no more. */diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.javaindex 5ae0b6c722..c54bad1714 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java@@ -3,6 +3,9 @@ package io.github.hectorvent.floci.services.stepfunctions.model;import com.fasterxml.jackson.annotation.JsonIgnoreProperties;import io.quarkus.runtime.annotations.RegisterForReflection;+import java.util.ArrayList;+import java.util.List;+@RegisterForReflection@JsonIgnoreProperties(ignoreUnknown = true)public class Execution {@@ -16,6 +19,7 @@ public class Execution {private Double stopDate;private String error;private String cause;+ private List<HistoryEvent> history = new ArrayList<>();public Execution() {this.startDate = System.currentTimeMillis() / 1000.0;@@ -50,4 +54,9 @@ public class Execution {public String getCause() { return cause; }public void setCause(String cause) { this.cause = cause; }++ public List<HistoryEvent> getHistory() { return history; }+ public void setHistory(List<HistoryEvent> history) {+ this.history = history != null ? history : new ArrayList<>();+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsExecutionHistoryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsExecutionHistoryIntegrationTest.javaindex 53c4607a40..fd0fb48b30 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsExecutionHistoryIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsExecutionHistoryIntegrationTest.java@@ -62,6 +62,16 @@ class StepFunctionsExecutionHistoryIntegrationTest {String executionArn = startExecution(stateMachineArn);waitForExecution(executionArn);+ given()+ .header("X-Amz-Target", "AWSStepFunctions.DescribeExecution")+ .contentType(SFN_CONTENT_TYPE)+ .body(String.format("{\"executionArn\":\"%s\"}", executionArn))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("history", nullValue());+given().header("X-Amz-Target", "AWSStepFunctions.GetExecutionHistory").contentType(SFN_CONTENT_TYPE)diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsServicePersistenceTest.javaindex 8cbf72e92a..235ad232ff 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsServicePersistenceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsServicePersistenceTest.java@@ -4,11 +4,18 @@ import com.fasterxml.jackson.core.type.TypeReference;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.PersistentStorage;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.stepfunctions.model.Activity;+import io.github.hectorvent.floci.services.stepfunctions.model.ActivityTask;import io.github.hectorvent.floci.services.stepfunctions.model.Execution;import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;+import io.github.hectorvent.floci.services.stepfunctions.model.MockedTestCase;+import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;+import jakarta.enterprise.inject.Instance;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;import org.mockito.Mockito;import java.text.MessageFormat;@@ -17,20 +24,33 @@ import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Optional;+import java.nio.file.Path;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;+import java.util.concurrent.atomic.AtomicInteger;import java.util.logging.Handler;import java.util.logging.Level;import java.util.logging.LogRecord;import java.util.logging.Logger;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.Mockito.verifyNoInteractions;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyList;+import static org.mockito.ArgumentMatchers.isNull;+import static org.mockito.Mockito.doAnswer;/** Verifies executions abandoned by a restart reach a terminal status when the emulator comes back. */class StepFunctionsServicePersistenceTest {+ @TempDir+ Path tempDir;+private static final String DEFAULT_ACCOUNT = "000000000000";private static final String OTHER_ACCOUNT = "222222222222";private static final String EXECUTION_ARN =@@ -46,6 +66,121 @@ class StepFunctionsServicePersistenceTest {private final SfnMockLoader mockLoader = Mockito.mock(SfnMockLoader.class);private final RegionResolver regionResolver = Mockito.mock(RegionResolver.class);+ @Test+ void historyCheckpointCallbackRunsAtBoundedIntervals() {+ AtomicInteger lastCheckpoint = new AtomicInteger();+ StepFunctionsService.ExecutionHistory history =+ new StepFunctionsService.ExecutionHistory(eventCount -> {+ if (eventCount % 100 == 0) {+ lastCheckpoint.set(eventCount);+ }+ });++ for (int eventId = 1; eventId <= 99; eventId++) {+ HistoryEvent event = new HistoryEvent();+ event.setId((long) eventId);+ history.add(event);+ }+ assertEquals(0, lastCheckpoint.get());++ HistoryEvent checkpointEvent = new HistoryEvent();+ checkpointEvent.setId(100L);+ history.add(checkpointEvent);++ assertEquals(100, lastCheckpoint.get());+ }++ @Test+ void completedExecutionRetainsExactHistoryAfterReload() {+ PersistentTestStorageFactory storage = new PersistentTestStorageFactory(tempDir);+ StateMachine stateMachine = stateMachine();+ storage.create("stepfunctions", "sfn-state-machines.json",+ new TypeReference<Map<String, StateMachine>>() {})+ .putForAccount(DEFAULT_ACCOUNT, STATE_MACHINE_ARN, stateMachine);+ Mockito.when(regionResolver.buildArn("states", "us-east-1",+ "execution:TestStateMachine:completed"))+ .thenReturn(EXECUTION_ARN);+ doAnswer(invocation -> {+ Execution execution = invocation.getArgument(1);+ List<HistoryEvent> history = invocation.getArgument(2);+ HistoryEvent succeeded = new HistoryEvent();+ succeeded.setId(2L);+ succeeded.setPreviousEventId(1L);+ succeeded.setType("ExecutionSucceeded");+ succeeded.setDetails(Map.of("output", "{\"result\":true}"));+ history.add(succeeded);+ execution.setStatus("SUCCEEDED");+ execution.setOutput("{\"result\":true}");+ invocation.<java.util.function.BiConsumer<Execution, List<HistoryEvent>>>getArgument(4)+ .accept(execution, history);+ return null;+ }).when(aslExecutor).executeAsync(any(StateMachine.class), any(Execution.class), anyList(),+ isNull(), any());++ StepFunctionsService beforeRestart = serviceWithStorage(storage);+ Execution started = beforeRestart.startExecution(STATE_MACHINE_ARN, "completed", "{}", "us-east-1");+ List<HistoryEvent> expectedHistory = beforeRestart.getExecutionHistory(started.getExecutionArn());+ storage.flushAll();++ StepFunctionsService afterRestart = serviceWithStorage(new PersistentTestStorageFactory(tempDir));++ assertEquals("SUCCEEDED", afterRestart.describeExecution(EXECUTION_ARN).getStatus());+ assertEquals("{\"result\":true}", afterRestart.describeExecution(EXECUTION_ARN).getOutput());+ assertHistoryEquals(expectedHistory, afterRestart.getExecutionHistory(EXECUTION_ARN));+ }++ @Test+ void waitingExecutionIsAbortedWithHistoryAndOldTaskTokenIsRejectedAfterReload() {+ PersistentTestStorageFactory storage = new PersistentTestStorageFactory(tempDir);+ Mockito.when(regionResolver.buildArn("states", "us-east-1",+ "execution:TestStateMachine:waiting"))+ .thenReturn("arn:aws:states:us-east-1:000000000000:execution:TestStateMachine:waiting");+ Mockito.when(regionResolver.buildArn("states", "us-east-1", "activity:waiting"))+ .thenReturn("arn:aws:states:us-east-1:000000000000:activity:waiting");+ AtomicReference<StepFunctionsService> serviceReference = new AtomicReference<>();+ Instance<StepFunctionsService> serviceInstance = Mockito.mock(Instance.class);+ Mockito.when(serviceInstance.get()).thenAnswer(ignored -> serviceReference.get());+ RestartableAslExecutor realExecutor = realExecutor(serviceInstance);+ StepFunctionsService beforeRestart = serviceWithStorage(storage, realExecutor);+ serviceReference.set(beforeRestart);+ Activity activity = beforeRestart.createActivity("waiting", "us-east-1", Map.of());+ StateMachine stateMachine = stateMachine(activity.getActivityArn());+ storage.create("stepfunctions", "sfn-state-machines.json",+ new TypeReference<Map<String, StateMachine>>() {})+ .putForAccount(DEFAULT_ACCOUNT, STATE_MACHINE_ARN, stateMachine);+ Execution started = beforeRestart.startExecution(STATE_MACHINE_ARN, "waiting", "{}", "us-east-1");+ ActivityTask task = beforeRestart.getActivityTask(activity.getActivityArn(), "worker");+ String taskToken = task.getTaskToken();+ List<HistoryEvent> expectedHistory = new ArrayList<>(beforeRestart.getExecutionHistory(started.getExecutionArn()));+ storage.flushAll();+ realExecutor.stop();++ try {+ StepFunctionsService afterRestart = serviceWithStorage(new PersistentTestStorageFactory(tempDir));+ afterRestart.abortAbandonedExecutions();++ Execution reloaded = afterRestart.describeExecution(started.getExecutionArn());+ assertEquals("ABORTED", reloaded.getStatus());+ assertNull(reloaded.getOutput());+ assertNull(reloaded.getError());+ assertNull(reloaded.getCause());+ HistoryEvent aborted = new HistoryEvent();+ aborted.setId(expectedHistory.size() + 1L);+ aborted.setPreviousEventId((long) expectedHistory.size());+ aborted.setType("ExecutionAborted");+ aborted.setDetails(Map.of());+ expectedHistory.add(aborted);+ List<HistoryEvent> actualHistory = afterRestart.getExecutionHistory(started.getExecutionArn());+ aborted.setTimestamp(actualHistory.getLast().getTimestamp());+ assertHistoryEquals(expectedHistory, actualHistory);+ assertFalse(afterRestart.sendTaskSuccess(taskToken, "{\"done\":true}"));+ } finally {+ beforeRestart.abortAbandonedExecutions();+ beforeRestart.clear();+ assertTrue(realExecutor.awaitCompletion());+ }+ }+@Testvoid abandonedRunningExecutionIsAbortedOnRestart() {SharedStorageFactory storage = new SharedStorageFactory();@@ -219,12 +354,87 @@ class StepFunctionsServicePersistenceTest {return execution;}+ private static StateMachine stateMachine() {+ StateMachine stateMachine = new StateMachine();+ stateMachine.setStateMachineArn(STATE_MACHINE_ARN);+ stateMachine.setName("TestStateMachine");+ stateMachine.setRoleArn("arn:aws:iam::000000000000:role/test-role");+ stateMachine.setType("STANDARD");+ stateMachine.setDefinition("{\"StartAt\":\"Done\",\"States\":{\"Done\":{\"Type\":\"Pass\",\"End\":true}}}");+ return stateMachine;+ }++ private static StateMachine stateMachine(String activityArn) {+ StateMachine stateMachine = stateMachine();+ stateMachine.setDefinition("{\"StartAt\":\"Wait\",\"States\":{\"Wait\":{\"Type\":\"Task\",\"Resource\":\""+ + activityArn + ".waitForTaskToken\",\"End\":true}}}");+ return stateMachine;+ }++ private static RestartableAslExecutor realExecutor(Instance<StepFunctionsService> serviceInstance) {+ return new RestartableAslExecutor(serviceInstance);+ }++ private static final class RestartableAslExecutor extends AslExecutor {+ private final CountDownLatch completion = new CountDownLatch(1);++ private RestartableAslExecutor(Instance<StepFunctionsService> serviceInstance) {+ super(null, null, null, null, null, null, null, null, null, null,+ null, null, null, new ObjectMapper(), null, serviceInstance, null, null, null);+ }++ private boolean awaitCompletion() {+ try {+ return completion.await(5, TimeUnit.SECONDS);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ return false;+ }+ }++ @Override+ public void executeAsync(StateMachine stateMachine, Execution execution,+ List<HistoryEvent> history, MockedTestCase mockedTestCase,+ java.util.function.BiConsumer<Execution, List<HistoryEvent>> onUpdate) {+ super.executeAsync(stateMachine, execution, history, mockedTestCase,+ (updatedExecution, updatedHistory) -> {+ try {+ onUpdate.accept(updatedExecution, updatedHistory);+ } finally {+ completion.countDown();+ }+ });+ }++ @Override+ void stop() {+ // Keep the waiting worker alive until the test has reloaded the persisted snapshot.+ }+ }++ private static void assertHistoryEquals(List<HistoryEvent> expected, List<HistoryEvent> actual) {+ assertEquals(expected.size(), actual.size());+ for (int index = 0; index < expected.size(); index++) {+ HistoryEvent expectedEvent = expected.get(index);+ HistoryEvent actualEvent = actual.get(index);+ assertEquals(expectedEvent.getId(), actualEvent.getId());+ assertEquals(expectedEvent.getTimestamp(), actualEvent.getTimestamp());+ assertEquals(expectedEvent.getType(), actualEvent.getType());+ assertEquals(expectedEvent.getPreviousEventId(), actualEvent.getPreviousEventId());+ assertEquals(expectedEvent.getDetails(), actualEvent.getDetails());+ }+ }+private StepFunctionsService serviceWithStorage(StorageFactory storage) {- return new StepFunctionsService(storage, regionResolver, aslExecutor,+ return serviceWithStorage(storage, aslExecutor);+ }++ private StepFunctionsService serviceWithStorage(StorageFactory storage, AslExecutor executor) {+ return new StepFunctionsService(storage, regionResolver, executor,new ObjectMapper(), mockLoader);}- private static AccountAwareStorageBackend<Execution> executionStore(SharedStorageFactory storage) {+ private static AccountAwareStorageBackend<Execution> executionStore(StorageFactory storage) {return storage.create("stepfunctions", "sfn-executions.json",new TypeReference<Map<String, Execution>>() {});}@@ -245,4 +455,32 @@ class StepFunctionsServicePersistenceTest {fileName, ignored -> AccountAwareStorageBackend.inMemory(DEFAULT_ACCOUNT));}}++ private static final class PersistentTestStorageFactory extends StorageFactory {+ private final Path directory;+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();++ private PersistentTestStorageFactory(Path directory) {+ super(null, null);+ this.directory = directory;+ }++ @Override+ @SuppressWarnings("unchecked")+ public <V> AccountAwareStorageBackend<V> create(String serviceName,+ String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName, ignored -> {+ PersistentStorage<String, V> storage = new PersistentStorage<>(+ directory.resolve(fileName), typeReference);+ storage.load();+ return new AccountAwareStorageBackend<>(storage, null, DEFAULT_ACCOUNT);+ });+ }++ @Override+ public void flushAll() {+ stores.values().forEach(StorageBackend::flush);+ }+ }}