Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(stepfunctions): persist execution history across restarts (#3291)

Niels Kootstra35159c5aaabe
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.md
index fefadadd35..77a0abe100 100644
--- a/docs/services/step-functions.md
+++ b/docs/services/step-functions.md
@@ -510,11 +510,12 @@ of every account are swept, each written back under its own account. Executions
reached a terminal status are left untouched, and so is the status and `stopDate` of one this sweep
aborted on an earlier boot.
-Execution histories are held in memory, not in storage. The events recorded before the restart are
-gone, so the execution cannot be resumed, and `GetExecutionHistory` reports a single
-`ExecutionAborted` event, with an empty `executionAbortedEventDetails`, only for the boot that
-aborted it: after a further restart the execution is already terminal, no event is written, and the
-history is empty while `DescribeExecution` still reports the status and `stopDate`.
+Execution history is stored with the execution. While an execution is running, the current history
+is checkpointed every 100 events and when the execution reaches a terminal state. A graceful
+shutdown flushes the current execution state before the emulator stops. On restart, persisted
+history is retained, and a previously running execution is marked `ABORTED` with one
+`ExecutionAborted` event appended. After a further restart, the execution is already terminal, so
+no additional event is written.
## Configuration
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
index 1fc25d05df..c859702370 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
@@ -37,11 +37,13 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.function.IntConsumer;
@ApplicationScoped
public class StepFunctionsService implements Resettable, ResourceProvider {
private static final Logger LOG = Logger.getLogger(StepFunctionsService.class);
+ private static final int HISTORY_PERSIST_CHECKPOINT = 100;
private final StorageBackend<String, StateMachine> stateMachineStore;
// Account-aware: the startup sweep has no request context and must reach every account.
@@ -577,9 +579,11 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
exec.setName(execName);
exec.setInput(input);
exec.setStatus("RUNNING");
- executionStore.put(arn, exec);
-
- history = new ExecutionHistory();
+ history = new ExecutionHistory(eventCount -> {
+ if (eventCount % HISTORY_PERSIST_CHECKPOINT == 0) {
+ executionStore.put(arn, exec);
+ }
+ });
var startEvent = new HistoryEvent();
startEvent.setId(1L);
startEvent.setPreviousEventId(0L);
@@ -587,7 +591,9 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
startEvent.setDetails(Map.of("input", input != null ? input : "{}",
"roleArn", sm.getRoleArn() != null ? sm.getRoleArn() : "",
"inputDetails", Map.of("truncated", false)));
+ exec.setHistory(history);
history.add(startEvent);
+ executionStore.put(arn, exec);
historyCache.put(arn, history);
}
@@ -794,18 +800,23 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
if (cause != null) {
details.put("cause", cause);
}
- // An execution can outlive its history: the executions are stored, the histories are held in
- // memory only, so a restart in persistent mode brings a RUNNING execution back with nothing
- // behind it. The abort still gets recorded, against a history that starts here.
- historyCache.computeIfAbsent(arn, key -> new ExecutionHistory())
- .sealWith("ExecutionAborted", details);
+ // A restart can leave a RUNNING execution without its worker and token future. Preserve all
+ // persisted events, then append the terminal event that explains the deterministic recovery.
+ ExecutionHistory history = historyCache.computeIfAbsent(arn,
+ key -> new ExecutionHistory(exec.getHistory(), () -> { }, false));
+ exec.setHistory(history);
+ history.sealWith("ExecutionAborted", details);
return true;
}
public List<HistoryEvent> getExecutionHistory(String arn) {
- describeExecution(arn);
- ExecutionHistory history = historyCache.get(arn);
- return history != null ? history : Collections.emptyList();
+ Execution exec = describeExecution(arn);
+ return historyCache.computeIfAbsent(arn,
+ key -> new ExecutionHistory(exec.getHistory(), () -> { }, isTerminal(exec.getStatus())));
+ }
+
+ private static boolean isTerminal(String status) {
+ return !"RUNNING".equals(status);
}
/**
@@ -821,14 +832,37 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
private static final long serialVersionUID = 1L;
+ private final IntConsumer onAppend;
private boolean sealed;
+ ExecutionHistory() {
+ this(eventCount -> { });
+ }
+
+ ExecutionHistory(Runnable onAppend) {
+ this(eventCount -> onAppend.run());
+ }
+
+ ExecutionHistory(IntConsumer onAppend) {
+ this.onAppend = onAppend;
+ }
+
+ ExecutionHistory(List<HistoryEvent> events, Runnable onAppend, boolean sealed) {
+ super(events != null ? events : List.of());
+ this.onAppend = eventCount -> onAppend.run();
+ this.sealed = sealed;
+ }
+
@Override
public synchronized boolean add(HistoryEvent event) {
if (sealed) {
return false;
}
- return super.add(event);
+ boolean added = super.add(event);
+ if (added) {
+ onAppend.accept(size());
+ }
+ return added;
}
/** Appends the terminal event, numbered from the end of the history, and takes no more. */
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java
index 5ae0b6c722..c54bad1714 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java
@@ -3,6 +3,9 @@ package io.github.hectorvent.floci.services.stepfunctions.model;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.util.ArrayList;
+import java.util.List;
+
@RegisterForReflection
@JsonIgnoreProperties(ignoreUnknown = true)
public class Execution {
@@ -16,6 +19,7 @@ public class Execution {
private Double stopDate;
private String error;
private String cause;
+ private List<HistoryEvent> history = new ArrayList<>();
public Execution() {
this.startDate = System.currentTimeMillis() / 1000.0;
@@ -50,4 +54,9 @@ public class Execution {
public String getCause() { return cause; }
public void setCause(String cause) { this.cause = cause; }
+
+ public List<HistoryEvent> getHistory() { return history; }
+ public void setHistory(List<HistoryEvent> history) {
+ this.history = history != null ? history : new ArrayList<>();
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsExecutionHistoryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsExecutionHistoryIntegrationTest.java
index 53c4607a40..fd0fb48b30 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsExecutionHistoryIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsExecutionHistoryIntegrationTest.java
@@ -62,6 +62,16 @@ class StepFunctionsExecutionHistoryIntegrationTest {
String executionArn = startExecution(stateMachineArn);
waitForExecution(executionArn);
+ given()
+ .header("X-Amz-Target", "AWSStepFunctions.DescribeExecution")
+ .contentType(SFN_CONTENT_TYPE)
+ .body(String.format("{\"executionArn\":\"%s\"}", executionArn))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("history", nullValue());
+
given()
.header("X-Amz-Target", "AWSStepFunctions.GetExecutionHistory")
.contentType(SFN_CONTENT_TYPE)
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsServicePersistenceTest.java
index 8cbf72e92a..235ad232ff 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsServicePersistenceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsServicePersistenceTest.java
@@ -4,11 +4,18 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.PersistentStorage;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.stepfunctions.model.Activity;
+import io.github.hectorvent.floci.services.stepfunctions.model.ActivityTask;
import io.github.hectorvent.floci.services.stepfunctions.model.Execution;
import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;
+import io.github.hectorvent.floci.services.stepfunctions.model.MockedTestCase;
+import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;
+import jakarta.enterprise.inject.Instance;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mockito;
import java.text.MessageFormat;
@@ -17,20 +24,33 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.nio.file.Path;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.doAnswer;
/** Verifies executions abandoned by a restart reach a terminal status when the emulator comes back. */
class StepFunctionsServicePersistenceTest {
+ @TempDir
+ Path tempDir;
+
private static final String DEFAULT_ACCOUNT = "000000000000";
private static final String OTHER_ACCOUNT = "222222222222";
private static final String EXECUTION_ARN =
@@ -46,6 +66,121 @@ class StepFunctionsServicePersistenceTest {
private final SfnMockLoader mockLoader = Mockito.mock(SfnMockLoader.class);
private final RegionResolver regionResolver = Mockito.mock(RegionResolver.class);
+ @Test
+ void historyCheckpointCallbackRunsAtBoundedIntervals() {
+ AtomicInteger lastCheckpoint = new AtomicInteger();
+ StepFunctionsService.ExecutionHistory history =
+ new StepFunctionsService.ExecutionHistory(eventCount -> {
+ if (eventCount % 100 == 0) {
+ lastCheckpoint.set(eventCount);
+ }
+ });
+
+ for (int eventId = 1; eventId <= 99; eventId++) {
+ HistoryEvent event = new HistoryEvent();
+ event.setId((long) eventId);
+ history.add(event);
+ }
+ assertEquals(0, lastCheckpoint.get());
+
+ HistoryEvent checkpointEvent = new HistoryEvent();
+ checkpointEvent.setId(100L);
+ history.add(checkpointEvent);
+
+ assertEquals(100, lastCheckpoint.get());
+ }
+
+ @Test
+ void completedExecutionRetainsExactHistoryAfterReload() {
+ PersistentTestStorageFactory storage = new PersistentTestStorageFactory(tempDir);
+ StateMachine stateMachine = stateMachine();
+ storage.create("stepfunctions", "sfn-state-machines.json",
+ new TypeReference<Map<String, StateMachine>>() {})
+ .putForAccount(DEFAULT_ACCOUNT, STATE_MACHINE_ARN, stateMachine);
+ Mockito.when(regionResolver.buildArn("states", "us-east-1",
+ "execution:TestStateMachine:completed"))
+ .thenReturn(EXECUTION_ARN);
+ doAnswer(invocation -> {
+ Execution execution = invocation.getArgument(1);
+ List<HistoryEvent> history = invocation.getArgument(2);
+ HistoryEvent succeeded = new HistoryEvent();
+ succeeded.setId(2L);
+ succeeded.setPreviousEventId(1L);
+ succeeded.setType("ExecutionSucceeded");
+ succeeded.setDetails(Map.of("output", "{\"result\":true}"));
+ history.add(succeeded);
+ execution.setStatus("SUCCEEDED");
+ execution.setOutput("{\"result\":true}");
+ invocation.<java.util.function.BiConsumer<Execution, List<HistoryEvent>>>getArgument(4)
+ .accept(execution, history);
+ return null;
+ }).when(aslExecutor).executeAsync(any(StateMachine.class), any(Execution.class), anyList(),
+ isNull(), any());
+
+ StepFunctionsService beforeRestart = serviceWithStorage(storage);
+ Execution started = beforeRestart.startExecution(STATE_MACHINE_ARN, "completed", "{}", "us-east-1");
+ List<HistoryEvent> expectedHistory = beforeRestart.getExecutionHistory(started.getExecutionArn());
+ storage.flushAll();
+
+ StepFunctionsService afterRestart = serviceWithStorage(new PersistentTestStorageFactory(tempDir));
+
+ assertEquals("SUCCEEDED", afterRestart.describeExecution(EXECUTION_ARN).getStatus());
+ assertEquals("{\"result\":true}", afterRestart.describeExecution(EXECUTION_ARN).getOutput());
+ assertHistoryEquals(expectedHistory, afterRestart.getExecutionHistory(EXECUTION_ARN));
+ }
+
+ @Test
+ void waitingExecutionIsAbortedWithHistoryAndOldTaskTokenIsRejectedAfterReload() {
+ PersistentTestStorageFactory storage = new PersistentTestStorageFactory(tempDir);
+ Mockito.when(regionResolver.buildArn("states", "us-east-1",
+ "execution:TestStateMachine:waiting"))
+ .thenReturn("arn:aws:states:us-east-1:000000000000:execution:TestStateMachine:waiting");
+ Mockito.when(regionResolver.buildArn("states", "us-east-1", "activity:waiting"))
+ .thenReturn("arn:aws:states:us-east-1:000000000000:activity:waiting");
+ AtomicReference<StepFunctionsService> serviceReference = new AtomicReference<>();
+ Instance<StepFunctionsService> serviceInstance = Mockito.mock(Instance.class);
+ Mockito.when(serviceInstance.get()).thenAnswer(ignored -> serviceReference.get());
+ RestartableAslExecutor realExecutor = realExecutor(serviceInstance);
+ StepFunctionsService beforeRestart = serviceWithStorage(storage, realExecutor);
+ serviceReference.set(beforeRestart);
+ Activity activity = beforeRestart.createActivity("waiting", "us-east-1", Map.of());
+ StateMachine stateMachine = stateMachine(activity.getActivityArn());
+ storage.create("stepfunctions", "sfn-state-machines.json",
+ new TypeReference<Map<String, StateMachine>>() {})
+ .putForAccount(DEFAULT_ACCOUNT, STATE_MACHINE_ARN, stateMachine);
+ Execution started = beforeRestart.startExecution(STATE_MACHINE_ARN, "waiting", "{}", "us-east-1");
+ ActivityTask task = beforeRestart.getActivityTask(activity.getActivityArn(), "worker");
+ String taskToken = task.getTaskToken();
+ List<HistoryEvent> expectedHistory = new ArrayList<>(beforeRestart.getExecutionHistory(started.getExecutionArn()));
+ storage.flushAll();
+ realExecutor.stop();
+
+ try {
+ StepFunctionsService afterRestart = serviceWithStorage(new PersistentTestStorageFactory(tempDir));
+ afterRestart.abortAbandonedExecutions();
+
+ Execution reloaded = afterRestart.describeExecution(started.getExecutionArn());
+ assertEquals("ABORTED", reloaded.getStatus());
+ assertNull(reloaded.getOutput());
+ assertNull(reloaded.getError());
+ assertNull(reloaded.getCause());
+ HistoryEvent aborted = new HistoryEvent();
+ aborted.setId(expectedHistory.size() + 1L);
+ aborted.setPreviousEventId((long) expectedHistory.size());
+ aborted.setType("ExecutionAborted");
+ aborted.setDetails(Map.of());
+ expectedHistory.add(aborted);
+ List<HistoryEvent> actualHistory = afterRestart.getExecutionHistory(started.getExecutionArn());
+ aborted.setTimestamp(actualHistory.getLast().getTimestamp());
+ assertHistoryEquals(expectedHistory, actualHistory);
+ assertFalse(afterRestart.sendTaskSuccess(taskToken, "{\"done\":true}"));
+ } finally {
+ beforeRestart.abortAbandonedExecutions();
+ beforeRestart.clear();
+ assertTrue(realExecutor.awaitCompletion());
+ }
+ }
+
@Test
void abandonedRunningExecutionIsAbortedOnRestart() {
SharedStorageFactory storage = new SharedStorageFactory();
@@ -219,12 +354,87 @@ class StepFunctionsServicePersistenceTest {
return execution;
}
+ private static StateMachine stateMachine() {
+ StateMachine stateMachine = new StateMachine();
+ stateMachine.setStateMachineArn(STATE_MACHINE_ARN);
+ stateMachine.setName("TestStateMachine");
+ stateMachine.setRoleArn("arn:aws:iam::000000000000:role/test-role");
+ stateMachine.setType("STANDARD");
+ stateMachine.setDefinition("{\"StartAt\":\"Done\",\"States\":{\"Done\":{\"Type\":\"Pass\",\"End\":true}}}");
+ return stateMachine;
+ }
+
+ private static StateMachine stateMachine(String activityArn) {
+ StateMachine stateMachine = stateMachine();
+ stateMachine.setDefinition("{\"StartAt\":\"Wait\",\"States\":{\"Wait\":{\"Type\":\"Task\",\"Resource\":\""
+ + activityArn + ".waitForTaskToken\",\"End\":true}}}");
+ return stateMachine;
+ }
+
+ private static RestartableAslExecutor realExecutor(Instance<StepFunctionsService> serviceInstance) {
+ return new RestartableAslExecutor(serviceInstance);
+ }
+
+ private static final class RestartableAslExecutor extends AslExecutor {
+ private final CountDownLatch completion = new CountDownLatch(1);
+
+ private RestartableAslExecutor(Instance<StepFunctionsService> serviceInstance) {
+ super(null, null, null, null, null, null, null, null, null, null,
+ null, null, null, new ObjectMapper(), null, serviceInstance, null, null, null);
+ }
+
+ private boolean awaitCompletion() {
+ try {
+ return completion.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return false;
+ }
+ }
+
+ @Override
+ public void executeAsync(StateMachine stateMachine, Execution execution,
+ List<HistoryEvent> history, MockedTestCase mockedTestCase,
+ java.util.function.BiConsumer<Execution, List<HistoryEvent>> onUpdate) {
+ super.executeAsync(stateMachine, execution, history, mockedTestCase,
+ (updatedExecution, updatedHistory) -> {
+ try {
+ onUpdate.accept(updatedExecution, updatedHistory);
+ } finally {
+ completion.countDown();
+ }
+ });
+ }
+
+ @Override
+ void stop() {
+ // Keep the waiting worker alive until the test has reloaded the persisted snapshot.
+ }
+ }
+
+ private static void assertHistoryEquals(List<HistoryEvent> expected, List<HistoryEvent> actual) {
+ assertEquals(expected.size(), actual.size());
+ for (int index = 0; index < expected.size(); index++) {
+ HistoryEvent expectedEvent = expected.get(index);
+ HistoryEvent actualEvent = actual.get(index);
+ assertEquals(expectedEvent.getId(), actualEvent.getId());
+ assertEquals(expectedEvent.getTimestamp(), actualEvent.getTimestamp());
+ assertEquals(expectedEvent.getType(), actualEvent.getType());
+ assertEquals(expectedEvent.getPreviousEventId(), actualEvent.getPreviousEventId());
+ assertEquals(expectedEvent.getDetails(), actualEvent.getDetails());
+ }
+ }
+
private StepFunctionsService serviceWithStorage(StorageFactory storage) {
- return new StepFunctionsService(storage, regionResolver, aslExecutor,
+ return serviceWithStorage(storage, aslExecutor);
+ }
+
+ private StepFunctionsService serviceWithStorage(StorageFactory storage, AslExecutor executor) {
+ return new StepFunctionsService(storage, regionResolver, executor,
new ObjectMapper(), mockLoader);
}
- private static AccountAwareStorageBackend<Execution> executionStore(SharedStorageFactory storage) {
+ private static AccountAwareStorageBackend<Execution> executionStore(StorageFactory storage) {
return storage.create("stepfunctions", "sfn-executions.json",
new TypeReference<Map<String, Execution>>() {});
}
@@ -245,4 +455,32 @@ class StepFunctionsServicePersistenceTest {
fileName, ignored -> AccountAwareStorageBackend.inMemory(DEFAULT_ACCOUNT));
}
}
+
+ private static final class PersistentTestStorageFactory extends StorageFactory {
+ private final Path directory;
+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
+
+ private PersistentTestStorageFactory(Path directory) {
+ super(null, null);
+ this.directory = directory;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> AccountAwareStorageBackend<V> create(String serviceName,
+ String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName, ignored -> {
+ PersistentStorage<String, V> storage = new PersistentStorage<>(
+ directory.resolve(fileName), typeReference);
+ storage.load();
+ return new AccountAwareStorageBackend<>(storage, null, DEFAULT_ACCOUNT);
+ });
+ }
+
+ @Override
+ public void flushAll() {
+ stores.values().forEach(StorageBackend::flush);
+ }
+ }
}