Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(stepfunctions): retain claimed callback outcomes (#6311)

Andre Kurait2babe62ce4ec
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index 47e85e5f2c..76beabd891 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -60,6 +60,7 @@ import io.github.hectorvent.floci.services.scheduler.model.ScheduleRequest;
import io.github.hectorvent.floci.services.sns.SnsJsonHandler;
import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;
import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService.TokenTimeout;
+import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService.TokenWait;
import io.github.hectorvent.floci.services.stepfunctions.model.Execution;
import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;
import io.github.hectorvent.floci.services.stepfunctions.model.MapRun;
@@ -1084,13 +1085,12 @@ public class AslExecutor {
try {
return future.get(wakeAtNanos - nanoTime.getAsLong(), TimeUnit.NANOSECONDS);
} catch (java.util.concurrent.TimeoutException e) {
- TokenTimeout expired = sfnService.get().expirePendingToken(taskToken, nanoTime.getAsLong(),
+ TokenWait decision = sfnService.get().expirePendingToken(taskToken, nanoTime.getAsLong(),
executionDeadlineNanos, timeoutDeadlineNanos, heartbeatSeconds);
- if (expired == TokenTimeout.CLAIMED) {
- // Claiming removes the token before completing its future outside the lock.
- return future.get();
+ if (decision.timeout() == TokenTimeout.CLAIMED) {
+ return decision.result().get();
}
- RuntimeException timeout = switch (expired) {
+ RuntimeException timeout = switch (decision.timeout()) {
case EXECUTION -> new ExecutionTimedOutException();
case TASK -> new TaskTimedOutException("States.Timeout");
case HEARTBEAT -> new TaskTimedOutException("States.HeartbeatTimeout");
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
index d081dcb753..d6d2c4dd8e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
@@ -62,13 +62,33 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
private final Map<String, ExecutionHistory> historyCache = new ConcurrentHashMap<>();
private final Map<String, BlockingQueue<ActivityTask>> activityQueues = new ConcurrentHashMap<>();
private final Map<String, PendingTask> pendingTaskTokens = new ConcurrentHashMap<>();
+ private int tokenResetsInProgress;
// Only recent closures are classified as TaskTimedOut: five minutes, at most 1,024 tokens.
// Evicted/older tokens remain InvalidToken. No task payloads are retained.
private final Map<String, ClosedTask> closedTaskTokens = new LinkedHashMap<>();
private static final long CLOSED_TOKEN_RETENTION_NANOS = TimeUnit.MINUTES.toNanos(5);
private static final int MAX_CLOSED_TOKENS = 1024;
- private record PendingTask(String accountId, CompletableFuture<JsonNode> future) {}
+ private record PendingTask(String accountId, CompletableFuture<JsonNode> future, TaskResult result) {}
+
+ record TaskResult(JsonNode output, AslExecutor.FailStateException failure) {
+ JsonNode get() {
+ if (failure != null) {
+ throw failure;
+ }
+ return output;
+ }
+
+ void complete(CompletableFuture<JsonNode> future) {
+ if (failure != null) {
+ future.completeExceptionally(failure);
+ } else {
+ future.complete(output);
+ }
+ }
+ }
+
+ record TokenWait(TokenTimeout timeout, TaskResult result) {}
private record ClosedTask(String accountId, long closedAtNanos) {}
// When each pending token last showed progress, so a Task's HeartbeatSeconds can bound the gap
// between heartbeats instead of the whole wait.
@@ -157,12 +177,29 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
public void clear() {
historyCache.clear();
activityQueues.clear();
- pendingTaskTokens.values().forEach(f -> f.future().completeExceptionally(new RuntimeException("StepFunctionsService cleared")));
- pendingTaskTokens.clear();
+ Map<String, PendingTask> pending;
synchronized (this) {
+ tokenResetsInProgress++;
+ pendingTaskTokens.replaceAll((token, task) -> task.result() != null ? task
+ : new PendingTask(task.accountId(), task.future(), new TaskResult(null,
+ new AslExecutor.FailStateException("States.TaskFailed", "StepFunctionsService cleared"))));
+ pending = new HashMap<>(pendingTaskTokens);
closedTaskTokens.clear();
+ taskHeartbeatNanos.clear();
+ }
+ try {
+ for (Map.Entry<String, PendingTask> entry : pending.entrySet()) {
+ PendingTask task = entry.getValue();
+ task.result().complete(task.future());
+ synchronized (this) {
+ pendingTaskTokens.remove(entry.getKey(), task);
+ }
+ }
+ } finally {
+ synchronized (this) {
+ tokenResetsInProgress--;
+ }
}
- taskHeartbeatNanos.clear();
}
@Override
@@ -1196,10 +1233,17 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
queue.add(new ActivityTask(taskToken, input));
}
- public synchronized CompletableFuture<JsonNode> registerPendingToken(String token, String accountId) {
+ public CompletableFuture<JsonNode> registerPendingToken(String token, String accountId) {
+ Objects.requireNonNull(accountId);
CompletableFuture<JsonNode> future = new CompletableFuture<>();
- pendingTaskTokens.put(token, new PendingTask(Objects.requireNonNull(accountId), future));
- taskHeartbeatNanos.put(token, System.nanoTime());
+ synchronized (this) {
+ if (tokenResetsInProgress == 0) {
+ pendingTaskTokens.put(token, new PendingTask(accountId, future, null));
+ taskHeartbeatNanos.put(token, System.nanoTime());
+ return future;
+ }
+ }
+ future.completeExceptionally(new AslExecutor.FailStateException("States.TaskFailed", "StepFunctionsService cleared"));
return future;
}
@@ -1216,10 +1260,14 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
enum TokenTimeout { NONE, CLAIMED, EXECUTION, TASK, HEARTBEAT }
/** Selects an expired clock and closes its token under the callback and heartbeat lock. */
- synchronized TokenTimeout expirePendingToken(String taskToken, long now, long executionDeadlineNanos,
+ synchronized TokenWait expirePendingToken(String taskToken, long now, long executionDeadlineNanos,
long timeoutDeadlineNanos, int heartbeatSeconds) {
- if (!pendingTaskTokens.containsKey(taskToken)) {
- return TokenTimeout.CLAIMED;
+ PendingTask pending = pendingTaskTokens.get(taskToken);
+ if (pending == null) {
+ return new TokenWait(TokenTimeout.TASK, null);
+ }
+ if (pending.result() != null) {
+ return new TokenWait(TokenTimeout.CLAIMED, pending.result());
}
TokenTimeout expired;
if (now >= executionDeadlineNanos) {
@@ -1230,10 +1278,10 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
+ TimeUnit.SECONDS.toNanos(heartbeatSeconds)) {
expired = TokenTimeout.HEARTBEAT;
} else {
- return TokenTimeout.NONE;
+ return new TokenWait(TokenTimeout.NONE, null);
}
discardPendingToken(taskToken);
- return expired;
+ return new TokenWait(expired, null);
}
/** Retires a token when its task stops waiting, including timeout and resource failure. */
@@ -1262,6 +1310,9 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
PendingTask pending = token == null ? null : pendingTaskTokens.get(token);
if (pending != null) {
requireTokenOwner(pending.accountId(), accountId);
+ if (pending.result() != null) {
+ throw new AwsException("TaskTimedOut", "The associated task has already been closed.", 400);
+ }
return pending;
}
ClosedTask closed = token == null ? null : closedTaskTokens.get(token);
@@ -1282,12 +1333,14 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
}
}
- private synchronized CompletableFuture<JsonNode> claimPendingTask(String token) {
- PendingTask pending = requirePendingTask(token);
- pendingTaskTokens.remove(token);
- taskHeartbeatNanos.remove(token);
- rememberClosed(token, pending.accountId());
- return pending.future();
+ CompletableFuture<JsonNode> claimPendingTask(String token, TaskResult result) {
+ synchronized (this) {
+ PendingTask pending = requirePendingTask(token);
+ pendingTaskTokens.put(token, new PendingTask(pending.accountId(), pending.future(), result));
+ taskHeartbeatNanos.remove(token);
+ rememberClosed(token, pending.accountId());
+ return pending.future();
+ }
}
/** Claims an owned callback exactly once; completion runs outside the token lock. */
@@ -1304,14 +1357,16 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
if (result == null || result.isMissingNode()) {
throw new AwsException("InvalidOutput", "Output is not valid JSON.", 400);
}
- CompletableFuture<JsonNode> future = claimPendingTask(taskToken);
- future.complete(result);
+ TaskResult completion = new TaskResult(result, null);
+ CompletableFuture<JsonNode> future = claimPendingTask(taskToken, completion);
+ completion.complete(future);
return true;
}
public boolean sendTaskFailure(String taskToken, String cause, String error) {
- CompletableFuture<JsonNode> future = claimPendingTask(taskToken);
- future.completeExceptionally(new AslExecutor.FailStateException(error, cause));
+ TaskResult completion = new TaskResult(null, new AslExecutor.FailStateException(error, cause));
+ CompletableFuture<JsonNode> future = claimPendingTask(taskToken, completion);
+ completion.complete(future);
return true;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.java
index dbd15a68ca..2b3c559dd9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.java
@@ -5,15 +5,21 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService.TaskResult;
import jakarta.enterprise.inject.Instance;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.LongSupplier;
@@ -21,8 +27,13 @@ import java.util.function.LongSupplier;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@Timeout(10)
@@ -106,6 +117,124 @@ class AslExecutorTokenTimeoutTest {
assertFalse(future.isCancelled());
}
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ @SuppressWarnings("unchecked")
+ void claimedResultDoesNotWaitForCallbackCompletion(boolean failure) throws Exception {
+ when(resolver.getAccountId()).thenReturn(OWNER);
+ StepFunctionsService callbackService = spy(service);
+ CompletableFuture<JsonNode> future = callbackService.registerPendingToken("stalled", OWNER);
+ CountDownLatch claimed = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ CompletableFuture<JsonNode> result = (CompletableFuture<JsonNode>) invocation.callRealMethod();
+ claimed.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ return result;
+ }).when(callbackService).claimPendingTask(eq("stalled"), any(TaskResult.class));
+ List<String> completions = new ArrayList<>();
+ future.whenComplete((value, exception) ->
+ completions.add(Thread.holdsLock(callbackService) ? "locked" : "unlocked"));
+ ExecutorService worker = Executors.newSingleThreadExecutor();
+ try {
+ Future<Boolean> callback = worker.submit(() -> failure
+ ? callbackService.sendTaskFailure("stalled", "cause", "CallbackError")
+ : callbackService.sendTaskSuccess("stalled", "{\"accepted\":true}"));
+ assertTrue(claimed.await(5, TimeUnit.SECONDS));
+ AslExecutor executor = executor(callbackService, () -> 1);
+ assertTimeoutPreemptively(Duration.ofSeconds(1), () -> {
+ if (failure) {
+ AslExecutor.FailStateException error = assertThrows(AslExecutor.FailStateException.class,
+ () -> executor.awaitToken(future, mapper.readTree("{}"), "stalled", 1, 1));
+ assertEquals("CallbackError", error.error);
+ assertEquals("cause", error.cause);
+ } else {
+ assertTrue(executor.awaitToken(future, mapper.readTree("{}"), "stalled", 1, 1)
+ .path("accepted").asBoolean());
+ }
+ });
+ assertFalse(future.isDone(), "the waiter must not depend on callback-thread completion");
+ assertEquals("TaskTimedOut", callbackResult(() -> callbackService.sendTaskHeartbeat("stalled")));
+ release.countDown();
+ assertTrue(callback.get(5, TimeUnit.SECONDS));
+ assertEquals(List.of("unlocked"), completions);
+ assertFalse(future.isCancelled());
+ } finally {
+ release.countDown();
+ worker.shutdownNow();
+ assertTrue(worker.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ void discardedTokenDoesNotLookLikeAnAcceptedCallback() throws Exception {
+ CompletableFuture<JsonNode> future = service.registerPendingToken("discarded", OWNER);
+ service.discardPendingToken("discarded");
+
+ assertTimeoutPreemptively(Duration.ofSeconds(1), () -> assertThrows(AslExecutor.TaskTimedOutException.class,
+ () -> executor(() -> 1).awaitToken(future, mapper.readTree("{}"), "discarded", 1, 1)));
+ assertTrue(future.isCancelled());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void clearPreservesClaimedOutcomeAndCompletesOutsideLock(boolean failure) throws Exception {
+ when(resolver.getAccountId()).thenReturn(OWNER);
+ CompletableFuture<JsonNode> future = service.registerPendingToken("cleared", OWNER);
+ JsonNode output = mapper.readTree("{\"accepted\":true}");
+ AslExecutor.FailStateException error = new AslExecutor.FailStateException("CallbackError", "cause");
+ service.claimPendingTask("cleared", failure ? new TaskResult(null, error) : new TaskResult(output, null));
+ List<String> completions = new ArrayList<>();
+ future.whenComplete((value, exception) -> completions.add(Thread.holdsLock(service) ? "locked" : "unlocked"));
+
+ service.clear();
+
+ if (failure) {
+ assertEquals(error, assertThrows(AslExecutor.FailStateException.class,
+ () -> executor(() -> 1).awaitToken(future, mapper.readTree("{}"), "cleared", 1, 1)));
+ } else {
+ assertEquals(output, executor(() -> 1).awaitToken(future, mapper.readTree("{}"), "cleared", 1, 1));
+ }
+ assertEquals(List.of("unlocked"), completions);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void registrationDuringResetIsRetiredUntilAllResetsFinish(boolean overlappingReset) throws Exception {
+ when(resolver.getAccountId()).thenReturn(OWNER);
+ CompletableFuture<JsonNode> first = service.registerPendingToken("first", OWNER);
+ CountDownLatch completing = new CountDownLatch(1);
+ CompletableFuture<Void> release = new CompletableFuture<>();
+ List<Boolean> completionLocks = new ArrayList<>();
+ first.whenComplete((value, error) -> {
+ completionLocks.add(Thread.holdsLock(service));
+ completing.countDown();
+ release.join();
+ });
+ ExecutorService worker = Executors.newSingleThreadExecutor();
+ try {
+ Future<?> reset = worker.submit(service::clear);
+ assertTrue(completing.await(5, TimeUnit.SECONDS));
+ if (overlappingReset) {
+ service.clear();
+ }
+ CompletableFuture<JsonNode> duringReset = service.registerPendingToken("during-reset", OWNER);
+ assertTrue(duringReset.isCompletedExceptionally());
+ assertEquals("InvalidToken", callbackResult(() -> service.sendTaskSuccess("during-reset", "{}")));
+ release.complete(null);
+ reset.get(5, TimeUnit.SECONDS);
+ assertEquals(List.of(false), completionLocks);
+
+ CompletableFuture<JsonNode> afterReset = service.registerPendingToken("after-reset", OWNER);
+ service.sendTaskSuccess("after-reset", "{}");
+ assertEquals(mapper.readTree("{}"), afterReset.get(5, TimeUnit.SECONDS));
+ } finally {
+ release.complete(null);
+ worker.shutdownNow();
+ assertTrue(worker.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
private String callbackResult(Runnable callback) {
try {
callback.run();
@@ -117,8 +246,13 @@ class AslExecutorTokenTimeoutTest {
@SuppressWarnings("unchecked")
private AslExecutor executor(LongSupplier clock) {
+ return executor(service, clock);
+ }
+
+ @SuppressWarnings("unchecked")
+ private AslExecutor executor(StepFunctionsService target, LongSupplier clock) {
Instance<StepFunctionsService> instance = mock(Instance.class);
- when(instance.get()).thenReturn(service);
+ when(instance.get()).thenReturn(target);
return AslExecutorFixture.builder().objectMapper(mapper).sfnService(instance).nanoTime(clock).build();
}
}