Light, fluffy, and always free - The AWS Local Emulator alternative
fix(stepfunctions): retain claimed callback outcomes (#6311)
Andre Kurait2babe62ce4ec
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.javaindex 47e85e5f2c..76beabd891 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java@@ -60,6 +60,7 @@ import io.github.hectorvent.floci.services.scheduler.model.ScheduleRequest;import io.github.hectorvent.floci.services.sns.SnsJsonHandler;import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService.TokenTimeout;+import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService.TokenWait;import io.github.hectorvent.floci.services.stepfunctions.model.Execution;import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;import io.github.hectorvent.floci.services.stepfunctions.model.MapRun;@@ -1084,13 +1085,12 @@ public class AslExecutor {try {return future.get(wakeAtNanos - nanoTime.getAsLong(), TimeUnit.NANOSECONDS);} catch (java.util.concurrent.TimeoutException e) {- TokenTimeout expired = sfnService.get().expirePendingToken(taskToken, nanoTime.getAsLong(),+ TokenWait decision = sfnService.get().expirePendingToken(taskToken, nanoTime.getAsLong(),executionDeadlineNanos, timeoutDeadlineNanos, heartbeatSeconds);- if (expired == TokenTimeout.CLAIMED) {- // Claiming removes the token before completing its future outside the lock.- return future.get();+ if (decision.timeout() == TokenTimeout.CLAIMED) {+ return decision.result().get();}- RuntimeException timeout = switch (expired) {+ RuntimeException timeout = switch (decision.timeout()) {case EXECUTION -> new ExecutionTimedOutException();case TASK -> new TaskTimedOutException("States.Timeout");case HEARTBEAT -> new TaskTimedOutException("States.HeartbeatTimeout");diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.javaindex d081dcb753..d6d2c4dd8e 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java@@ -62,13 +62,33 @@ public class StepFunctionsService implements Resettable, ResourceProvider {private final Map<String, ExecutionHistory> historyCache = new ConcurrentHashMap<>();private final Map<String, BlockingQueue<ActivityTask>> activityQueues = new ConcurrentHashMap<>();private final Map<String, PendingTask> pendingTaskTokens = new ConcurrentHashMap<>();+ private int tokenResetsInProgress;// Only recent closures are classified as TaskTimedOut: five minutes, at most 1,024 tokens.// Evicted/older tokens remain InvalidToken. No task payloads are retained.private final Map<String, ClosedTask> closedTaskTokens = new LinkedHashMap<>();private static final long CLOSED_TOKEN_RETENTION_NANOS = TimeUnit.MINUTES.toNanos(5);private static final int MAX_CLOSED_TOKENS = 1024;- private record PendingTask(String accountId, CompletableFuture<JsonNode> future) {}+ private record PendingTask(String accountId, CompletableFuture<JsonNode> future, TaskResult result) {}++ record TaskResult(JsonNode output, AslExecutor.FailStateException failure) {+ JsonNode get() {+ if (failure != null) {+ throw failure;+ }+ return output;+ }++ void complete(CompletableFuture<JsonNode> future) {+ if (failure != null) {+ future.completeExceptionally(failure);+ } else {+ future.complete(output);+ }+ }+ }++ record TokenWait(TokenTimeout timeout, TaskResult result) {}private record ClosedTask(String accountId, long closedAtNanos) {}// When each pending token last showed progress, so a Task's HeartbeatSeconds can bound the gap// between heartbeats instead of the whole wait.@@ -157,12 +177,29 @@ public class StepFunctionsService implements Resettable, ResourceProvider {public void clear() {historyCache.clear();activityQueues.clear();- pendingTaskTokens.values().forEach(f -> f.future().completeExceptionally(new RuntimeException("StepFunctionsService cleared")));- pendingTaskTokens.clear();+ Map<String, PendingTask> pending;synchronized (this) {+ tokenResetsInProgress++;+ pendingTaskTokens.replaceAll((token, task) -> task.result() != null ? task+ : new PendingTask(task.accountId(), task.future(), new TaskResult(null,+ new AslExecutor.FailStateException("States.TaskFailed", "StepFunctionsService cleared"))));+ pending = new HashMap<>(pendingTaskTokens);closedTaskTokens.clear();+ taskHeartbeatNanos.clear();+ }+ try {+ for (Map.Entry<String, PendingTask> entry : pending.entrySet()) {+ PendingTask task = entry.getValue();+ task.result().complete(task.future());+ synchronized (this) {+ pendingTaskTokens.remove(entry.getKey(), task);+ }+ }+ } finally {+ synchronized (this) {+ tokenResetsInProgress--;+ }}- taskHeartbeatNanos.clear();}@Override@@ -1196,10 +1233,17 @@ public class StepFunctionsService implements Resettable, ResourceProvider {queue.add(new ActivityTask(taskToken, input));}- public synchronized CompletableFuture<JsonNode> registerPendingToken(String token, String accountId) {+ public CompletableFuture<JsonNode> registerPendingToken(String token, String accountId) {+ Objects.requireNonNull(accountId);CompletableFuture<JsonNode> future = new CompletableFuture<>();- pendingTaskTokens.put(token, new PendingTask(Objects.requireNonNull(accountId), future));- taskHeartbeatNanos.put(token, System.nanoTime());+ synchronized (this) {+ if (tokenResetsInProgress == 0) {+ pendingTaskTokens.put(token, new PendingTask(accountId, future, null));+ taskHeartbeatNanos.put(token, System.nanoTime());+ return future;+ }+ }+ future.completeExceptionally(new AslExecutor.FailStateException("States.TaskFailed", "StepFunctionsService cleared"));return future;}@@ -1216,10 +1260,14 @@ public class StepFunctionsService implements Resettable, ResourceProvider {enum TokenTimeout { NONE, CLAIMED, EXECUTION, TASK, HEARTBEAT }/** Selects an expired clock and closes its token under the callback and heartbeat lock. */- synchronized TokenTimeout expirePendingToken(String taskToken, long now, long executionDeadlineNanos,+ synchronized TokenWait expirePendingToken(String taskToken, long now, long executionDeadlineNanos,long timeoutDeadlineNanos, int heartbeatSeconds) {- if (!pendingTaskTokens.containsKey(taskToken)) {- return TokenTimeout.CLAIMED;+ PendingTask pending = pendingTaskTokens.get(taskToken);+ if (pending == null) {+ return new TokenWait(TokenTimeout.TASK, null);+ }+ if (pending.result() != null) {+ return new TokenWait(TokenTimeout.CLAIMED, pending.result());}TokenTimeout expired;if (now >= executionDeadlineNanos) {@@ -1230,10 +1278,10 @@ public class StepFunctionsService implements Resettable, ResourceProvider {+ TimeUnit.SECONDS.toNanos(heartbeatSeconds)) {expired = TokenTimeout.HEARTBEAT;} else {- return TokenTimeout.NONE;+ return new TokenWait(TokenTimeout.NONE, null);}discardPendingToken(taskToken);- return expired;+ return new TokenWait(expired, null);}/** Retires a token when its task stops waiting, including timeout and resource failure. */@@ -1262,6 +1310,9 @@ public class StepFunctionsService implements Resettable, ResourceProvider {PendingTask pending = token == null ? null : pendingTaskTokens.get(token);if (pending != null) {requireTokenOwner(pending.accountId(), accountId);+ if (pending.result() != null) {+ throw new AwsException("TaskTimedOut", "The associated task has already been closed.", 400);+ }return pending;}ClosedTask closed = token == null ? null : closedTaskTokens.get(token);@@ -1282,12 +1333,14 @@ public class StepFunctionsService implements Resettable, ResourceProvider {}}- private synchronized CompletableFuture<JsonNode> claimPendingTask(String token) {- PendingTask pending = requirePendingTask(token);- pendingTaskTokens.remove(token);- taskHeartbeatNanos.remove(token);- rememberClosed(token, pending.accountId());- return pending.future();+ CompletableFuture<JsonNode> claimPendingTask(String token, TaskResult result) {+ synchronized (this) {+ PendingTask pending = requirePendingTask(token);+ pendingTaskTokens.put(token, new PendingTask(pending.accountId(), pending.future(), result));+ taskHeartbeatNanos.remove(token);+ rememberClosed(token, pending.accountId());+ return pending.future();+ }}/** Claims an owned callback exactly once; completion runs outside the token lock. */@@ -1304,14 +1357,16 @@ public class StepFunctionsService implements Resettable, ResourceProvider {if (result == null || result.isMissingNode()) {throw new AwsException("InvalidOutput", "Output is not valid JSON.", 400);}- CompletableFuture<JsonNode> future = claimPendingTask(taskToken);- future.complete(result);+ TaskResult completion = new TaskResult(result, null);+ CompletableFuture<JsonNode> future = claimPendingTask(taskToken, completion);+ completion.complete(future);return true;}public boolean sendTaskFailure(String taskToken, String cause, String error) {- CompletableFuture<JsonNode> future = claimPendingTask(taskToken);- future.completeExceptionally(new AslExecutor.FailStateException(error, cause));+ TaskResult completion = new TaskResult(null, new AslExecutor.FailStateException(error, cause));+ CompletableFuture<JsonNode> future = claimPendingTask(taskToken, completion);+ completion.complete(future);return true;}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.javaindex dbd15a68ca..2b3c559dd9 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.java@@ -5,15 +5,21 @@ import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService.TaskResult;import jakarta.enterprise.inject.Instance;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.Timeout;import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.ValueSource;+import java.time.Duration;import java.util.ArrayList;import java.util.List;import java.util.concurrent.CompletableFuture;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicInteger;import java.util.function.LongSupplier;@@ -21,8 +27,13 @@ import java.util.function.LongSupplier;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.spy;import static org.mockito.Mockito.when;@Timeout(10)@@ -106,6 +117,124 @@ class AslExecutorTokenTimeoutTest {assertFalse(future.isCancelled());}+ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ @SuppressWarnings("unchecked")+ void claimedResultDoesNotWaitForCallbackCompletion(boolean failure) throws Exception {+ when(resolver.getAccountId()).thenReturn(OWNER);+ StepFunctionsService callbackService = spy(service);+ CompletableFuture<JsonNode> future = callbackService.registerPendingToken("stalled", OWNER);+ CountDownLatch claimed = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ doAnswer(invocation -> {+ CompletableFuture<JsonNode> result = (CompletableFuture<JsonNode>) invocation.callRealMethod();+ claimed.countDown();+ assertTrue(release.await(5, TimeUnit.SECONDS));+ return result;+ }).when(callbackService).claimPendingTask(eq("stalled"), any(TaskResult.class));+ List<String> completions = new ArrayList<>();+ future.whenComplete((value, exception) ->+ completions.add(Thread.holdsLock(callbackService) ? "locked" : "unlocked"));+ ExecutorService worker = Executors.newSingleThreadExecutor();+ try {+ Future<Boolean> callback = worker.submit(() -> failure+ ? callbackService.sendTaskFailure("stalled", "cause", "CallbackError")+ : callbackService.sendTaskSuccess("stalled", "{\"accepted\":true}"));+ assertTrue(claimed.await(5, TimeUnit.SECONDS));+ AslExecutor executor = executor(callbackService, () -> 1);+ assertTimeoutPreemptively(Duration.ofSeconds(1), () -> {+ if (failure) {+ AslExecutor.FailStateException error = assertThrows(AslExecutor.FailStateException.class,+ () -> executor.awaitToken(future, mapper.readTree("{}"), "stalled", 1, 1));+ assertEquals("CallbackError", error.error);+ assertEquals("cause", error.cause);+ } else {+ assertTrue(executor.awaitToken(future, mapper.readTree("{}"), "stalled", 1, 1)+ .path("accepted").asBoolean());+ }+ });+ assertFalse(future.isDone(), "the waiter must not depend on callback-thread completion");+ assertEquals("TaskTimedOut", callbackResult(() -> callbackService.sendTaskHeartbeat("stalled")));+ release.countDown();+ assertTrue(callback.get(5, TimeUnit.SECONDS));+ assertEquals(List.of("unlocked"), completions);+ assertFalse(future.isCancelled());+ } finally {+ release.countDown();+ worker.shutdownNow();+ assertTrue(worker.awaitTermination(5, TimeUnit.SECONDS));+ }+ }++ @Test+ void discardedTokenDoesNotLookLikeAnAcceptedCallback() throws Exception {+ CompletableFuture<JsonNode> future = service.registerPendingToken("discarded", OWNER);+ service.discardPendingToken("discarded");++ assertTimeoutPreemptively(Duration.ofSeconds(1), () -> assertThrows(AslExecutor.TaskTimedOutException.class,+ () -> executor(() -> 1).awaitToken(future, mapper.readTree("{}"), "discarded", 1, 1)));+ assertTrue(future.isCancelled());+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void clearPreservesClaimedOutcomeAndCompletesOutsideLock(boolean failure) throws Exception {+ when(resolver.getAccountId()).thenReturn(OWNER);+ CompletableFuture<JsonNode> future = service.registerPendingToken("cleared", OWNER);+ JsonNode output = mapper.readTree("{\"accepted\":true}");+ AslExecutor.FailStateException error = new AslExecutor.FailStateException("CallbackError", "cause");+ service.claimPendingTask("cleared", failure ? new TaskResult(null, error) : new TaskResult(output, null));+ List<String> completions = new ArrayList<>();+ future.whenComplete((value, exception) -> completions.add(Thread.holdsLock(service) ? "locked" : "unlocked"));++ service.clear();++ if (failure) {+ assertEquals(error, assertThrows(AslExecutor.FailStateException.class,+ () -> executor(() -> 1).awaitToken(future, mapper.readTree("{}"), "cleared", 1, 1)));+ } else {+ assertEquals(output, executor(() -> 1).awaitToken(future, mapper.readTree("{}"), "cleared", 1, 1));+ }+ assertEquals(List.of("unlocked"), completions);+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void registrationDuringResetIsRetiredUntilAllResetsFinish(boolean overlappingReset) throws Exception {+ when(resolver.getAccountId()).thenReturn(OWNER);+ CompletableFuture<JsonNode> first = service.registerPendingToken("first", OWNER);+ CountDownLatch completing = new CountDownLatch(1);+ CompletableFuture<Void> release = new CompletableFuture<>();+ List<Boolean> completionLocks = new ArrayList<>();+ first.whenComplete((value, error) -> {+ completionLocks.add(Thread.holdsLock(service));+ completing.countDown();+ release.join();+ });+ ExecutorService worker = Executors.newSingleThreadExecutor();+ try {+ Future<?> reset = worker.submit(service::clear);+ assertTrue(completing.await(5, TimeUnit.SECONDS));+ if (overlappingReset) {+ service.clear();+ }+ CompletableFuture<JsonNode> duringReset = service.registerPendingToken("during-reset", OWNER);+ assertTrue(duringReset.isCompletedExceptionally());+ assertEquals("InvalidToken", callbackResult(() -> service.sendTaskSuccess("during-reset", "{}")));+ release.complete(null);+ reset.get(5, TimeUnit.SECONDS);+ assertEquals(List.of(false), completionLocks);++ CompletableFuture<JsonNode> afterReset = service.registerPendingToken("after-reset", OWNER);+ service.sendTaskSuccess("after-reset", "{}");+ assertEquals(mapper.readTree("{}"), afterReset.get(5, TimeUnit.SECONDS));+ } finally {+ release.complete(null);+ worker.shutdownNow();+ assertTrue(worker.awaitTermination(5, TimeUnit.SECONDS));+ }+ }+private String callbackResult(Runnable callback) {try {callback.run();@@ -117,8 +246,13 @@ class AslExecutorTokenTimeoutTest {@SuppressWarnings("unchecked")private AslExecutor executor(LongSupplier clock) {+ return executor(service, clock);+ }++ @SuppressWarnings("unchecked")+ private AslExecutor executor(StepFunctionsService target, LongSupplier clock) {Instance<StepFunctionsService> instance = mock(Instance.class);- when(instance.get()).thenReturn(service);+ when(instance.get()).thenReturn(target);return AslExecutorFixture.builder().objectMapper(mapper).sfnService(instance).nanoTime(clock).build();}}