Light, fluffy, and always free - The AWS Local Emulator alternative
fix(stepfunctions): atomically close expired callback tokens (#6307)
Andre Kuraitd7db2276df38
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.javaindex 1f45df6bd3..47e85e5f2c 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java@@ -59,6 +59,7 @@ import io.github.hectorvent.floci.services.scheduler.model.Schedule;import io.github.hectorvent.floci.services.scheduler.model.ScheduleRequest;import io.github.hectorvent.floci.services.sns.SnsJsonHandler;import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;+import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService.TokenTimeout;import io.github.hectorvent.floci.services.stepfunctions.model.Execution;import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;import io.github.hectorvent.floci.services.stepfunctions.model.MapRun;@@ -1073,7 +1074,7 @@ public class AslExecutor {* picks the task up, which is the instant it emits ActivityStarted; Floci emits that event at* schedule time, so there is no later instant to anchor on here.*/- private JsonNode awaitToken(CompletableFuture<JsonNode> future, JsonNode stateDef, String taskToken,+ JsonNode awaitToken(CompletableFuture<JsonNode> future, JsonNode stateDef, String taskToken,long executionDeadlineNanos, long timeoutDeadlineNanos) throws Exception {int heartbeatSeconds = stateDef.path("HeartbeatSeconds").asInt(0);try {@@ -1083,22 +1084,21 @@ public class AslExecutor {try {return future.get(wakeAtNanos - nanoTime.getAsLong(), TimeUnit.NANOSECONDS);} catch (java.util.concurrent.TimeoutException e) {- // The execution's budget is read first: when it is the clock that ran out, the- // execution ends as TIMED_OUT and the task's own timeout never applies.- if (nanoTime.getAsLong() >= executionDeadlineNanos) {- future.cancel(true);- throw new ExecutionTimedOutException();- }- if (nanoTime.getAsLong() >= timeoutDeadlineNanos) {- future.cancel(true);- throw new TaskTimedOutException("States.Timeout");+ TokenTimeout expired = sfnService.get().expirePendingToken(taskToken, nanoTime.getAsLong(),+ executionDeadlineNanos, timeoutDeadlineNanos, heartbeatSeconds);+ if (expired == TokenTimeout.CLAIMED) {+ // Claiming removes the token before completing its future outside the lock.+ return future.get();}- // Read the gap again rather than trusting the one this thread parked on: a- // heartbeat that landed meanwhile has already moved it past now, and the task- // goes back to waiting on the later deadline.- if (nanoTime.getAsLong() >= heartbeatDeadlineNanos(taskToken, heartbeatSeconds)) {+ RuntimeException timeout = switch (expired) {+ case EXECUTION -> new ExecutionTimedOutException();+ case TASK -> new TaskTimedOutException("States.Timeout");+ case HEARTBEAT -> new TaskTimedOutException("States.HeartbeatTimeout");+ case NONE, CLAIMED -> null;+ };+ if (timeout != null) {future.cancel(true);- throw new TaskTimedOutException("States.HeartbeatTimeout");+ throw timeout;}}}diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.javaindex 02f6636019..d081dcb753 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java@@ -1213,6 +1213,29 @@ public class StepFunctionsService implements Resettable, ResourceProvider {return reportedAt != null ? reportedAt : System.nanoTime();}+ enum TokenTimeout { NONE, CLAIMED, EXECUTION, TASK, HEARTBEAT }++ /** Selects an expired clock and closes its token under the callback and heartbeat lock. */+ synchronized TokenTimeout expirePendingToken(String taskToken, long now, long executionDeadlineNanos,+ long timeoutDeadlineNanos, int heartbeatSeconds) {+ if (!pendingTaskTokens.containsKey(taskToken)) {+ return TokenTimeout.CLAIMED;+ }+ TokenTimeout expired;+ if (now >= executionDeadlineNanos) {+ expired = TokenTimeout.EXECUTION;+ } else if (now >= timeoutDeadlineNanos) {+ expired = TokenTimeout.TASK;+ } else if (heartbeatSeconds > 0 && now >= lastTaskHeartbeatNanos(taskToken)+ + TimeUnit.SECONDS.toNanos(heartbeatSeconds)) {+ expired = TokenTimeout.HEARTBEAT;+ } else {+ return TokenTimeout.NONE;+ }+ discardPendingToken(taskToken);+ return expired;+ }+/** Retires a token when its task stops waiting, including timeout and resource failure. */public synchronized void discardPendingToken(String taskToken) {if (taskToken != null) {diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.javanew file mode 100644index 0000000000..dbd15a68ca--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.java@@ -0,0 +1,124 @@+package io.github.hectorvent.floci.services.stepfunctions;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import jakarta.enterprise.inject.Instance;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.function.LongSupplier;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++@Timeout(10)+class AslExecutorTokenTimeoutTest {++ private static final String OWNER = "111111111111";+ private final ObjectMapper mapper = new ObjectMapper();+ private final RegionResolver resolver = mock(RegionResolver.class);+ private final StepFunctionsService service = new StepFunctionsService(mock(StorageFactory.class),+ resolver, mock(AslExecutor.class), mapper, mock(SfnMockLoader.class),+ mock(StepFunctionsEventPublisher.class));++ @ParameterizedTest+ @ValueSource(strings = {"execution", "task", "heartbeat"})+ void timeoutClosesTokenBeforeCancellingFuture(String clock) throws Exception {+ when(resolver.getAccountId()).thenReturn(OWNER);+ CompletableFuture<JsonNode> future = service.registerPendingToken("expired", OWNER);+ long now = service.lastTaskHeartbeatNanos("expired") + TimeUnit.SECONDS.toNanos(2);+ List<String> callbackResults = new ArrayList<>();+ future.whenComplete((result, error) -> {+ callbackResults.add(Thread.holdsLock(service) ? "locked" : "unlocked");+ callbackResults.add(callbackResult(() -> service.sendTaskSuccess("expired", "{}")));+ callbackResults.add(callbackResult(() -> service.sendTaskFailure("expired", "cause", "error")));+ callbackResults.add(callbackResult(() -> service.sendTaskHeartbeat("expired")));+ });+ long executionDeadline = "execution".equals(clock) ? now : Long.MAX_VALUE;+ long taskDeadline = "task".equals(clock) ? now : Long.MAX_VALUE;+ JsonNode definition = mapper.readTree("heartbeat".equals(clock) ? "{\"HeartbeatSeconds\":1}" : "{}");++ RuntimeException failure = assertThrows(RuntimeException.class,+ () -> executor(() -> now).awaitToken(future, definition, "expired",+ executionDeadline, taskDeadline));++ if ("execution".equals(clock)) {+ assertTrue(failure instanceof AslExecutor.ExecutionTimedOutException);+ } else {+ assertTrue(failure instanceof AslExecutor.TaskTimedOutException);+ }+ assertTrue(future.isCancelled());+ assertEquals(List.of("unlocked", "TaskTimedOut", "TaskTimedOut", "TaskTimedOut"), callbackResults);+ }++ @Test+ void callbackClaimedBeforeTimeoutDecisionKeepsItsResult() throws Exception {+ when(resolver.getAccountId()).thenReturn(OWNER);+ CompletableFuture<JsonNode> future = service.registerPendingToken("winner", OWNER);+ AtomicInteger clockReads = new AtomicInteger();+ AslExecutor executor = executor(() -> {+ if (clockReads.incrementAndGet() == 2) {+ service.sendTaskSuccess("winner", "{\"accepted\":true}");+ }+ return 1;+ });++ JsonNode result = executor.awaitToken(future, mapper.readTree("{}"), "winner", Long.MAX_VALUE, 1);++ assertTrue(result.path("accepted").asBoolean());+ assertFalse(future.isCancelled());+ }++ @Test+ void heartbeatBeforeTimeoutDecisionExtendsTheWait() throws Exception {+ when(resolver.getAccountId()).thenReturn(OWNER);+ CompletableFuture<JsonNode> future = service.registerPendingToken("heartbeat", OWNER);+ long now = service.lastTaskHeartbeatNanos("heartbeat") + TimeUnit.SECONDS.toNanos(1);+ AtomicInteger clockReads = new AtomicInteger();+ AslExecutor executor = executor(() -> {+ int read = clockReads.incrementAndGet();+ if (read == 2) {+ service.sendTaskHeartbeat("heartbeat");+ } else if (read == 3) {+ service.sendTaskSuccess("heartbeat", "{\"accepted\":true}");+ }+ return now;+ });++ JsonNode result = executor.awaitToken(future, mapper.readTree("{\"HeartbeatSeconds\":1}"),+ "heartbeat", Long.MAX_VALUE, Long.MAX_VALUE);++ assertTrue(result.path("accepted").asBoolean());+ assertFalse(future.isCancelled());+ }++ private String callbackResult(Runnable callback) {+ try {+ callback.run();+ return "accepted";+ } catch (AwsException error) {+ return error.getErrorCode();+ }+ }++ @SuppressWarnings("unchecked")+ private AslExecutor executor(LongSupplier clock) {+ Instance<StepFunctionsService> instance = mock(Instance.class);+ when(instance.get()).thenReturn(service);+ return AslExecutorFixture.builder().objectMapper(mapper).sfnService(instance).nanoTime(clock).build();+ }+}