Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(stepfunctions): atomically close expired callback tokens (#6307)

Andre Kuraitd7db2276df38
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index 1f45df6bd3..47e85e5f2c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -59,6 +59,7 @@ import io.github.hectorvent.floci.services.scheduler.model.Schedule;
import io.github.hectorvent.floci.services.scheduler.model.ScheduleRequest;
import io.github.hectorvent.floci.services.sns.SnsJsonHandler;
import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;
+import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService.TokenTimeout;
import io.github.hectorvent.floci.services.stepfunctions.model.Execution;
import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;
import io.github.hectorvent.floci.services.stepfunctions.model.MapRun;
@@ -1073,7 +1074,7 @@ public class AslExecutor {
* picks the task up, which is the instant it emits ActivityStarted; Floci emits that event at
* schedule time, so there is no later instant to anchor on here.
*/
- private JsonNode awaitToken(CompletableFuture<JsonNode> future, JsonNode stateDef, String taskToken,
+ JsonNode awaitToken(CompletableFuture<JsonNode> future, JsonNode stateDef, String taskToken,
long executionDeadlineNanos, long timeoutDeadlineNanos) throws Exception {
int heartbeatSeconds = stateDef.path("HeartbeatSeconds").asInt(0);
try {
@@ -1083,22 +1084,21 @@ public class AslExecutor {
try {
return future.get(wakeAtNanos - nanoTime.getAsLong(), TimeUnit.NANOSECONDS);
} catch (java.util.concurrent.TimeoutException e) {
- // The execution's budget is read first: when it is the clock that ran out, the
- // execution ends as TIMED_OUT and the task's own timeout never applies.
- if (nanoTime.getAsLong() >= executionDeadlineNanos) {
- future.cancel(true);
- throw new ExecutionTimedOutException();
- }
- if (nanoTime.getAsLong() >= timeoutDeadlineNanos) {
- future.cancel(true);
- throw new TaskTimedOutException("States.Timeout");
+ TokenTimeout expired = sfnService.get().expirePendingToken(taskToken, nanoTime.getAsLong(),
+ executionDeadlineNanos, timeoutDeadlineNanos, heartbeatSeconds);
+ if (expired == TokenTimeout.CLAIMED) {
+ // Claiming removes the token before completing its future outside the lock.
+ return future.get();
}
- // Read the gap again rather than trusting the one this thread parked on: a
- // heartbeat that landed meanwhile has already moved it past now, and the task
- // goes back to waiting on the later deadline.
- if (nanoTime.getAsLong() >= heartbeatDeadlineNanos(taskToken, heartbeatSeconds)) {
+ RuntimeException timeout = switch (expired) {
+ case EXECUTION -> new ExecutionTimedOutException();
+ case TASK -> new TaskTimedOutException("States.Timeout");
+ case HEARTBEAT -> new TaskTimedOutException("States.HeartbeatTimeout");
+ case NONE, CLAIMED -> null;
+ };
+ if (timeout != null) {
future.cancel(true);
- throw new TaskTimedOutException("States.HeartbeatTimeout");
+ throw timeout;
}
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
index 02f6636019..d081dcb753 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
@@ -1213,6 +1213,29 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
return reportedAt != null ? reportedAt : System.nanoTime();
}
+ enum TokenTimeout { NONE, CLAIMED, EXECUTION, TASK, HEARTBEAT }
+
+ /** Selects an expired clock and closes its token under the callback and heartbeat lock. */
+ synchronized TokenTimeout expirePendingToken(String taskToken, long now, long executionDeadlineNanos,
+ long timeoutDeadlineNanos, int heartbeatSeconds) {
+ if (!pendingTaskTokens.containsKey(taskToken)) {
+ return TokenTimeout.CLAIMED;
+ }
+ TokenTimeout expired;
+ if (now >= executionDeadlineNanos) {
+ expired = TokenTimeout.EXECUTION;
+ } else if (now >= timeoutDeadlineNanos) {
+ expired = TokenTimeout.TASK;
+ } else if (heartbeatSeconds > 0 && now >= lastTaskHeartbeatNanos(taskToken)
+ + TimeUnit.SECONDS.toNanos(heartbeatSeconds)) {
+ expired = TokenTimeout.HEARTBEAT;
+ } else {
+ return TokenTimeout.NONE;
+ }
+ discardPendingToken(taskToken);
+ return expired;
+ }
+
/** Retires a token when its task stops waiting, including timeout and resource failure. */
public synchronized void discardPendingToken(String taskToken) {
if (taskToken != null) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.java
new file mode 100644
index 0000000000..dbd15a68ca
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorTokenTimeoutTest.java
@@ -0,0 +1,124 @@
+package io.github.hectorvent.floci.services.stepfunctions;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import jakarta.enterprise.inject.Instance;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.LongSupplier;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@Timeout(10)
+class AslExecutorTokenTimeoutTest {
+
+ private static final String OWNER = "111111111111";
+ private final ObjectMapper mapper = new ObjectMapper();
+ private final RegionResolver resolver = mock(RegionResolver.class);
+ private final StepFunctionsService service = new StepFunctionsService(mock(StorageFactory.class),
+ resolver, mock(AslExecutor.class), mapper, mock(SfnMockLoader.class),
+ mock(StepFunctionsEventPublisher.class));
+
+ @ParameterizedTest
+ @ValueSource(strings = {"execution", "task", "heartbeat"})
+ void timeoutClosesTokenBeforeCancellingFuture(String clock) throws Exception {
+ when(resolver.getAccountId()).thenReturn(OWNER);
+ CompletableFuture<JsonNode> future = service.registerPendingToken("expired", OWNER);
+ long now = service.lastTaskHeartbeatNanos("expired") + TimeUnit.SECONDS.toNanos(2);
+ List<String> callbackResults = new ArrayList<>();
+ future.whenComplete((result, error) -> {
+ callbackResults.add(Thread.holdsLock(service) ? "locked" : "unlocked");
+ callbackResults.add(callbackResult(() -> service.sendTaskSuccess("expired", "{}")));
+ callbackResults.add(callbackResult(() -> service.sendTaskFailure("expired", "cause", "error")));
+ callbackResults.add(callbackResult(() -> service.sendTaskHeartbeat("expired")));
+ });
+ long executionDeadline = "execution".equals(clock) ? now : Long.MAX_VALUE;
+ long taskDeadline = "task".equals(clock) ? now : Long.MAX_VALUE;
+ JsonNode definition = mapper.readTree("heartbeat".equals(clock) ? "{\"HeartbeatSeconds\":1}" : "{}");
+
+ RuntimeException failure = assertThrows(RuntimeException.class,
+ () -> executor(() -> now).awaitToken(future, definition, "expired",
+ executionDeadline, taskDeadline));
+
+ if ("execution".equals(clock)) {
+ assertTrue(failure instanceof AslExecutor.ExecutionTimedOutException);
+ } else {
+ assertTrue(failure instanceof AslExecutor.TaskTimedOutException);
+ }
+ assertTrue(future.isCancelled());
+ assertEquals(List.of("unlocked", "TaskTimedOut", "TaskTimedOut", "TaskTimedOut"), callbackResults);
+ }
+
+ @Test
+ void callbackClaimedBeforeTimeoutDecisionKeepsItsResult() throws Exception {
+ when(resolver.getAccountId()).thenReturn(OWNER);
+ CompletableFuture<JsonNode> future = service.registerPendingToken("winner", OWNER);
+ AtomicInteger clockReads = new AtomicInteger();
+ AslExecutor executor = executor(() -> {
+ if (clockReads.incrementAndGet() == 2) {
+ service.sendTaskSuccess("winner", "{\"accepted\":true}");
+ }
+ return 1;
+ });
+
+ JsonNode result = executor.awaitToken(future, mapper.readTree("{}"), "winner", Long.MAX_VALUE, 1);
+
+ assertTrue(result.path("accepted").asBoolean());
+ assertFalse(future.isCancelled());
+ }
+
+ @Test
+ void heartbeatBeforeTimeoutDecisionExtendsTheWait() throws Exception {
+ when(resolver.getAccountId()).thenReturn(OWNER);
+ CompletableFuture<JsonNode> future = service.registerPendingToken("heartbeat", OWNER);
+ long now = service.lastTaskHeartbeatNanos("heartbeat") + TimeUnit.SECONDS.toNanos(1);
+ AtomicInteger clockReads = new AtomicInteger();
+ AslExecutor executor = executor(() -> {
+ int read = clockReads.incrementAndGet();
+ if (read == 2) {
+ service.sendTaskHeartbeat("heartbeat");
+ } else if (read == 3) {
+ service.sendTaskSuccess("heartbeat", "{\"accepted\":true}");
+ }
+ return now;
+ });
+
+ JsonNode result = executor.awaitToken(future, mapper.readTree("{\"HeartbeatSeconds\":1}"),
+ "heartbeat", Long.MAX_VALUE, Long.MAX_VALUE);
+
+ assertTrue(result.path("accepted").asBoolean());
+ assertFalse(future.isCancelled());
+ }
+
+ private String callbackResult(Runnable callback) {
+ try {
+ callback.run();
+ return "accepted";
+ } catch (AwsException error) {
+ return error.getErrorCode();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private AslExecutor executor(LongSupplier clock) {
+ Instance<StepFunctionsService> instance = mock(Instance.class);
+ when(instance.get()).thenReturn(service);
+ return AslExecutorFixture.builder().objectMapper(mapper).sfnService(instance).nanoTime(clock).build();
+ }
+}