Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

perf(stepfunctions): wake nested waits on execution status changes (#6513)

Andre Kuraitfe928d67ab23
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index cc43c15caa..96a77a5757 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -62,6 +62,7 @@ import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;
import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService.TokenTimeout;
import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService.TokenWait;
import io.github.hectorvent.floci.services.stepfunctions.model.Execution;
+import io.github.hectorvent.floci.services.stepfunctions.model.ExecutionStatusSubscription;
import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;
import io.github.hectorvent.floci.services.stepfunctions.model.MapRun;
import io.github.hectorvent.floci.services.stepfunctions.model.MockedResponseStep;
@@ -278,12 +279,15 @@ public class AslExecutor {
private final CustomResourceLiveness customResourceLiveness;
private final Clock clock;
private final Sleeper sleeper;
+ private final NestedWaiter nestedWaiter;
// Null in production, where the ceiling comes from the Step Functions config. Tests pin it.
private final Integer maxWaitSecondsOverride;
private final Duration expressMaxExecutionTime;
// The monotonic clock every execution and task deadline is measured on. Tests move it forward.
private final LongSupplier nanoTime;
private final Map<String, ActiveMockExecution> activeMocks = new ConcurrentHashMap<>();
+ private final Set<ExecutionStatusSubscription> nestedWaits = new HashSet<>();
+ private long nestedWaitGeneration;
private final ExecutorService executor = Executors.newCachedThreadPool(r -> {
Thread t = new Thread(r, "sfn-executor");
t.setDaemon(true);
@@ -312,7 +316,7 @@ public class AslExecutor {
objectMapper, jsonataEvaluator, sfnService, config, vertx, customResourceLiveness,
Clock.systemUTC(), TimeUnit.NANOSECONDS::sleep, null, redshiftDataHandler, serviceRegistry,
nestedTaskCredentials, lambdaTaskAuthorizer, null, null, apiGatewayTaskInvoker, emrTaskInvoker,
- emrServerlessTaskInvoker);
+ emrServerlessTaskInvoker, null);
}
AslExecutor(LambdaExecutorService lambdaExecutor, LambdaTargetResolver targetResolver,
@@ -331,7 +335,7 @@ public class AslExecutor {
NestedTaskCredentials nestedTaskCredentials, LambdaTaskAuthorizer lambdaTaskAuthorizer,
Duration expressMaxExecutionTimeOverride, LongSupplier nanoTimeOverride,
ApiGatewayTaskInvoker apiGatewayTaskInvoker, EmrTaskInvoker emrTaskInvoker,
- EmrServerlessTaskInvoker emrServerlessTaskInvoker) {
+ EmrServerlessTaskInvoker emrServerlessTaskInvoker, NestedWaiter nestedWaiter) {
this.apiGatewayTaskInvoker = apiGatewayTaskInvoker;
this.emrTaskInvoker = emrTaskInvoker;
this.emrServerlessTaskInvoker = emrServerlessTaskInvoker;
@@ -367,6 +371,7 @@ public class AslExecutor {
this.config = config;
this.clock = clock;
this.sleeper = sleeper;
+ this.nestedWaiter = nestedWaiter != null ? nestedWaiter : ExecutionStatusSubscription::awaitChange;
this.maxWaitSecondsOverride = maxWaitSecondsOverride;
this.nanoTime = nanoTimeOverride != null ? nanoTimeOverride : System::nanoTime;
this.expressMaxExecutionTime = expressMaxExecutionTimeOverride != null
@@ -398,7 +403,7 @@ public class AslExecutor {
eventBridgeHandler, schedulerService, schedulerController, rdsDataService,
objectMapper, jsonataEvaluator, sfnService, config, vertx, customResourceLiveness,
Clock.systemUTC(), TimeUnit.NANOSECONDS::sleep, null, null, null, null, null, null, null, null, null,
- null);
+ null, null);
}
// Kept for tests that do not exercise the Redshift Data API integration.
@@ -419,7 +424,7 @@ public class AslExecutor {
ec2Service, s3Service, ecsService, ecsJsonHandler,
eventBridgeHandler, schedulerService, schedulerController, rdsDataService,
objectMapper, jsonataEvaluator, sfnService, config, vertx, customResourceLiveness,
- clock, sleeper, maxWaitSecondsOverride, null, null, null, null, null, null, null, null, null);
+ clock, sleeper, maxWaitSecondsOverride, null, null, null, null, null, null, null, null, null, null);
}
AslExecutor(LambdaExecutorService lambdaExecutor, LambdaFunctionStore functionStore,
@@ -441,6 +446,13 @@ public class AslExecutor {
vertx, customResourceLiveness, clock, sleeper, maxWaitSecondsOverride);
}
+ /** Test seam paired with the monotonic clock for nested deadline and cut coordination. */
+ @FunctionalInterface
+ interface NestedWaiter {
+ void awaitChange(ExecutionStatusSubscription subscription, long observed, long timeoutNanos)
+ throws InterruptedException;
+ }
+
/** Test seam: lets Wait states be exercised without real time passing. */
@FunctionalInterface
interface Sleeper {
@@ -1925,6 +1937,7 @@ public class AslExecutor {
long executionDeadlineNanos, long taskDeadlineNanos,
JsonNode rawParameters, Consumer<JsonNode> jobSubmitted)
throws Exception {
+ long waitGeneration = nestedWaitGeneration();
String smArn = input.path("StateMachineArn").asText(null);
if (smArn == null || smArn.isBlank()) {
throw new FailStateException("States.TaskFailed",
@@ -1951,7 +1964,7 @@ public class AslExecutor {
childName = null;
}
- io.github.hectorvent.floci.services.stepfunctions.model.Execution exec;
+ Execution exec;
try {
exec = sfnService.get().startExecution(smArn, childName, childInput, region);
} catch (AwsException e) {
@@ -1981,44 +1994,98 @@ public class AslExecutor {
throw e;
}
- // .sync or .sync:2 polls until terminal. Whatever else ends the wait aborts the child, the way
+ // .sync or .sync:2 waits for terminal publication. Whatever else ends the wait aborts the child, the way
// AWS does (measured: child ABORTED, no error, the cause below): the Task's own TimeoutSeconds,
// which fails the state with States.Timeout; the parent execution's budget, which ends it
// TIMED_OUT; a failure in a sibling Parallel branch, which interrupts this one; and a
- // StopExecution on the parent, so a stopped execution does not keep a polling worker.
- while (true) {
- try {
- sleepOrTimeOutTask(TimeUnit.MILLISECONDS.toNanos(SYNC_POLL_INTERVAL_MS),
- executionDeadlineNanos, taskDeadlineNanos);
- } catch (TaskTimedOutException | ExecutionTimedOutException | InterruptedException e) {
- abortChildExecution(execArn, executionArn);
- throw e;
- }
- if (abortedByCaller(executionArn)) {
- abortChildExecution(execArn, executionArn);
- throw new ExecutionAbortedException();
+ // StopExecution on the parent. Register before reading either execution to cover
+ // completion during registration and between a state read and parking.
+ ExecutionStatusSubscription waiting = new ExecutionStatusSubscription(exec, nestedWaitParent(executionArn));
+ registerNestedWait(waiting, waitGeneration);
+ try (waiting) {
+ while (true) {
+ if (waiting.cancelled()) {
+ abortChildExecution(execArn, executionArn);
+ throw new FailStateException("States.TaskFailed", "StepFunctionsService cleared");
+ }
+ long observed = waiting.version();
+ checkNestedTaskDeadline(executionDeadlineNanos, taskDeadlineNanos);
+ if (abortedByCaller(executionArn)) {
+ abortChildExecution(execArn, executionArn);
+ throw new ExecutionAbortedException();
+ }
+ Execution current = sfnService.get().describeExecution(execArn);
+ String status;
+ // Terminal fields are written under this monitor, status last.
+ synchronized (current) {
+ status = current.getStatus();
+ }
+ if ("SUCCEEDED".equals(status)) {
+ return nestedExecutionEnvelope(current, true, ".sync:2".equals(mode));
+ }
+ if (!"RUNNING".equals(status)) {
+ throw new FailStateException("States.TaskFailed",
+ nestedExecutionEnvelope(current, false, false).toString());
+ }
+ nestedWaiter.awaitChange(waiting, observed,
+ Math.min(executionDeadlineNanos, taskDeadlineNanos) - nanoTime.getAsLong());
}
- io.github.hectorvent.floci.services.stepfunctions.model.Execution current =
- sfnService.get().describeExecution(execArn);
- String status;
- // The child's worker and StopExecution write its terminal fields under this monitor,
- // status last. Reading the status under it makes everything written before it visible,
- // so a terminal status is never seen without the error, cause and stop date behind it.
- synchronized (current) {
- status = current.getStatus();
+ } catch (TaskTimedOutException | ExecutionTimedOutException | InterruptedException e) {
+ abortChildExecution(execArn, executionArn);
+ throw e;
+ } finally {
+ synchronized (nestedWaits) {
+ nestedWaits.remove(waiting);
}
- if ("RUNNING".equals(status)) {
- continue;
+ }
+ }
+
+ private long nestedWaitGeneration() {
+ synchronized (nestedWaits) {
+ return nestedWaitGeneration;
+ }
+ }
+
+ private void registerNestedWait(ExecutionStatusSubscription waiting, long generation) {
+ synchronized (nestedWaits) {
+ if (generation != nestedWaitGeneration) {
+ waiting.cancel();
+ } else {
+ nestedWaits.add(waiting);
}
- if ("SUCCEEDED".equals(status)) {
- // Both modes return the envelope (measured); .sync:2 differs only in carrying
- // Input and Output as JSON values where .sync carries them as JSON strings.
- return nestedExecutionEnvelope(current, true, ".sync:2".equals(mode));
+ }
+ }
+
+ void clearNestedExecutionWaits() {
+ synchronized (nestedWaits) {
+ nestedWaitGeneration++;
+ for (ExecutionStatusSubscription waiting : nestedWaits) {
+ waiting.cancel();
}
- // However the child ended, FAILED, TIMED_OUT or ABORTED, and whatever its own error, the
- // parent sees States.TaskFailed (measured), so a Catch on the child's error never fires.
- throw new FailStateException("States.TaskFailed",
- nestedExecutionEnvelope(current, false, false).toString());
+ nestedWaits.clear();
+ }
+ }
+
+ private Execution nestedWaitParent(String executionArn) {
+ if (executionArn == null) {
+ return null;
+ }
+ try {
+ return RequestScopes.callAs(AwsArnUtils.parse(executionArn).accountId(),
+ () -> sfnService.get().describeExecution(executionArn));
+ } catch (AwsException e) {
+ LOG.warnv("Parent execution {0} vanished before its nested wait: {1}", executionArn, e.getMessage());
+ return null;
+ }
+ }
+
+ private void checkNestedTaskDeadline(long executionDeadlineNanos, long taskDeadlineNanos) {
+ long now = nanoTime.getAsLong();
+ if (now >= executionDeadlineNanos) {
+ throw new ExecutionTimedOutException();
+ }
+ if (now >= taskDeadlineNanos) {
+ throw new TaskTimedOutException("States.Timeout");
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
index b647e4b083..142849dece 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
@@ -176,6 +176,9 @@ public class StepFunctionsService implements Resettable, ResourceProvider {
}
public void clear() {
+ if (aslExecutor != null) {
+ aslExecutor.clearNestedExecutionWaits();
+ }
historyCache.clear();
activityQueues.clear();
Map<String, PendingTask> pending;
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java
index 77d4b2b25c..7fecbc7c6e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java
@@ -1,10 +1,13 @@
package io.github.hectorvent.floci.services.stepfunctions.model;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
@RegisterForReflection
@JsonIgnoreProperties(ignoreUnknown = true)
@@ -22,6 +25,8 @@ public class Execution {
private String error;
private String cause;
private List<HistoryEvent> history = new ArrayList<>();
+ @JsonIgnore
+ private transient Set<ExecutionStatusSubscription> statusSubscriptions;
public Execution() {
this.startDate = System.currentTimeMillis() / 1000.0;
@@ -47,7 +52,36 @@ public class Execution {
public void setName(String name) { this.name = name; }
public String getStatus() { return status; }
- public void setStatus(String status) { this.status = status; }
+ public synchronized void setStatus(String status) {
+ this.status = status;
+ if (!"RUNNING".equals(status) && statusSubscriptions != null) {
+ Set<ExecutionStatusSubscription> subscriptions = statusSubscriptions;
+ statusSubscriptions = null;
+ for (ExecutionStatusSubscription subscription : subscriptions) {
+ subscription.signal();
+ }
+ }
+ }
+
+ synchronized void registerStatusSubscription(ExecutionStatusSubscription subscription) {
+ if (!"RUNNING".equals(status)) {
+ subscription.signal();
+ return;
+ }
+ if (statusSubscriptions == null) {
+ statusSubscriptions = new HashSet<>();
+ }
+ statusSubscriptions.add(subscription);
+ }
+
+ synchronized void unregisterStatusSubscription(ExecutionStatusSubscription subscription) {
+ if (statusSubscriptions != null) {
+ statusSubscriptions.remove(subscription);
+ if (statusSubscriptions.isEmpty()) {
+ statusSubscriptions = null;
+ }
+ }
+ }
public String getInput() { return input; }
public void setInput(String input) { this.input = input; }
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/ExecutionStatusSubscription.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/ExecutionStatusSubscription.java
new file mode 100644
index 0000000000..b59e56d8c8
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/ExecutionStatusSubscription.java
@@ -0,0 +1,70 @@
+package io.github.hectorvent.floci.services.stepfunctions.model;
+
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A scoped wakeup for a child execution ending or its parent being stopped.
+ * Read the version before authoritative state so completion cannot be lost
+ * between that read and parking. Closing removes both registrations.
+ */
+public final class ExecutionStatusSubscription implements AutoCloseable {
+ private final Execution child;
+ private final Execution parent;
+ private long version;
+ private boolean closed;
+ private boolean cancelled;
+
+ public ExecutionStatusSubscription(Execution child, Execution parent) {
+ this.child = Objects.requireNonNull(child);
+ this.parent = parent == child ? null : parent;
+ child.registerStatusSubscription(this);
+ if (this.parent != null) {
+ this.parent.registerStatusSubscription(this);
+ }
+ }
+
+ public synchronized long version() {
+ return version;
+ }
+
+ synchronized void signal() {
+ version++;
+ notifyAll();
+ }
+
+ public synchronized void cancel() {
+ cancelled = true;
+ notifyAll();
+ }
+
+ public synchronized boolean cancelled() {
+ return cancelled;
+ }
+
+ public synchronized boolean awaitChange(long observed, long timeoutNanos) throws InterruptedException {
+ // The executor owns the deadline and its clock. Recheck there after every wake,
+ // including a spurious wake, rather than creating a second real-clock deadline.
+ if (!closed && !cancelled && version == observed && timeoutNanos > 0) {
+ TimeUnit.NANOSECONDS.timedWait(this, timeoutNanos);
+ }
+ return closed || cancelled || version != observed;
+ }
+
+ @Override
+ public void close() {
+ synchronized (this) {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ notifyAll();
+ }
+ // setStatus holds an execution monitor while signalling. Do not take
+ // an execution monitor while holding this subscription's monitor.
+ child.unregisterStatusSubscription(this);
+ if (parent != null) {
+ parent.unregisterStatusSubscription(this);
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorFixture.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorFixture.java
index 884faff3ac..ea33f0a894 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorFixture.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorFixture.java
@@ -119,6 +119,7 @@ final class AslExecutorFixture {
private LambdaTaskAuthorizer lambdaTaskAuthorizer;
private Duration expressMaxExecutionTime;
private LongSupplier nanoTime;
+ private AslExecutor.NestedWaiter nestedWaiter;
private EmrTaskInvoker emrTaskInvoker;
private EmrServerlessTaskInvoker emrServerlessTaskInvoker;
@@ -270,6 +271,11 @@ final class AslExecutorFixture {
return this;
}
+ Builder nestedWaiter(AslExecutor.NestedWaiter nestedWaiter) {
+ this.nestedWaiter = nestedWaiter;
+ return this;
+ }
+
Builder nanoTime(LongSupplier nanoTime) {
this.nanoTime = nanoTime;
return this;
@@ -294,7 +300,7 @@ final class AslExecutorFixture {
jsonataEvaluator != null ? jsonataEvaluator : new JsonataEvaluator(objectMapper),
sfnService, config, vertx, customResourceLiveness, clock, sleeper, maxWaitSeconds,
redshiftDataHandler, serviceRegistry, null, lambdaTaskAuthorizer, expressMaxExecutionTime,
- nanoTime, null, emrTaskInvoker, emrServerlessTaskInvoker);
+ nanoTime, null, emrTaskInvoker, emrServerlessTaskInvoker, nestedWaiter);
}
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.java
index 3c17e349b7..86acc0062b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.java
@@ -3,25 +3,39 @@ package io.github.hectorvent.floci.services.stepfunctions;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.services.stepfunctions.model.Execution;
+import io.github.hectorvent.floci.services.stepfunctions.model.ExecutionStatusSubscription;
import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;
import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;
import jakarta.enterprise.inject.Instance;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
-import java.util.ArrayList;
+import java.time.Duration;
import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.LongSupplier;
+import static org.awaitility.Awaitility.await;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -48,6 +62,7 @@ class AslExecutorNestedSyncTimeoutTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private StepFunctionsService sfnService;
private List<HistoryEvent> history;
+ private AslExecutor.NestedWaiter nestedWaiter;
@BeforeEach
void setUp() {
@@ -86,23 +101,24 @@ class AslExecutorNestedSyncTimeoutTest {
verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());
}
- @Test
- void syncIsNotCappedByAPollCountWhenNoTimeoutSecondsIsDeclared() throws Exception {
- AtomicInteger polls = new AtomicInteger();
+ @ParameterizedTest
+ @ValueSource(strings = {".sync", ".sync:2"})
+ void syncCompletionDoesNotDependOnPolling(String mode) throws Exception {
+ AtomicLong nanos = new AtomicLong();
when(sfnService.describeExecution(CHILD_ARN)).thenAnswer(invocation -> {
- if (polls.incrementAndGet() < 700) {
- return child("RUNNING");
- }
+ nanos.addAndGet(TimeUnit.MILLISECONDS.toNanos(350));
Execution done = child("SUCCEEDED");
done.setOutput("{\"answer\":42}");
return done;
});
- Execution execution = run(newExecutor(nanos -> { }), ".sync", 0, false);
+ Execution execution = run(newExecutor(pause -> {
+ throw new AssertionError("Nested execution completion must not use a polling sleep");
+ }, nanos::get), mode, 0, false, 0, true);
assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());
- assertEquals(700, polls.get());
- assertTrue(objectMapper.readTree(execution.getOutput()).path("Output").asText().contains("42"));
+ assertEquals(TimeUnit.MILLISECONDS.toNanos(350), nanos.get());
+ assertTrue(objectMapper.readTree(execution.getOutput()).path("Output").toString().contains("42"));
verify(sfnService, never()).stopExecution(any(), any(), any());
}
@@ -120,18 +136,19 @@ class AslExecutorNestedSyncTimeoutTest {
@Test
void stopExecutionOnTheParentEndsTheWaitAndAbortsTheChild() {
- AtomicInteger parentReads = new AtomicInteger();
- when(sfnService.describeExecution(PARENT_ARN))
- .thenAnswer(invocation -> parent(parentReads.incrementAndGet() < 3 ? "RUNNING" : "ABORTED"));
+ Execution parent = parent("RUNNING");
+ when(sfnService.describeExecution(PARENT_ARN)).thenReturn(parent);
AtomicInteger childPolls = new AtomicInteger();
when(sfnService.describeExecution(CHILD_ARN)).thenAnswer(invocation -> {
childPolls.incrementAndGet();
+ parent.setStatus("ABORTED");
return child("RUNNING");
});
- run(newExecutor(nanos -> { }), ".sync", 0, false);
+ run(newExecutor(nanos -> { throw new AssertionError("Parent cancellation must notify the wait"); }),
+ ".sync", 0, false);
- assertEquals(2, childPolls.get(), "polling stops as soon as the parent reads ABORTED");
+ assertEquals(1, childPolls.get(), "the parent cancellation wakes the wait without rereading its child");
verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());
List<String> types = history.stream().map(HistoryEvent::getType).toList();
assertTrue(types.stream().noneMatch(type -> type.equals("TaskTimedOut")
@@ -139,6 +156,186 @@ class AslExecutorNestedSyncTimeoutTest {
&& !type.equals("ExecutionStarted")), "an aborted execution gets no terminal event from the worker: " + types);
}
+ @ParameterizedTest
+ @ValueSource(strings = {".sync", ".sync:2"})
+ void childCompletionWakesAParkedParentWithoutPolling(String mode) throws Exception {
+ Execution live = child("RUNNING");
+ when(sfnService.startExecution(any(), any(), any(), any())).thenReturn(live);
+ CountDownLatch firstRead = new CountDownLatch(1);
+ AtomicReference<Thread> waitingThread = new AtomicReference<>();
+ when(sfnService.describeExecution(CHILD_ARN)).thenAnswer(invocation -> {
+ waitingThread.set(Thread.currentThread());
+ firstRead.countDown();
+ return live;
+ });
+ AslExecutor executor = newExecutor(nanos -> {
+ throw new AssertionError("Nested completion must not use the polling sleeper");
+ });
+ CompletableFuture<Execution> result = CompletableFuture.supplyAsync(
+ () -> run(executor, mode, 60, false));
+ try {
+ assertTrue(firstRead.await(2, TimeUnit.SECONDS));
+ await().atMost(Duration.ofSeconds(2))
+ .until(() -> waitingThread.get().getState() == Thread.State.TIMED_WAITING);
+ verify(sfnService, times(1)).describeExecution(CHILD_ARN);
+ synchronized (live) {
+ live.setOutput("{\"answer\":42}");
+ live.setStopDate(2.0);
+ live.setStatus("SUCCEEDED");
+ }
+
+ Execution execution = result.get(2, TimeUnit.SECONDS);
+ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());
+ assertTrue(objectMapper.readTree(execution.getOutput()).path("Output").toString().contains("42"));
+ verify(sfnService, times(2)).describeExecution(CHILD_ARN);
+ verify(sfnService, never()).stopExecution(any(), any(), any());
+ } finally {
+ if (!result.isDone() && waitingThread.get() != null) {
+ waitingThread.get().interrupt();
+ }
+ }
+ }
+
+ @Test
+ void clearCancelsAnActiveNestedWaitAndAbortsItsChild() {
+ AslExecutor executor = newExecutor(nanos -> {
+ throw new AssertionError("Reset must notify the wait");
+ });
+ when(sfnService.describeExecution(CHILD_ARN)).thenAnswer(invocation -> {
+ executor.clearNestedExecutionWaits();
+ return child("RUNNING");
+ });
+
+ Execution execution = run(executor, ".sync", 0, false);
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.TaskFailed", execution.getError());
+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());
+ }
+
+ @Test
+ void clearDuringChildStartCannotLeaveAnUnregisteredWait() {
+ AslExecutor executor = newExecutor(nanos -> {
+ throw new AssertionError("Reset during submission must not start polling");
+ });
+ when(sfnService.startExecution(any(), any(), any(), any())).thenAnswer(invocation -> {
+ executor.clearNestedExecutionWaits();
+ return child("RUNNING");
+ });
+
+ Execution execution = run(executor, ".sync:2", 0, false);
+
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.TaskFailed", execution.getError());
+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {".sync", ".sync:2"})
+ void clearDuringInputEncodingCancelsTheNestedWait(String mode) throws Exception {
+ ObjectMapper mapper = spy(objectMapper);
+ AslExecutor executor = newExecutor(nanos -> {
+ throw new AssertionError("Reset during input encoding must not start polling");
+ }, System::nanoTime, mapper);
+ AtomicInteger resets = new AtomicInteger();
+ JsonNode childInput = objectMapper.readTree("{\"resetDuringEncoding\":true}");
+ doAnswer(invocation -> {
+ resets.incrementAndGet();
+ executor.clearNestedExecutionWaits();
+ return invocation.callRealMethod();
+ }).when(mapper).writeValueAsString(eq(childInput));
+ when(sfnService.describeExecution(CHILD_ARN)).thenReturn(child("SUCCEEDED"));
+
+ Execution execution = run(executor, mode, 0, false, 0, false,
+ ", \"Input\": {\"resetDuringEncoding\":true}");
+
+ assertEquals(1, resets.get(), "reset occurs while encoding the child's input");
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.TaskFailed", execution.getError());
+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {".sync", ".sync:2"})
+ @Timeout(5)
+ void injectedClockReachesNestedTaskDeadline(String mode) {
+ AtomicLong nanos = new AtomicLong();
+ AtomicInteger waits = new AtomicInteger();
+ nestedWaiter = (subscription, observed, remaining) -> {
+ assertEquals(TimeUnit.SECONDS.toNanos(60), remaining);
+ waits.incrementAndGet();
+ nanos.addAndGet(remaining);
+ };
+ when(sfnService.describeExecution(CHILD_ARN)).thenReturn(child("RUNNING"));
+
+ Execution execution = run(newExecutor(pause -> {
+ throw new AssertionError("Nested waits must not poll through the sleeper");
+ }, nanos::get), mode, 60, true);
+
+ assertEquals(1, waits.get());
+ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());
+ assertTrue(history.stream().anyMatch(event -> "TaskTimedOut".equals(event.getType())));
+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {".sync", ".sync:2"})
+ @Timeout(5)
+ void injectedClockReachesParentDeadlineBeforeNestedTaskDeadline(String mode) {
+ AtomicLong nanos = new AtomicLong();
+ AtomicInteger waits = new AtomicInteger();
+ nestedWaiter = (subscription, observed, remaining) -> {
+ assertEquals(TimeUnit.SECONDS.toNanos(30), remaining);
+ waits.incrementAndGet();
+ nanos.addAndGet(remaining);
+ };
+ when(sfnService.describeExecution(CHILD_ARN)).thenReturn(child("RUNNING"));
+
+ Execution execution = run(newExecutor(pause -> {
+ throw new AssertionError("Nested waits must not poll through the sleeper");
+ }, nanos::get), mode, 60, false, 30);
+
+ assertEquals(1, waits.get());
+ assertEquals("TIMED_OUT", execution.getStatus());
+ assertTrue(history.stream().noneMatch(event -> "TaskTimedOut".equals(event.getType())));
+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {".sync", ".sync:2"})
+ void clockAdvanceOnAParkedWaitIsCheckedAfterWake(String mode) throws Exception {
+ AtomicLong nanos = new AtomicLong();
+ AtomicReference<ExecutionStatusSubscription> parked = new AtomicReference<>();
+ AtomicReference<Thread> waiter = new AtomicReference<>();
+ nestedWaiter = (subscription, observed, remaining) -> {
+ parked.set(subscription);
+ waiter.set(Thread.currentThread());
+ subscription.awaitChange(observed, remaining);
+ };
+ when(sfnService.describeExecution(CHILD_ARN)).thenReturn(child("RUNNING"));
+ AslExecutor executor = newExecutor(pause -> {
+ throw new AssertionError("Nested waits must not poll through the sleeper");
+ }, nanos::get);
+ CompletableFuture<Execution> result = CompletableFuture.supplyAsync(
+ () -> run(executor, mode, 60, true));
+ try {
+ await().atMost(Duration.ofSeconds(2)).until(() -> waiter.get() != null
+ && waiter.get().getState() == Thread.State.TIMED_WAITING);
+ ExecutionStatusSubscription subscription = parked.get();
+ synchronized (subscription) {
+ nanos.set(TimeUnit.SECONDS.toNanos(60));
+ subscription.notifyAll();
+ }
+ Execution execution = result.get(2, TimeUnit.SECONDS);
+ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());
+ assertTrue(history.stream().anyMatch(event -> "TaskTimedOut".equals(event.getType())));
+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());
+ } finally {
+ executor.clearNestedExecutionWaits();
+ result.get(2, TimeUnit.SECONDS);
+ }
+ }
+
private static Execution child(String status) {
Execution current = new Execution();
current.setExecutionArn(CHILD_ARN);
@@ -156,15 +353,25 @@ class AslExecutorNestedSyncTimeoutTest {
return current;
}
- @SuppressWarnings("unchecked")
private AslExecutor newExecutor(AslExecutor.Sleeper sleeper) {
+ return newExecutor(sleeper, System::nanoTime);
+ }
+
+ private AslExecutor newExecutor(AslExecutor.Sleeper sleeper, LongSupplier nanoTime) {
+ return newExecutor(sleeper, nanoTime, objectMapper);
+ }
+
+ @SuppressWarnings("unchecked")
+ private AslExecutor newExecutor(AslExecutor.Sleeper sleeper, LongSupplier nanoTime, ObjectMapper mapper) {
Instance<StepFunctionsService> instance = mock(Instance.class);
when(instance.get()).thenReturn(sfnService);
return AslExecutorFixture.builder()
- .objectMapper(objectMapper)
+ .objectMapper(mapper)
.sfnService(instance)
.customResourceLiveness(null)
.sleeper(sleeper)
+ .nanoTime(nanoTime)
+ .nestedWaiter(nestedWaiter)
.build();
}
@@ -173,7 +380,17 @@ class AslExecutorNestedSyncTimeoutTest {
}
private Execution run(AslExecutor executor, String mode, int timeoutSeconds, boolean catchTimeout,
- int executionTimeoutSeconds) {
+ int executionTimeoutSeconds) {
+ return run(executor, mode, timeoutSeconds, catchTimeout, executionTimeoutSeconds, false);
+ }
+
+ private Execution run(AslExecutor executor, String mode, int timeoutSeconds, boolean catchTimeout,
+ int executionTimeoutSeconds, boolean asynchronous) {
+ return run(executor, mode, timeoutSeconds, catchTimeout, executionTimeoutSeconds, asynchronous, "");
+ }
+
+ private Execution run(AslExecutor executor, String mode, int timeoutSeconds, boolean catchTimeout,
+ int executionTimeoutSeconds, boolean asynchronous, String additionalParameters) {
String timeout = timeoutSeconds > 0 ? "\"TimeoutSeconds\": %d,".formatted(timeoutSeconds) : "";
String catcher = catchTimeout
? "\"Catch\": [{\"ErrorEquals\": [\"States.Timeout\"], \"Next\": \"Recover\"}],"
@@ -191,13 +408,13 @@ class AslExecutorNestedSyncTimeoutTest {
"Resource": "arn:aws:states:::states:startExecution%s",
%s
%s
- "Parameters": { "StateMachineArn": "%s" },
+ "Parameters": { "StateMachineArn": "%s"%s },
"End": true
},
"Recover": { "Type": "Pass", "End": true }
}
}
- """.formatted(budget, mode, timeout, catcher, CHILD_SM_ARN);
+ """.formatted(budget, mode, timeout, catcher, CHILD_SM_ARN, additionalParameters);
StateMachine stateMachine = new StateMachine();
stateMachine.setName("parent");
@@ -211,8 +428,15 @@ class AslExecutorNestedSyncTimeoutTest {
execution.setStateMachineArn(stateMachine.getStateMachineArn());
execution.setInput("{}");
- history = new ArrayList<>();
- executor.executeSync(stateMachine, execution, history, (updated, events) -> { });
+ history = new CopyOnWriteArrayList<>();
+ if (asynchronous) {
+ executor.executeAsync(stateMachine, execution, history, (updated, events) -> { });
+ await().atMost(Duration.ofSeconds(10))
+ .until(() -> history.stream().anyMatch(event -> event.getType().startsWith("Execution")
+ && !event.getType().equals("ExecutionStarted")));
+ } else {
+ executor.executeSync(stateMachine, execution, history, (updated, events) -> { });
+ }
return execution;
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorParallelCutTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorParallelCutTest.java
index 6980caa809..68dd7ab2f8 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorParallelCutTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorParallelCutTest.java
@@ -46,6 +46,7 @@ import static org.mockito.Mockito.when;
* <p>The failing branch pauses in a Wait that the sleeper holds until every other branch is
* parked on its own wait, so the cut always lands while the others are mid-state.
*/
+@Timeout(20)
class AslExecutorParallelCutTest {
private static final String REGION = "us-east-2";
@@ -105,6 +106,7 @@ class AslExecutorParallelCutTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private StepFunctionsService sfnService;
private List<HistoryEvent> history;
+ private Runnable nestedWaitStarted = () -> { };
@BeforeEach
void setUp() {
@@ -376,20 +378,23 @@ class AslExecutorParallelCutTest {
/**
* The failing branch's one-second Pause is held until {@code waitingBranches} other branch
- * threads have started a sleep of their own (a {@code .sync} poll, or a longer Wait); every
+ * threads have entered a nested status wait or a longer Wait; every
* other sleep is real, so the cut interrupts it.
*/
- private static AslExecutor.Sleeper sleeper(int waitingBranches) {
+ private AslExecutor.Sleeper sleeper(int waitingBranches) {
CountDownLatch othersWaiting = new CountDownLatch(waitingBranches);
Set<Thread> seen = ConcurrentHashMap.newKeySet();
+ nestedWaitStarted = () -> {
+ if (seen.add(Thread.currentThread())) {
+ othersWaiting.countDown();
+ }
+ };
return nanos -> {
if (nanos == PAUSE_NANOS) {
- othersWaiting.await();
+ assertTrue(othersWaiting.await(5, TimeUnit.SECONDS), "other branches must enter their waits");
return;
}
- if (seen.add(Thread.currentThread())) {
- othersWaiting.countDown();
- }
+ nestedWaitStarted.run();
TimeUnit.NANOSECONDS.sleep(nanos);
};
}
@@ -403,6 +408,10 @@ class AslExecutorParallelCutTest {
.sfnService(instance)
.customResourceLiveness(null)
.sleeper(sleeper)
+ .nestedWaiter((subscription, observed, nanos) -> {
+ nestedWaitStarted.run();
+ subscription.awaitChange(observed, nanos);
+ })
.maxWaitSeconds(30)
.build();
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/model/ExecutionStatusSubscriptionTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/model/ExecutionStatusSubscriptionTest.java
new file mode 100644
index 0000000000..eb2f34e116
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/model/ExecutionStatusSubscriptionTest.java
@@ -0,0 +1,154 @@
+package io.github.hectorvent.floci.services.stepfunctions.model;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.Field;
+import java.time.Duration;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.*;
+
+class ExecutionStatusSubscriptionTest {
+ @Test
+ void completionBeforeRegistrationDoesNotPark() throws Exception {
+ Execution child = new Execution();
+ child.setStatus("SUCCEEDED");
+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, null)) {
+ assertTrue(subscription.version() > 0);
+ assertNull(registrations(child));
+ }
+ }
+
+ @Test
+ void completionBetweenStateReadAndParkingCannotBeLost() throws Exception {
+ Execution child = new Execution();
+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, null)) {
+ long observed = subscription.version();
+ child.setOutput("{\"answer\":42}");
+ child.setStatus("SUCCEEDED");
+ assertTimeoutPreemptively(Duration.ofSeconds(1),
+ () -> assertTrue(subscription.awaitChange(observed, TimeUnit.MINUTES.toNanos(1))));
+ assertEquals("{\"answer\":42}", child.getOutput());
+ assertNull(registrations(child));
+ }
+ }
+
+ @Test
+ void parentTerminationWakesTheWaitAndCloseReleasesBothRegistrations() throws Exception {
+ Execution child = new Execution();
+ Execution parent = new Execution();
+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, parent);
+ ExecutorService executor = Executors.newSingleThreadExecutor()) {
+ long observed = subscription.version();
+ CountDownLatch entered = new CountDownLatch(1);
+ CompletableFuture<Boolean> result = CompletableFuture.supplyAsync(() -> {
+ entered.countDown();
+ try {
+ return subscription.awaitChange(observed, TimeUnit.SECONDS.toNanos(5));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(e);
+ }
+ }, executor);
+ assertTrue(entered.await(1, TimeUnit.SECONDS));
+ parent.setStatus("ABORTED");
+ assertTrue(result.get(1, TimeUnit.SECONDS));
+ assertEquals("RUNNING", child.getStatus());
+ }
+ assertNull(registrations(child));
+ assertNull(registrations(parent));
+ }
+
+ @Test
+ void unrelatedExecutionDoesNotChangeTheObservedVersion() throws Exception {
+ Execution child = new Execution();
+ Execution parent = new Execution();
+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, parent)) {
+ long observed = subscription.version();
+ new Execution().setStatus("SUCCEEDED");
+ assertEquals(observed, subscription.version());
+ assertFalse(subscription.awaitChange(observed, 0));
+ }
+ }
+
+ @Test
+ void timeoutAndCancellationDoNotManufactureTerminalStatus() throws Exception {
+ Execution child = new Execution();
+ Execution parent = new Execution();
+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, parent)) {
+ long observed = subscription.version();
+ assertFalse(subscription.awaitChange(observed, TimeUnit.MILLISECONDS.toNanos(1)));
+ subscription.cancel();
+ assertTrue(subscription.cancelled());
+ assertTrue(subscription.awaitChange(observed, TimeUnit.MINUTES.toNanos(1)));
+ assertEquals("RUNNING", child.getStatus());
+ assertEquals("RUNNING", parent.getStatus());
+ }
+ assertNull(registrations(child));
+ assertNull(registrations(parent));
+ }
+
+ @Test
+ void wakeWithoutStatusChangeReturnsToTheExecutorsDeadlineClock() throws Exception {
+ Execution child = new Execution();
+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, null);
+ ExecutorService executor = Executors.newSingleThreadExecutor()) {
+ AtomicReference<Thread> waiter = new AtomicReference<>();
+ CompletableFuture<Boolean> result = CompletableFuture.supplyAsync(() -> {
+ waiter.set(Thread.currentThread());
+ try {
+ return subscription.awaitChange(subscription.version(), TimeUnit.MINUTES.toNanos(1));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(e);
+ }
+ }, executor);
+ try {
+ await().atMost(Duration.ofSeconds(2)).until(() -> waiter.get() != null
+ && waiter.get().getState() == Thread.State.TIMED_WAITING);
+ synchronized (subscription) {
+ subscription.notifyAll();
+ }
+ assertFalse(result.get(1, TimeUnit.SECONDS));
+ assertEquals("RUNNING", child.getStatus());
+ } finally {
+ subscription.cancel();
+ }
+ }
+ }
+
+ @Test
+ void duplicateTargetsAndRepeatedCloseLeaveNoRegistration() throws Exception {
+ Execution child = new Execution();
+ ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, child);
+ subscription.close();
+ subscription.close();
+ assertNull(registrations(child));
+ }
+
+ @Test
+ void ephemeralRegistrationsAreAbsentFromPersistedExecution() throws Exception {
+ Execution child = new Execution();
+ ObjectMapper mapper = new ObjectMapper();
+ String before = mapper.writeValueAsString(child);
+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, null)) {
+ assertEquals(before, mapper.writeValueAsString(child));
+ Execution restored = mapper.readValue(before, Execution.class);
+ assertNull(registrations(restored));
+ assertEquals("RUNNING", restored.getStatus());
+ }
+ }
+
+ private static Object registrations(Execution execution) throws Exception {
+ Field field = Execution.class.getDeclaredField("statusSubscriptions");
+ field.setAccessible(true);
+ return field.get(execution);
+ }
+}