Light, fluffy, and always free - The AWS Local Emulator alternative
perf(stepfunctions): wake nested waits on execution status changes (#6513)
Andre Kuraitfe928d67ab23
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.javaindex cc43c15caa..96a77a5757 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java@@ -62,6 +62,7 @@ import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService.TokenTimeout;import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService.TokenWait;import io.github.hectorvent.floci.services.stepfunctions.model.Execution;+import io.github.hectorvent.floci.services.stepfunctions.model.ExecutionStatusSubscription;import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;import io.github.hectorvent.floci.services.stepfunctions.model.MapRun;import io.github.hectorvent.floci.services.stepfunctions.model.MockedResponseStep;@@ -278,12 +279,15 @@ public class AslExecutor {private final CustomResourceLiveness customResourceLiveness;private final Clock clock;private final Sleeper sleeper;+ private final NestedWaiter nestedWaiter;// Null in production, where the ceiling comes from the Step Functions config. Tests pin it.private final Integer maxWaitSecondsOverride;private final Duration expressMaxExecutionTime;// The monotonic clock every execution and task deadline is measured on. Tests move it forward.private final LongSupplier nanoTime;private final Map<String, ActiveMockExecution> activeMocks = new ConcurrentHashMap<>();+ private final Set<ExecutionStatusSubscription> nestedWaits = new HashSet<>();+ private long nestedWaitGeneration;private final ExecutorService executor = Executors.newCachedThreadPool(r -> {Thread t = new Thread(r, "sfn-executor");t.setDaemon(true);@@ -312,7 +316,7 @@ public class AslExecutor {objectMapper, jsonataEvaluator, sfnService, config, vertx, customResourceLiveness,Clock.systemUTC(), TimeUnit.NANOSECONDS::sleep, null, redshiftDataHandler, serviceRegistry,nestedTaskCredentials, lambdaTaskAuthorizer, null, null, apiGatewayTaskInvoker, emrTaskInvoker,- emrServerlessTaskInvoker);+ emrServerlessTaskInvoker, null);}AslExecutor(LambdaExecutorService lambdaExecutor, LambdaTargetResolver targetResolver,@@ -331,7 +335,7 @@ public class AslExecutor {NestedTaskCredentials nestedTaskCredentials, LambdaTaskAuthorizer lambdaTaskAuthorizer,Duration expressMaxExecutionTimeOverride, LongSupplier nanoTimeOverride,ApiGatewayTaskInvoker apiGatewayTaskInvoker, EmrTaskInvoker emrTaskInvoker,- EmrServerlessTaskInvoker emrServerlessTaskInvoker) {+ EmrServerlessTaskInvoker emrServerlessTaskInvoker, NestedWaiter nestedWaiter) {this.apiGatewayTaskInvoker = apiGatewayTaskInvoker;this.emrTaskInvoker = emrTaskInvoker;this.emrServerlessTaskInvoker = emrServerlessTaskInvoker;@@ -367,6 +371,7 @@ public class AslExecutor {this.config = config;this.clock = clock;this.sleeper = sleeper;+ this.nestedWaiter = nestedWaiter != null ? nestedWaiter : ExecutionStatusSubscription::awaitChange;this.maxWaitSecondsOverride = maxWaitSecondsOverride;this.nanoTime = nanoTimeOverride != null ? nanoTimeOverride : System::nanoTime;this.expressMaxExecutionTime = expressMaxExecutionTimeOverride != null@@ -398,7 +403,7 @@ public class AslExecutor {eventBridgeHandler, schedulerService, schedulerController, rdsDataService,objectMapper, jsonataEvaluator, sfnService, config, vertx, customResourceLiveness,Clock.systemUTC(), TimeUnit.NANOSECONDS::sleep, null, null, null, null, null, null, null, null, null,- null);+ null, null);}// Kept for tests that do not exercise the Redshift Data API integration.@@ -419,7 +424,7 @@ public class AslExecutor {ec2Service, s3Service, ecsService, ecsJsonHandler,eventBridgeHandler, schedulerService, schedulerController, rdsDataService,objectMapper, jsonataEvaluator, sfnService, config, vertx, customResourceLiveness,- clock, sleeper, maxWaitSecondsOverride, null, null, null, null, null, null, null, null, null);+ clock, sleeper, maxWaitSecondsOverride, null, null, null, null, null, null, null, null, null, null);}AslExecutor(LambdaExecutorService lambdaExecutor, LambdaFunctionStore functionStore,@@ -441,6 +446,13 @@ public class AslExecutor {vertx, customResourceLiveness, clock, sleeper, maxWaitSecondsOverride);}+ /** Test seam paired with the monotonic clock for nested deadline and cut coordination. */+ @FunctionalInterface+ interface NestedWaiter {+ void awaitChange(ExecutionStatusSubscription subscription, long observed, long timeoutNanos)+ throws InterruptedException;+ }+/** Test seam: lets Wait states be exercised without real time passing. */@FunctionalInterfaceinterface Sleeper {@@ -1925,6 +1937,7 @@ public class AslExecutor {long executionDeadlineNanos, long taskDeadlineNanos,JsonNode rawParameters, Consumer<JsonNode> jobSubmitted)throws Exception {+ long waitGeneration = nestedWaitGeneration();String smArn = input.path("StateMachineArn").asText(null);if (smArn == null || smArn.isBlank()) {throw new FailStateException("States.TaskFailed",@@ -1951,7 +1964,7 @@ public class AslExecutor {childName = null;}- io.github.hectorvent.floci.services.stepfunctions.model.Execution exec;+ Execution exec;try {exec = sfnService.get().startExecution(smArn, childName, childInput, region);} catch (AwsException e) {@@ -1981,44 +1994,98 @@ public class AslExecutor {throw e;}- // .sync or .sync:2 polls until terminal. Whatever else ends the wait aborts the child, the way+ // .sync or .sync:2 waits for terminal publication. Whatever else ends the wait aborts the child, the way// AWS does (measured: child ABORTED, no error, the cause below): the Task's own TimeoutSeconds,// which fails the state with States.Timeout; the parent execution's budget, which ends it// TIMED_OUT; a failure in a sibling Parallel branch, which interrupts this one; and a- // StopExecution on the parent, so a stopped execution does not keep a polling worker.- while (true) {- try {- sleepOrTimeOutTask(TimeUnit.MILLISECONDS.toNanos(SYNC_POLL_INTERVAL_MS),- executionDeadlineNanos, taskDeadlineNanos);- } catch (TaskTimedOutException | ExecutionTimedOutException | InterruptedException e) {- abortChildExecution(execArn, executionArn);- throw e;- }- if (abortedByCaller(executionArn)) {- abortChildExecution(execArn, executionArn);- throw new ExecutionAbortedException();+ // StopExecution on the parent. Register before reading either execution to cover+ // completion during registration and between a state read and parking.+ ExecutionStatusSubscription waiting = new ExecutionStatusSubscription(exec, nestedWaitParent(executionArn));+ registerNestedWait(waiting, waitGeneration);+ try (waiting) {+ while (true) {+ if (waiting.cancelled()) {+ abortChildExecution(execArn, executionArn);+ throw new FailStateException("States.TaskFailed", "StepFunctionsService cleared");+ }+ long observed = waiting.version();+ checkNestedTaskDeadline(executionDeadlineNanos, taskDeadlineNanos);+ if (abortedByCaller(executionArn)) {+ abortChildExecution(execArn, executionArn);+ throw new ExecutionAbortedException();+ }+ Execution current = sfnService.get().describeExecution(execArn);+ String status;+ // Terminal fields are written under this monitor, status last.+ synchronized (current) {+ status = current.getStatus();+ }+ if ("SUCCEEDED".equals(status)) {+ return nestedExecutionEnvelope(current, true, ".sync:2".equals(mode));+ }+ if (!"RUNNING".equals(status)) {+ throw new FailStateException("States.TaskFailed",+ nestedExecutionEnvelope(current, false, false).toString());+ }+ nestedWaiter.awaitChange(waiting, observed,+ Math.min(executionDeadlineNanos, taskDeadlineNanos) - nanoTime.getAsLong());}- io.github.hectorvent.floci.services.stepfunctions.model.Execution current =- sfnService.get().describeExecution(execArn);- String status;- // The child's worker and StopExecution write its terminal fields under this monitor,- // status last. Reading the status under it makes everything written before it visible,- // so a terminal status is never seen without the error, cause and stop date behind it.- synchronized (current) {- status = current.getStatus();+ } catch (TaskTimedOutException | ExecutionTimedOutException | InterruptedException e) {+ abortChildExecution(execArn, executionArn);+ throw e;+ } finally {+ synchronized (nestedWaits) {+ nestedWaits.remove(waiting);}- if ("RUNNING".equals(status)) {- continue;+ }+ }++ private long nestedWaitGeneration() {+ synchronized (nestedWaits) {+ return nestedWaitGeneration;+ }+ }++ private void registerNestedWait(ExecutionStatusSubscription waiting, long generation) {+ synchronized (nestedWaits) {+ if (generation != nestedWaitGeneration) {+ waiting.cancel();+ } else {+ nestedWaits.add(waiting);}- if ("SUCCEEDED".equals(status)) {- // Both modes return the envelope (measured); .sync:2 differs only in carrying- // Input and Output as JSON values where .sync carries them as JSON strings.- return nestedExecutionEnvelope(current, true, ".sync:2".equals(mode));+ }+ }++ void clearNestedExecutionWaits() {+ synchronized (nestedWaits) {+ nestedWaitGeneration++;+ for (ExecutionStatusSubscription waiting : nestedWaits) {+ waiting.cancel();}- // However the child ended, FAILED, TIMED_OUT or ABORTED, and whatever its own error, the- // parent sees States.TaskFailed (measured), so a Catch on the child's error never fires.- throw new FailStateException("States.TaskFailed",- nestedExecutionEnvelope(current, false, false).toString());+ nestedWaits.clear();+ }+ }++ private Execution nestedWaitParent(String executionArn) {+ if (executionArn == null) {+ return null;+ }+ try {+ return RequestScopes.callAs(AwsArnUtils.parse(executionArn).accountId(),+ () -> sfnService.get().describeExecution(executionArn));+ } catch (AwsException e) {+ LOG.warnv("Parent execution {0} vanished before its nested wait: {1}", executionArn, e.getMessage());+ return null;+ }+ }++ private void checkNestedTaskDeadline(long executionDeadlineNanos, long taskDeadlineNanos) {+ long now = nanoTime.getAsLong();+ if (now >= executionDeadlineNanos) {+ throw new ExecutionTimedOutException();+ }+ if (now >= taskDeadlineNanos) {+ throw new TaskTimedOutException("States.Timeout");}}diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.javaindex b647e4b083..142849dece 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java@@ -176,6 +176,9 @@ public class StepFunctionsService implements Resettable, ResourceProvider {}public void clear() {+ if (aslExecutor != null) {+ aslExecutor.clearNestedExecutionWaits();+ }historyCache.clear();activityQueues.clear();Map<String, PendingTask> pending;diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.javaindex 77d4b2b25c..7fecbc7c6e 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/Execution.java@@ -1,10 +1,13 @@package io.github.hectorvent.floci.services.stepfunctions.model;+import com.fasterxml.jackson.annotation.JsonIgnore;import com.fasterxml.jackson.annotation.JsonIgnoreProperties;import io.quarkus.runtime.annotations.RegisterForReflection;import java.util.ArrayList;+import java.util.HashSet;import java.util.List;+import java.util.Set;@RegisterForReflection@JsonIgnoreProperties(ignoreUnknown = true)@@ -22,6 +25,8 @@ public class Execution {private String error;private String cause;private List<HistoryEvent> history = new ArrayList<>();+ @JsonIgnore+ private transient Set<ExecutionStatusSubscription> statusSubscriptions;public Execution() {this.startDate = System.currentTimeMillis() / 1000.0;@@ -47,7 +52,36 @@ public class Execution {public void setName(String name) { this.name = name; }public String getStatus() { return status; }- public void setStatus(String status) { this.status = status; }+ public synchronized void setStatus(String status) {+ this.status = status;+ if (!"RUNNING".equals(status) && statusSubscriptions != null) {+ Set<ExecutionStatusSubscription> subscriptions = statusSubscriptions;+ statusSubscriptions = null;+ for (ExecutionStatusSubscription subscription : subscriptions) {+ subscription.signal();+ }+ }+ }++ synchronized void registerStatusSubscription(ExecutionStatusSubscription subscription) {+ if (!"RUNNING".equals(status)) {+ subscription.signal();+ return;+ }+ if (statusSubscriptions == null) {+ statusSubscriptions = new HashSet<>();+ }+ statusSubscriptions.add(subscription);+ }++ synchronized void unregisterStatusSubscription(ExecutionStatusSubscription subscription) {+ if (statusSubscriptions != null) {+ statusSubscriptions.remove(subscription);+ if (statusSubscriptions.isEmpty()) {+ statusSubscriptions = null;+ }+ }+ }public String getInput() { return input; }public void setInput(String input) { this.input = input; }diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/ExecutionStatusSubscription.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/ExecutionStatusSubscription.javanew file mode 100644index 0000000000..b59e56d8c8--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/model/ExecutionStatusSubscription.java@@ -0,0 +1,70 @@+package io.github.hectorvent.floci.services.stepfunctions.model;++import java.util.Objects;+import java.util.concurrent.TimeUnit;++/**+ * A scoped wakeup for a child execution ending or its parent being stopped.+ * Read the version before authoritative state so completion cannot be lost+ * between that read and parking. Closing removes both registrations.+ */+public final class ExecutionStatusSubscription implements AutoCloseable {+ private final Execution child;+ private final Execution parent;+ private long version;+ private boolean closed;+ private boolean cancelled;++ public ExecutionStatusSubscription(Execution child, Execution parent) {+ this.child = Objects.requireNonNull(child);+ this.parent = parent == child ? null : parent;+ child.registerStatusSubscription(this);+ if (this.parent != null) {+ this.parent.registerStatusSubscription(this);+ }+ }++ public synchronized long version() {+ return version;+ }++ synchronized void signal() {+ version++;+ notifyAll();+ }++ public synchronized void cancel() {+ cancelled = true;+ notifyAll();+ }++ public synchronized boolean cancelled() {+ return cancelled;+ }++ public synchronized boolean awaitChange(long observed, long timeoutNanos) throws InterruptedException {+ // The executor owns the deadline and its clock. Recheck there after every wake,+ // including a spurious wake, rather than creating a second real-clock deadline.+ if (!closed && !cancelled && version == observed && timeoutNanos > 0) {+ TimeUnit.NANOSECONDS.timedWait(this, timeoutNanos);+ }+ return closed || cancelled || version != observed;+ }++ @Override+ public void close() {+ synchronized (this) {+ if (closed) {+ return;+ }+ closed = true;+ notifyAll();+ }+ // setStatus holds an execution monitor while signalling. Do not take+ // an execution monitor while holding this subscription's monitor.+ child.unregisterStatusSubscription(this);+ if (parent != null) {+ parent.unregisterStatusSubscription(this);+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorFixture.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorFixture.javaindex 884faff3ac..ea33f0a894 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorFixture.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorFixture.java@@ -119,6 +119,7 @@ final class AslExecutorFixture {private LambdaTaskAuthorizer lambdaTaskAuthorizer;private Duration expressMaxExecutionTime;private LongSupplier nanoTime;+ private AslExecutor.NestedWaiter nestedWaiter;private EmrTaskInvoker emrTaskInvoker;private EmrServerlessTaskInvoker emrServerlessTaskInvoker;@@ -270,6 +271,11 @@ final class AslExecutorFixture {return this;}+ Builder nestedWaiter(AslExecutor.NestedWaiter nestedWaiter) {+ this.nestedWaiter = nestedWaiter;+ return this;+ }+Builder nanoTime(LongSupplier nanoTime) {this.nanoTime = nanoTime;return this;@@ -294,7 +300,7 @@ final class AslExecutorFixture {jsonataEvaluator != null ? jsonataEvaluator : new JsonataEvaluator(objectMapper),sfnService, config, vertx, customResourceLiveness, clock, sleeper, maxWaitSeconds,redshiftDataHandler, serviceRegistry, null, lambdaTaskAuthorizer, expressMaxExecutionTime,- nanoTime, null, emrTaskInvoker, emrServerlessTaskInvoker);+ nanoTime, null, emrTaskInvoker, emrServerlessTaskInvoker, nestedWaiter);}}}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.javaindex 3c17e349b7..86acc0062b 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorNestedSyncTimeoutTest.java@@ -3,25 +3,39 @@ package io.github.hectorvent.floci.services.stepfunctions;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.services.stepfunctions.model.Execution;+import io.github.hectorvent.floci.services.stepfunctions.model.ExecutionStatusSubscription;import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;import jakarta.enterprise.inject.Instance;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;-import java.util.ArrayList;+import java.time.Duration;import java.util.List;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.CopyOnWriteArrayList;+import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicLong;+import java.util.concurrent.atomic.AtomicReference;+import java.util.function.LongSupplier;+import static org.awaitility.Awaitility.await;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.eq;import static org.mockito.ArgumentMatchers.isNull;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;+import static org.mockito.Mockito.spy;+import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;@@ -48,6 +62,7 @@ class AslExecutorNestedSyncTimeoutTest {private final ObjectMapper objectMapper = new ObjectMapper();private StepFunctionsService sfnService;private List<HistoryEvent> history;+ private AslExecutor.NestedWaiter nestedWaiter;@BeforeEachvoid setUp() {@@ -86,23 +101,24 @@ class AslExecutorNestedSyncTimeoutTest {verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());}- @Test- void syncIsNotCappedByAPollCountWhenNoTimeoutSecondsIsDeclared() throws Exception {- AtomicInteger polls = new AtomicInteger();+ @ParameterizedTest+ @ValueSource(strings = {".sync", ".sync:2"})+ void syncCompletionDoesNotDependOnPolling(String mode) throws Exception {+ AtomicLong nanos = new AtomicLong();when(sfnService.describeExecution(CHILD_ARN)).thenAnswer(invocation -> {- if (polls.incrementAndGet() < 700) {- return child("RUNNING");- }+ nanos.addAndGet(TimeUnit.MILLISECONDS.toNanos(350));Execution done = child("SUCCEEDED");done.setOutput("{\"answer\":42}");return done;});- Execution execution = run(newExecutor(nanos -> { }), ".sync", 0, false);+ Execution execution = run(newExecutor(pause -> {+ throw new AssertionError("Nested execution completion must not use a polling sleep");+ }, nanos::get), mode, 0, false, 0, true);assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());- assertEquals(700, polls.get());- assertTrue(objectMapper.readTree(execution.getOutput()).path("Output").asText().contains("42"));+ assertEquals(TimeUnit.MILLISECONDS.toNanos(350), nanos.get());+ assertTrue(objectMapper.readTree(execution.getOutput()).path("Output").toString().contains("42"));verify(sfnService, never()).stopExecution(any(), any(), any());}@@ -120,18 +136,19 @@ class AslExecutorNestedSyncTimeoutTest {@Testvoid stopExecutionOnTheParentEndsTheWaitAndAbortsTheChild() {- AtomicInteger parentReads = new AtomicInteger();- when(sfnService.describeExecution(PARENT_ARN))- .thenAnswer(invocation -> parent(parentReads.incrementAndGet() < 3 ? "RUNNING" : "ABORTED"));+ Execution parent = parent("RUNNING");+ when(sfnService.describeExecution(PARENT_ARN)).thenReturn(parent);AtomicInteger childPolls = new AtomicInteger();when(sfnService.describeExecution(CHILD_ARN)).thenAnswer(invocation -> {childPolls.incrementAndGet();+ parent.setStatus("ABORTED");return child("RUNNING");});- run(newExecutor(nanos -> { }), ".sync", 0, false);+ run(newExecutor(nanos -> { throw new AssertionError("Parent cancellation must notify the wait"); }),+ ".sync", 0, false);- assertEquals(2, childPolls.get(), "polling stops as soon as the parent reads ABORTED");+ assertEquals(1, childPolls.get(), "the parent cancellation wakes the wait without rereading its child");verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());List<String> types = history.stream().map(HistoryEvent::getType).toList();assertTrue(types.stream().noneMatch(type -> type.equals("TaskTimedOut")@@ -139,6 +156,186 @@ class AslExecutorNestedSyncTimeoutTest {&& !type.equals("ExecutionStarted")), "an aborted execution gets no terminal event from the worker: " + types);}+ @ParameterizedTest+ @ValueSource(strings = {".sync", ".sync:2"})+ void childCompletionWakesAParkedParentWithoutPolling(String mode) throws Exception {+ Execution live = child("RUNNING");+ when(sfnService.startExecution(any(), any(), any(), any())).thenReturn(live);+ CountDownLatch firstRead = new CountDownLatch(1);+ AtomicReference<Thread> waitingThread = new AtomicReference<>();+ when(sfnService.describeExecution(CHILD_ARN)).thenAnswer(invocation -> {+ waitingThread.set(Thread.currentThread());+ firstRead.countDown();+ return live;+ });+ AslExecutor executor = newExecutor(nanos -> {+ throw new AssertionError("Nested completion must not use the polling sleeper");+ });+ CompletableFuture<Execution> result = CompletableFuture.supplyAsync(+ () -> run(executor, mode, 60, false));+ try {+ assertTrue(firstRead.await(2, TimeUnit.SECONDS));+ await().atMost(Duration.ofSeconds(2))+ .until(() -> waitingThread.get().getState() == Thread.State.TIMED_WAITING);+ verify(sfnService, times(1)).describeExecution(CHILD_ARN);+ synchronized (live) {+ live.setOutput("{\"answer\":42}");+ live.setStopDate(2.0);+ live.setStatus("SUCCEEDED");+ }++ Execution execution = result.get(2, TimeUnit.SECONDS);+ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());+ assertTrue(objectMapper.readTree(execution.getOutput()).path("Output").toString().contains("42"));+ verify(sfnService, times(2)).describeExecution(CHILD_ARN);+ verify(sfnService, never()).stopExecution(any(), any(), any());+ } finally {+ if (!result.isDone() && waitingThread.get() != null) {+ waitingThread.get().interrupt();+ }+ }+ }++ @Test+ void clearCancelsAnActiveNestedWaitAndAbortsItsChild() {+ AslExecutor executor = newExecutor(nanos -> {+ throw new AssertionError("Reset must notify the wait");+ });+ when(sfnService.describeExecution(CHILD_ARN)).thenAnswer(invocation -> {+ executor.clearNestedExecutionWaits();+ return child("RUNNING");+ });++ Execution execution = run(executor, ".sync", 0, false);++ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.TaskFailed", execution.getError());+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());+ }++ @Test+ void clearDuringChildStartCannotLeaveAnUnregisteredWait() {+ AslExecutor executor = newExecutor(nanos -> {+ throw new AssertionError("Reset during submission must not start polling");+ });+ when(sfnService.startExecution(any(), any(), any(), any())).thenAnswer(invocation -> {+ executor.clearNestedExecutionWaits();+ return child("RUNNING");+ });++ Execution execution = run(executor, ".sync:2", 0, false);++ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.TaskFailed", execution.getError());+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());+ }++ @ParameterizedTest+ @ValueSource(strings = {".sync", ".sync:2"})+ void clearDuringInputEncodingCancelsTheNestedWait(String mode) throws Exception {+ ObjectMapper mapper = spy(objectMapper);+ AslExecutor executor = newExecutor(nanos -> {+ throw new AssertionError("Reset during input encoding must not start polling");+ }, System::nanoTime, mapper);+ AtomicInteger resets = new AtomicInteger();+ JsonNode childInput = objectMapper.readTree("{\"resetDuringEncoding\":true}");+ doAnswer(invocation -> {+ resets.incrementAndGet();+ executor.clearNestedExecutionWaits();+ return invocation.callRealMethod();+ }).when(mapper).writeValueAsString(eq(childInput));+ when(sfnService.describeExecution(CHILD_ARN)).thenReturn(child("SUCCEEDED"));++ Execution execution = run(executor, mode, 0, false, 0, false,+ ", \"Input\": {\"resetDuringEncoding\":true}");++ assertEquals(1, resets.get(), "reset occurs while encoding the child's input");+ assertEquals("FAILED", execution.getStatus());+ assertEquals("States.TaskFailed", execution.getError());+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());+ }++ @ParameterizedTest+ @ValueSource(strings = {".sync", ".sync:2"})+ @Timeout(5)+ void injectedClockReachesNestedTaskDeadline(String mode) {+ AtomicLong nanos = new AtomicLong();+ AtomicInteger waits = new AtomicInteger();+ nestedWaiter = (subscription, observed, remaining) -> {+ assertEquals(TimeUnit.SECONDS.toNanos(60), remaining);+ waits.incrementAndGet();+ nanos.addAndGet(remaining);+ };+ when(sfnService.describeExecution(CHILD_ARN)).thenReturn(child("RUNNING"));++ Execution execution = run(newExecutor(pause -> {+ throw new AssertionError("Nested waits must not poll through the sleeper");+ }, nanos::get), mode, 60, true);++ assertEquals(1, waits.get());+ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());+ assertTrue(history.stream().anyMatch(event -> "TaskTimedOut".equals(event.getType())));+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());+ }++ @ParameterizedTest+ @ValueSource(strings = {".sync", ".sync:2"})+ @Timeout(5)+ void injectedClockReachesParentDeadlineBeforeNestedTaskDeadline(String mode) {+ AtomicLong nanos = new AtomicLong();+ AtomicInteger waits = new AtomicInteger();+ nestedWaiter = (subscription, observed, remaining) -> {+ assertEquals(TimeUnit.SECONDS.toNanos(30), remaining);+ waits.incrementAndGet();+ nanos.addAndGet(remaining);+ };+ when(sfnService.describeExecution(CHILD_ARN)).thenReturn(child("RUNNING"));++ Execution execution = run(newExecutor(pause -> {+ throw new AssertionError("Nested waits must not poll through the sleeper");+ }, nanos::get), mode, 60, false, 30);++ assertEquals(1, waits.get());+ assertEquals("TIMED_OUT", execution.getStatus());+ assertTrue(history.stream().noneMatch(event -> "TaskTimedOut".equals(event.getType())));+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());+ }++ @ParameterizedTest+ @ValueSource(strings = {".sync", ".sync:2"})+ void clockAdvanceOnAParkedWaitIsCheckedAfterWake(String mode) throws Exception {+ AtomicLong nanos = new AtomicLong();+ AtomicReference<ExecutionStatusSubscription> parked = new AtomicReference<>();+ AtomicReference<Thread> waiter = new AtomicReference<>();+ nestedWaiter = (subscription, observed, remaining) -> {+ parked.set(subscription);+ waiter.set(Thread.currentThread());+ subscription.awaitChange(observed, remaining);+ };+ when(sfnService.describeExecution(CHILD_ARN)).thenReturn(child("RUNNING"));+ AslExecutor executor = newExecutor(pause -> {+ throw new AssertionError("Nested waits must not poll through the sleeper");+ }, nanos::get);+ CompletableFuture<Execution> result = CompletableFuture.supplyAsync(+ () -> run(executor, mode, 60, true));+ try {+ await().atMost(Duration.ofSeconds(2)).until(() -> waiter.get() != null+ && waiter.get().getState() == Thread.State.TIMED_WAITING);+ ExecutionStatusSubscription subscription = parked.get();+ synchronized (subscription) {+ nanos.set(TimeUnit.SECONDS.toNanos(60));+ subscription.notifyAll();+ }+ Execution execution = result.get(2, TimeUnit.SECONDS);+ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());+ assertTrue(history.stream().anyMatch(event -> "TaskTimedOut".equals(event.getType())));+ verify(sfnService).stopExecution(eq(CHILD_ARN), eq(ABORT_CAUSE), isNull());+ } finally {+ executor.clearNestedExecutionWaits();+ result.get(2, TimeUnit.SECONDS);+ }+ }+private static Execution child(String status) {Execution current = new Execution();current.setExecutionArn(CHILD_ARN);@@ -156,15 +353,25 @@ class AslExecutorNestedSyncTimeoutTest {return current;}- @SuppressWarnings("unchecked")private AslExecutor newExecutor(AslExecutor.Sleeper sleeper) {+ return newExecutor(sleeper, System::nanoTime);+ }++ private AslExecutor newExecutor(AslExecutor.Sleeper sleeper, LongSupplier nanoTime) {+ return newExecutor(sleeper, nanoTime, objectMapper);+ }++ @SuppressWarnings("unchecked")+ private AslExecutor newExecutor(AslExecutor.Sleeper sleeper, LongSupplier nanoTime, ObjectMapper mapper) {Instance<StepFunctionsService> instance = mock(Instance.class);when(instance.get()).thenReturn(sfnService);return AslExecutorFixture.builder()- .objectMapper(objectMapper)+ .objectMapper(mapper).sfnService(instance).customResourceLiveness(null).sleeper(sleeper)+ .nanoTime(nanoTime)+ .nestedWaiter(nestedWaiter).build();}@@ -173,7 +380,17 @@ class AslExecutorNestedSyncTimeoutTest {}private Execution run(AslExecutor executor, String mode, int timeoutSeconds, boolean catchTimeout,- int executionTimeoutSeconds) {+ int executionTimeoutSeconds) {+ return run(executor, mode, timeoutSeconds, catchTimeout, executionTimeoutSeconds, false);+ }++ private Execution run(AslExecutor executor, String mode, int timeoutSeconds, boolean catchTimeout,+ int executionTimeoutSeconds, boolean asynchronous) {+ return run(executor, mode, timeoutSeconds, catchTimeout, executionTimeoutSeconds, asynchronous, "");+ }++ private Execution run(AslExecutor executor, String mode, int timeoutSeconds, boolean catchTimeout,+ int executionTimeoutSeconds, boolean asynchronous, String additionalParameters) {String timeout = timeoutSeconds > 0 ? "\"TimeoutSeconds\": %d,".formatted(timeoutSeconds) : "";String catcher = catchTimeout? "\"Catch\": [{\"ErrorEquals\": [\"States.Timeout\"], \"Next\": \"Recover\"}],"@@ -191,13 +408,13 @@ class AslExecutorNestedSyncTimeoutTest {"Resource": "arn:aws:states:::states:startExecution%s",%s%s- "Parameters": { "StateMachineArn": "%s" },+ "Parameters": { "StateMachineArn": "%s"%s },"End": true},"Recover": { "Type": "Pass", "End": true }}}- """.formatted(budget, mode, timeout, catcher, CHILD_SM_ARN);+ """.formatted(budget, mode, timeout, catcher, CHILD_SM_ARN, additionalParameters);StateMachine stateMachine = new StateMachine();stateMachine.setName("parent");@@ -211,8 +428,15 @@ class AslExecutorNestedSyncTimeoutTest {execution.setStateMachineArn(stateMachine.getStateMachineArn());execution.setInput("{}");- history = new ArrayList<>();- executor.executeSync(stateMachine, execution, history, (updated, events) -> { });+ history = new CopyOnWriteArrayList<>();+ if (asynchronous) {+ executor.executeAsync(stateMachine, execution, history, (updated, events) -> { });+ await().atMost(Duration.ofSeconds(10))+ .until(() -> history.stream().anyMatch(event -> event.getType().startsWith("Execution")+ && !event.getType().equals("ExecutionStarted")));+ } else {+ executor.executeSync(stateMachine, execution, history, (updated, events) -> { });+ }return execution;}}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorParallelCutTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorParallelCutTest.javaindex 6980caa809..68dd7ab2f8 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorParallelCutTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorParallelCutTest.java@@ -46,6 +46,7 @@ import static org.mockito.Mockito.when;* <p>The failing branch pauses in a Wait that the sleeper holds until every other branch is* parked on its own wait, so the cut always lands while the others are mid-state.*/+@Timeout(20)class AslExecutorParallelCutTest {private static final String REGION = "us-east-2";@@ -105,6 +106,7 @@ class AslExecutorParallelCutTest {private final ObjectMapper objectMapper = new ObjectMapper();private StepFunctionsService sfnService;private List<HistoryEvent> history;+ private Runnable nestedWaitStarted = () -> { };@BeforeEachvoid setUp() {@@ -376,20 +378,23 @@ class AslExecutorParallelCutTest {/*** The failing branch's one-second Pause is held until {@code waitingBranches} other branch- * threads have started a sleep of their own (a {@code .sync} poll, or a longer Wait); every+ * threads have entered a nested status wait or a longer Wait; every* other sleep is real, so the cut interrupts it.*/- private static AslExecutor.Sleeper sleeper(int waitingBranches) {+ private AslExecutor.Sleeper sleeper(int waitingBranches) {CountDownLatch othersWaiting = new CountDownLatch(waitingBranches);Set<Thread> seen = ConcurrentHashMap.newKeySet();+ nestedWaitStarted = () -> {+ if (seen.add(Thread.currentThread())) {+ othersWaiting.countDown();+ }+ };return nanos -> {if (nanos == PAUSE_NANOS) {- othersWaiting.await();+ assertTrue(othersWaiting.await(5, TimeUnit.SECONDS), "other branches must enter their waits");return;}- if (seen.add(Thread.currentThread())) {- othersWaiting.countDown();- }+ nestedWaitStarted.run();TimeUnit.NANOSECONDS.sleep(nanos);};}@@ -403,6 +408,10 @@ class AslExecutorParallelCutTest {.sfnService(instance).customResourceLiveness(null).sleeper(sleeper)+ .nestedWaiter((subscription, observed, nanos) -> {+ nestedWaitStarted.run();+ subscription.awaitChange(observed, nanos);+ }).maxWaitSeconds(30).build();}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/model/ExecutionStatusSubscriptionTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/model/ExecutionStatusSubscriptionTest.javanew file mode 100644index 0000000000..eb2f34e116--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/model/ExecutionStatusSubscriptionTest.java@@ -0,0 +1,154 @@+package io.github.hectorvent.floci.services.stepfunctions.model;++import com.fasterxml.jackson.databind.ObjectMapper;+import org.junit.jupiter.api.Test;++import java.lang.reflect.Field;+import java.time.Duration;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;++import static org.awaitility.Awaitility.await;+import static org.junit.jupiter.api.Assertions.*;++class ExecutionStatusSubscriptionTest {+ @Test+ void completionBeforeRegistrationDoesNotPark() throws Exception {+ Execution child = new Execution();+ child.setStatus("SUCCEEDED");+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, null)) {+ assertTrue(subscription.version() > 0);+ assertNull(registrations(child));+ }+ }++ @Test+ void completionBetweenStateReadAndParkingCannotBeLost() throws Exception {+ Execution child = new Execution();+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, null)) {+ long observed = subscription.version();+ child.setOutput("{\"answer\":42}");+ child.setStatus("SUCCEEDED");+ assertTimeoutPreemptively(Duration.ofSeconds(1),+ () -> assertTrue(subscription.awaitChange(observed, TimeUnit.MINUTES.toNanos(1))));+ assertEquals("{\"answer\":42}", child.getOutput());+ assertNull(registrations(child));+ }+ }++ @Test+ void parentTerminationWakesTheWaitAndCloseReleasesBothRegistrations() throws Exception {+ Execution child = new Execution();+ Execution parent = new Execution();+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, parent);+ ExecutorService executor = Executors.newSingleThreadExecutor()) {+ long observed = subscription.version();+ CountDownLatch entered = new CountDownLatch(1);+ CompletableFuture<Boolean> result = CompletableFuture.supplyAsync(() -> {+ entered.countDown();+ try {+ return subscription.awaitChange(observed, TimeUnit.SECONDS.toNanos(5));+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AssertionError(e);+ }+ }, executor);+ assertTrue(entered.await(1, TimeUnit.SECONDS));+ parent.setStatus("ABORTED");+ assertTrue(result.get(1, TimeUnit.SECONDS));+ assertEquals("RUNNING", child.getStatus());+ }+ assertNull(registrations(child));+ assertNull(registrations(parent));+ }++ @Test+ void unrelatedExecutionDoesNotChangeTheObservedVersion() throws Exception {+ Execution child = new Execution();+ Execution parent = new Execution();+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, parent)) {+ long observed = subscription.version();+ new Execution().setStatus("SUCCEEDED");+ assertEquals(observed, subscription.version());+ assertFalse(subscription.awaitChange(observed, 0));+ }+ }++ @Test+ void timeoutAndCancellationDoNotManufactureTerminalStatus() throws Exception {+ Execution child = new Execution();+ Execution parent = new Execution();+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, parent)) {+ long observed = subscription.version();+ assertFalse(subscription.awaitChange(observed, TimeUnit.MILLISECONDS.toNanos(1)));+ subscription.cancel();+ assertTrue(subscription.cancelled());+ assertTrue(subscription.awaitChange(observed, TimeUnit.MINUTES.toNanos(1)));+ assertEquals("RUNNING", child.getStatus());+ assertEquals("RUNNING", parent.getStatus());+ }+ assertNull(registrations(child));+ assertNull(registrations(parent));+ }++ @Test+ void wakeWithoutStatusChangeReturnsToTheExecutorsDeadlineClock() throws Exception {+ Execution child = new Execution();+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, null);+ ExecutorService executor = Executors.newSingleThreadExecutor()) {+ AtomicReference<Thread> waiter = new AtomicReference<>();+ CompletableFuture<Boolean> result = CompletableFuture.supplyAsync(() -> {+ waiter.set(Thread.currentThread());+ try {+ return subscription.awaitChange(subscription.version(), TimeUnit.MINUTES.toNanos(1));+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AssertionError(e);+ }+ }, executor);+ try {+ await().atMost(Duration.ofSeconds(2)).until(() -> waiter.get() != null+ && waiter.get().getState() == Thread.State.TIMED_WAITING);+ synchronized (subscription) {+ subscription.notifyAll();+ }+ assertFalse(result.get(1, TimeUnit.SECONDS));+ assertEquals("RUNNING", child.getStatus());+ } finally {+ subscription.cancel();+ }+ }+ }++ @Test+ void duplicateTargetsAndRepeatedCloseLeaveNoRegistration() throws Exception {+ Execution child = new Execution();+ ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, child);+ subscription.close();+ subscription.close();+ assertNull(registrations(child));+ }++ @Test+ void ephemeralRegistrationsAreAbsentFromPersistedExecution() throws Exception {+ Execution child = new Execution();+ ObjectMapper mapper = new ObjectMapper();+ String before = mapper.writeValueAsString(child);+ try (ExecutionStatusSubscription subscription = new ExecutionStatusSubscription(child, null)) {+ assertEquals(before, mapper.writeValueAsString(child));+ Execution restored = mapper.readValue(before, Execution.class);+ assertNull(registrations(restored));+ assertEquals("RUNNING", restored.getStatus());+ }+ }++ private static Object registrations(Execution execution) throws Exception {+ Field field = Execution.class.getDeclaredField("statusSubscriptions");+ field.setAccessible(true);+ return field.get(execution);+ }+}