Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): end the invocation when a checkpoint closes a durable execution (#5299)
TATSUNO “Taz” Yasuhiroebd364e23e8c
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.javaindex 817a88910e..c5800aa1a1 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java@@ -146,6 +146,10 @@ public class LambdaExecutorService implements Resettable {return invoke(fn, payload, type, 0);}+ public InvokeResult invoke(LambdaFunction fn, byte[] payload, InvocationType type, String requestId) {+ return invoke(fn, payload, type, 0, null, null, requestId);+ }+/*** Invokes {@code fn}, carrying the number of invocations that the same originating event has* already caused. A direct invoke starts at zero; each destination delivery adds one, whether@@ -167,8 +171,11 @@ public class LambdaExecutorService implements Resettable {InvokeResult invoke(LambdaFunction fn, byte[] payload, InvocationType type, int chainDepth,String invokedQualifier, String clientContext) {- String requestId = UUID.randomUUID().toString();+ return invoke(fn, payload, type, chainDepth, invokedQualifier, clientContext, UUID.randomUUID().toString());+ }+ private InvokeResult invoke(LambdaFunction fn, byte[] payload, InvocationType type, int chainDepth,+ String invokedQualifier, String clientContext, String requestId) {if (type == InvocationType.DryRun) {return new InvokeResult(204, null, new byte[0], null, requestId);}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.javaindex d49dc086f0..8982346312 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java@@ -20,6 +20,7 @@ import java.util.List;import java.util.Map;import java.util.Objects;import java.util.function.Consumer;+import java.util.function.Function;import java.util.regex.Pattern;/**@@ -37,18 +38,31 @@ final class DurableCheckpointApplier {private DurableCheckpointApplier() {}- /** {@code chainedInvokes} names the CHAINED_INVOKE operations this batch started, for the service to run. */- record Outcome(boolean closed, List<String> chainedInvokes) {+ /**+ * A non-null {@code closingStatus} means the batch closed the execution, which the service then+ * closes. {@code chainedInvokes} names the CHAINED_INVOKE operations this batch started, for the+ * service to run. {@code unreachableChainedInvokes} holds those whose target cannot be invoked,+ * with the error that fails them, in batch order.+ */+ record Outcome(DurableExecutionStatus closingStatus, String closingResult, DurableErrorObject closingError,+ List<String> chainedInvokes, Map<String, AwsException> unreachableChainedInvokes) {++ boolean closed() {+ return closingStatus != null;+ }}- static Outcome apply(DurableExecution execution, List<DurableOperationUpdate> updates, long now) {+ /** {@code unreachableTarget} gives the error for a chained target that cannot be invoked, or null. */+ static Outcome apply(DurableExecution execution, List<DurableOperationUpdate> updates, long now,+ Function<String, AwsException> unreachableTarget) {validateBatch(updates);- Draft draft = new Draft(execution, now);+ Draft draft = new Draft(execution, now, unreachableTarget);for (DurableOperationUpdate update : updates) {draft.apply(update);}draft.commit();- return new Outcome(draft.closingStatus != null, List.copyOf(draft.chainedInvokes));+ return new Outcome(draft.closingStatus, draft.closingResult, draft.closingError,+ List.copyOf(draft.chainedInvokes), draft.unreachableChainedInvokes);}/** Completes waits, step retries and callback timeouts whose time has come. The sweeper and every checkpoint call it. */@@ -157,17 +171,20 @@ final class DurableCheckpointApplier {private final DurableExecution execution;private final long now;+ private final Function<String, AwsException> unreachableTarget;private final LinkedHashMap<String, DurableOperation> operations = new LinkedHashMap<>();private final List<Consumer<DurableExecution>> events = new ArrayList<>();private final List<String> chainedInvokes = new ArrayList<>();+ private final LinkedHashMap<String, AwsException> unreachableChainedInvokes = new LinkedHashMap<>();private long changeSequence;private DurableExecutionStatus closingStatus;private String closingResult;private DurableErrorObject closingError;- Draft(DurableExecution execution, long now) {+ Draft(DurableExecution execution, long now, Function<String, AwsException> unreachableTarget) {this.execution = execution;this.now = now;+ this.unreachableTarget = unreachableTarget;this.changeSequence = execution.getChangeSequence();for (Map.Entry<String, DurableOperation> entry : execution.getOperations().entrySet()) {operations.put(entry.getKey(), entry.getValue().copy());@@ -218,17 +235,6 @@ final class DurableCheckpointApplier {for (Consumer<DurableExecution> event : events) {event.accept(execution);}- if (closingStatus != null) {- execution.setStatus(closingStatus);- execution.setResult(closingResult);- execution.setError(closingError);- execution.setEndTimestamp(now);- if (closingStatus == DurableExecutionStatus.SUCCEEDED) {- DurableHistory.executionSucceeded(execution, now);- } else {- DurableHistory.executionEnded(execution, "ExecutionFailed", now);- }- }}private void applyExecution(DurableOperationUpdate update) {@@ -245,12 +251,6 @@ final class DurableCheckpointApplier {closingStatus = succeeded ? DurableExecutionStatus.SUCCEEDED : DurableExecutionStatus.FAILED;closingResult = succeeded ? update.payload() : null;closingError = succeeded ? null : update.error();- DurableOperation operation = operations.get(execution.getExecutionId());- if (operation != null) {- operation.setStatus(succeeded ? DurableOperationStatus.SUCCEEDED : DurableOperationStatus.FAILED);- operation.setEndTimestamp(now);- touch(operation);- }}private void applyContext(DurableOperationUpdate update, DurableOperation existing) {@@ -393,7 +393,19 @@ final class DurableCheckpointApplier {operation.setChainedFunctionName(update.chainedFunctionName());operation.setChainedTenantId(update.chainedTenantId());operation.setInputPayload(update.payload());- chainedInvokes.add(update.id());+ AwsException unreachable = unreachableTarget.apply(update.chainedFunctionName());+ if (unreachable == null) {+ chainedInvokes.add(update.id());+ return;+ }+ // AWS logs the start of a target it cannot invoke in batch order, naming only the function.+ Map<String, Object> details = new LinkedHashMap<>();+ details.put("FunctionName", update.chainedFunctionName());+ if (update.chainedTenantId() != null) {+ details.put("TenantId", update.chainedTenantId());+ }+ event(operation, "ChainedInvokeStarted", details);+ unreachableChainedInvokes.put(update.id(), unreachable);}/** A name that does not parse is left to the target lookup, which fails the operation. */diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.javaindex b792a9669e..b665e4e40f 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java@@ -35,6 +35,7 @@ import java.time.Clock;import java.time.Duration;import java.util.ArrayList;import java.util.Comparator;+import java.util.HashMap;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;@@ -122,7 +123,10 @@ public class DurableExecutionService implements Resettable {Long startedBefore, boolean reverseOrder, Integer maxItems, String marker) {}- /** The token is null when the batch closed the execution. The SDK reads that as "completed". */+ /**+ * The token is null and the operations are empty when the batch closed the execution. The SDK+ * reads that as "completed".+ */public record CheckpointResult(String checkpointToken, List<DurableOperation> newExecutionState) {}@@ -247,11 +251,25 @@ public class DurableExecutionService implements Resettable {}DurableExecution execution = loadWithCurrentToken(arn, checkpointToken);long now = clock.millis();- DurableCheckpointApplier.Outcome outcome = DurableCheckpointApplier.apply(execution, updates, now);+ Map<String, ResolvedDurableTarget> targets = new HashMap<>();+ DurableCheckpointApplier.Outcome outcome = DurableCheckpointApplier.apply(execution, updates, now,+ functionName -> unreachableTarget(execution, functionName, targets));+ for (Map.Entry<String, AwsException> unreachable : outcome.unreachableChainedInvokes().entrySet()) {+ completeChainedInvoke(execution, execution.getOperations().get(unreachable.getKey()),+ ChainedOutcome.failed(unreachable.getValue()), now, effects);+ }+ if (outcome.closed()) {+ // AWS ends the invocation here and ignores what the handler returns after it.+ DurableHistory.invocationCompleted(execution, execution.getCurrentInvocationStartedAt(), now,+ execution.getCurrentInvocationId(), null);+ execution.setCurrentInvocationId(null);+ }// AWS runs a target started in the batch that closes the execution, and ignores its result.List<DurableExecution> children = new ArrayList<>();for (String operationId : outcome.chainedInvokes()) {- startChainedInvoke(execution, execution.getOperations().get(operationId), now, children, effects);+ DurableOperation operation = execution.getOperations().get(operationId);+ startChainedInvoke(execution, operation, targets.get(operation.getChainedFunctionName()), now,+ children, effects);}if (!outcome.closed()) {DurableCheckpointApplier.fireDueTimers(execution, now);@@ -259,7 +277,8 @@ public class DurableExecutionService implements Resettable {List<DurableOperation> changed = unseenOperations(execution);String nextToken = null;if (outcome.closed()) {- closeBookkeeping(execution, now, effects);+ close(execution, outcome.closingStatus(), outcome.closingResult(), outcome.closingError(), now,+ effects);} else {execution.setCheckpointSequence(execution.getCheckpointSequence() + 1);nextToken = DurableTokens.checkpointToken(execution.getExecutionArn(),@@ -274,7 +293,7 @@ public class DurableExecutionService implements Resettable {for (DurableExecution child : children) {save(child);}- result = new CheckpointResult(nextToken, changed);+ result = new CheckpointResult(nextToken, outcome.closed() ? List.of() : changed);}runEffects(effects);return result;@@ -537,6 +556,7 @@ public class DurableExecutionService implements Resettable {return;}startedAt = clock.millis();+ execution.setCurrentInvocationStartedAt(startedAt);String token = DurableTokens.checkpointToken(execution.getExecutionArn(), invocationId,execution.getCheckpointSequence());List<String> updatedOperationIds = unseenOperations(execution).stream().map(DurableOperation::getId).toList();@@ -558,7 +578,7 @@ public class DurableExecutionService implements Resettable {result = new DurableInvocationResult(null, null, "ResourceNotFoundException");} else {try {- result = invoker.invoke(target, payload);+ result = invoker.invoke(target, payload, invocationId);} catch (AwsException e) {result = new DurableInvocationResult(null, DurableWire.functionErrorPayload(DurableErrorObject.of(e.getMessage(), e.getErrorCode())),@@ -719,13 +739,17 @@ public class DurableExecutionService implements Resettable {root.setEndTimestamp(now);root.setChangeSequence(execution.nextChangeSequence());}- switch (status) {+ closeEvent(execution, now);+ closeBookkeeping(execution, now, effects);+ }++ private static void closeEvent(DurableExecution execution, long now) {+ switch (execution.getStatus()) {case SUCCEEDED -> DurableHistory.executionSucceeded(execution, now);case TIMED_OUT -> DurableHistory.executionTimedOut(execution, now);case STOPPED -> DurableHistory.executionEnded(execution, "ExecutionStopped", now);default -> DurableHistory.executionEnded(execution, "ExecutionFailed", now);}- closeBookkeeping(execution, now, effects);}/** What every close shares, whether the applier or the service decided it. */@@ -765,25 +789,25 @@ public class DurableExecutionService implements Resettable {effects.add(() -> finishChainedInvoke(parent.accountId(), parent.storeKey(), operationId, childArn, outcome));}- /**- * Resolves the target of a CHAINED_INVOKE this checkpoint started. A target that cannot be- * invoked fails the operation at once, and ChainedInvokeStarted then names only the function.- */- private void startChainedInvoke(DurableExecution execution, DurableOperation operation, long now,- List<DurableExecution> children, List<Runnable> effects) {+ /** Null when the target can be invoked. The resolved target is kept in {@code targets} for the start. */+ private AwsException unreachableTarget(DurableExecution execution, String functionName,+ Map<String, ResolvedDurableTarget> targets) {+ try {+ targets.put(functionName, resolveChainedTarget(execution, functionName));+ return null;+ } catch (AwsException e) {+ return e;+ }+ }++ private void startChainedInvoke(DurableExecution execution, DurableOperation operation,+ ResolvedDurableTarget target, long now, List<DurableExecution> children,+ List<Runnable> effects) {Map<String, Object> details = new LinkedHashMap<>();details.put("FunctionName", operation.getChainedFunctionName());if (operation.getChainedTenantId() != null) {details.put("TenantId", operation.getChainedTenantId());}- ResolvedDurableTarget target;- try {- target = resolveChainedTarget(execution, operation.getChainedFunctionName());- } catch (AwsException e) {- DurableHistory.operationEvent(execution, operation, "ChainedInvokeStarted", now, details);- completeChainedInvoke(execution, operation, ChainedOutcome.failed(e), now, effects);- return;- }details.put("Input", DurableHistory.payloadWrapper(operation.getInputPayload()));details.put("ExecutedVersion", target.version());if (target.durable()) {@@ -854,7 +878,7 @@ public class DurableExecutionService implements Resettable {private ChainedOutcome invokePlainFunction(ResolvedDurableTarget target, byte[] payload) {DurableInvocationResult result;try {- result = invoker.invoke(target, payload);+ result = invoker.invoke(target, payload, UUID.randomUUID().toString());} catch (AwsException e) {return ChainedOutcome.failed(e);} catch (RuntimeException e) {diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableFunctionInvoker.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableFunctionInvoker.javaindex c0e1645de5..1522d80597 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableFunctionInvoker.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableFunctionInvoker.java@@ -10,7 +10,7 @@ public interface DurableFunctionInvoker {/** The function a durable execution runs on, or an AwsException when the name does not resolve. */ResolvedDurableTarget resolve(String accountId, String region, String functionName, String qualifier);- DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload);+ DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload, String requestId);record ResolvedDurableTarget(String accountId,String region,diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionInvoker.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionInvoker.javaindex 3fbaddc0c5..b5462b9549 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionInvoker.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionInvoker.java@@ -38,11 +38,11 @@ public class LambdaDurableFunctionInvoker implements DurableFunctionInvoker {* {@code $LATEST} execution resumes on whatever code $LATEST holds now, as on AWS.*/@Override- public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {+ public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload, String requestId) {LambdaFunction fn = targetResolver.resolveInvokeTargetForAccount(target.accountId(), target.region(),target.functionName(), target.version());InvokeResult result = RequestScopes.callAs(target.accountId(),- () -> executorService.invoke(fn, payload, InvocationType.RequestResponse));+ () -> executorService.invoke(fn, payload, InvocationType.RequestResponse, requestId));return new DurableInvocationResult(result.getRequestId(), result.getPayload(), result.getFunctionError());}}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.javaindex 9ab5b422f4..e57b7495bd 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java@@ -48,6 +48,7 @@ public class DurableExecution {private long seenSequence;private long checkpointSequence;private String currentInvocationId;+ private Long currentInvocationStartedAt;private boolean reinvokeRequested;private int consecutiveInvocationFailures;private Long nextInvocationAttemptAt;@@ -189,6 +190,11 @@ public class DurableExecution {public String getCurrentInvocationId() { return currentInvocationId; }public void setCurrentInvocationId(String currentInvocationId) { this.currentInvocationId = currentInvocationId; }+ public Long getCurrentInvocationStartedAt() { return currentInvocationStartedAt; }+ public void setCurrentInvocationStartedAt(Long currentInvocationStartedAt) {+ this.currentInvocationStartedAt = currentInvocationStartedAt;+ }+public boolean isReinvokeRequested() { return reinvokeRequested; }public void setReinvokeRequested(boolean reinvokeRequested) { this.reinvokeRequested = reinvokeRequested; }diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.javaindex adf25fb43f..699da6574b 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java@@ -12,6 +12,7 @@ import org.junit.jupiter.api.Test;import java.util.List;import java.util.Map;+import java.util.function.Function;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertThrows;@@ -20,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;class DurableCheckpointApplierTest {private static final long NOW = 1_700_000_000_000L;+ private static final Function<String, AwsException> REACHABLE = functionName -> null;private static final String EXECUTION_ARN ="arn:aws:lambda:us-east-1:000000000000:function:fn:1/durable-execution/exec/exec-id";@@ -56,7 +58,7 @@ class DurableCheckpointApplierTest {"Cannot update the same operation twice in a single request.");DurableExecution execution = execution();DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null), update("s1", null,- DurableOperationType.STEP, DurableOperationAction.SUCCEED, "1")), NOW);+ DurableOperationType.STEP, DurableOperationAction.SUCCEED, "1")), NOW, REACHABLE);assertEquals(DurableOperationStatus.SUCCEEDED, execution.getOperations().get("s1").getStatus());}@@ -64,7 +66,7 @@ class DurableCheckpointApplierTest {void parentMustBeAContextAndMetadataMustStayConsistent() {assertRejected(List.of(stepStart("s1", "missing")), "Invalid parent operation id.");DurableExecution execution = execution();- DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null)), NOW);+ DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null)), NOW, REACHABLE);assertEquals("Inconsistent operation type.", reject(execution, List.of(waitStart("s1"))));assertEquals("Inconsistent parent operation id.", reject(execution, List.of(stepStart("ctx", null),update("s1", "ctx", DurableOperationType.STEP, DurableOperationAction.SUCCEED, "1"))));@@ -73,7 +75,7 @@ class DurableCheckpointApplierTest {@Testvoid stepTransitionsAreGuarded() {DurableExecution execution = execution();- DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null)), NOW);+ DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null)), NOW, REACHABLE);assertEquals("Invalid current STEP state to start.", reject(execution, List.of(stepStart("s1", null))));assertEquals("Cannot provide an Error for SUCCEED action.", reject(execution, List.of(new DurableOperationUpdate("s1", null, null, DurableOperationType.STEP, null,@@ -81,7 +83,7 @@ class DurableCheckpointApplierTest {assertEquals("Invalid StepOptions for the given action.", reject(execution, List.of(update("s1", null, DurableOperationType.STEP, DurableOperationAction.RETRY, null))));DurableCheckpointApplier.apply(execution, List.of(update("s1", null, DurableOperationType.STEP,- DurableOperationAction.FAIL, null)), NOW);+ DurableOperationAction.FAIL, null)), NOW, REACHABLE);assertEquals("Invalid current STEP state to close.", reject(execution, List.of(update("s1", null, DurableOperationType.STEP, DurableOperationAction.SUCCEED, "1"))));}@@ -93,10 +95,10 @@ class DurableCheckpointApplierTest {assertRejected(List.of(update("w1", null, DurableOperationType.WAIT, DurableOperationAction.CANCEL, null)),"Cannot cancel a WAIT that does not exist or has already completed.");DurableExecution execution = execution();- DurableCheckpointApplier.apply(execution, List.of(waitStart("w1")), NOW);+ DurableCheckpointApplier.apply(execution, List.of(waitStart("w1")), NOW, REACHABLE);assertEquals("Cannot start a WAIT that already exist.", reject(execution, List.of(waitStart("w1"))));DurableCheckpointApplier.apply(execution, List.of(update("w1", null, DurableOperationType.WAIT,- DurableOperationAction.CANCEL, null)), NOW);+ DurableOperationAction.CANCEL, null)), NOW, REACHABLE);assertEquals(DurableOperationStatus.CANCELLED, execution.getOperations().get("w1").getStatus());}@@ -131,7 +133,7 @@ class DurableCheckpointApplierTest {DurableExecution execution = execution();DurableCheckpointApplier.Outcome outcome = DurableCheckpointApplier.apply(execution,List.of(chainedStart("i1", "arn:aws:lambda:us-east-1:000000000000:function:target-fn",- "\"" + "x".repeat(300 * 1024) + "\"")), NOW);+ "\"" + "x".repeat(300 * 1024) + "\"")), NOW, REACHABLE);assertEquals(List.of("i1"), outcome.chainedInvokes(), "the service runs what the batch started");DurableOperation invoke = execution.getOperations().get("i1");assertEquals(DurableOperationStatus.STARTED, invoke.getStatus());@@ -143,7 +145,7 @@ class DurableCheckpointApplierTest {@Testvoid aCallbackStartsWithAnIdThatNamesItsExecution() {DurableExecution execution = execution();- DurableCheckpointApplier.apply(execution, List.of(callbackStart("c1", 30, 10)), NOW);+ DurableCheckpointApplier.apply(execution, List.of(callbackStart("c1", 30, 10)), NOW, REACHABLE);DurableOperation callback = execution.getOperations().get("c1");assertEquals(DurableOperationStatus.STARTED, callback.getStatus());@@ -162,7 +164,7 @@ class DurableCheckpointApplierTest {void aCallbackTimesOutOnTheDeadlineItMissesFirst() {DurableExecution execution = execution();DurableCheckpointApplier.apply(execution, List.of(callbackStart("slow", 5, 0), callbackStart("silent", 60, 3)),- NOW);+ NOW, REACHABLE);DurableCheckpointApplier.fireDueTimers(execution, NOW + 3_000);DurableOperation silent = execution.getOperations().get("silent");@@ -197,7 +199,7 @@ class DurableCheckpointApplierTest {private static String reject(DurableExecution execution, List<DurableOperationUpdate> updates) {AwsException rejected = assertThrows(AwsException.class,- () -> DurableCheckpointApplier.apply(execution, updates, NOW));+ () -> DurableCheckpointApplier.apply(execution, updates, NOW, REACHABLE));assertEquals(400, rejected.getHttpStatus());assertEquals("InvalidParameterValueException", rejected.getErrorCode());return rejected.getMessage();diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.javaindex 291dded476..a3e44db74b 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java@@ -373,7 +373,7 @@ class DurableExecutionServicePersistenceTest {}@Override- public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {+ public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload, String requestId) {if (!target.durable()) {plainPayloads.add(new String(payload, StandardCharsets.UTF_8));return new DurableInvocationResult("req", plainOutput.getBytes(StandardCharsets.UTF_8), null);diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.javaindex db60cd679f..128ceed0eb 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java@@ -207,6 +207,26 @@ class DurableExecutionServiceTest {assertEquals("\"r\"", execution.getResult(), "the checkpointed result wins over the handler return");}+ @Test+ void aClosingCheckpointEndsTheInvocationAndTheHandlerReturnIsIgnored() {+ List<CheckpointResult> closing = new ArrayList<>();+ invoker.script(event -> {+ closing.add(checkpoint(event, token(event), List.of(step("s1", DurableOperationAction.START, null, null),+ executionSucceed("\"closed\""))));+ return functionError("{\"errorMessage\":\"after close\"}");+ });++ DurableExecution execution = service.get(start("exec-1", "{}", false).getExecutionArn());++ assertEquals(DurableExecutionStatus.SUCCEEDED, execution.getStatus());+ assertEquals("\"closed\"", execution.getResult());+ assertEquals(List.of("ExecutionStarted", "StepStarted", "InvocationCompleted", "ExecutionSucceeded"),+ eventTypes(execution));+ Map<?, ?> completed = (Map<?, ?>) execution.getHistory().get(2).getDetails().get("InvocationCompletedDetails");+ assertEquals(invoker.requestIds.get(0), completed.get("RequestId"));+ assertTrue(closing.get(0).newExecutionState().isEmpty());+ }+@Testvoid aRetriedCheckpointWithTheSameClientTokenGetsTheSameAnswer() {invoker.script(event -> {@@ -794,11 +814,27 @@ class DurableExecutionServiceTest {assertEquals(List.of("{\"n\":1}"), received);assertEquals("\"closed\"", execution.getResult());- assertTrue(eventTypes(execution).contains("ChainedInvokeStarted"));- assertFalse(eventTypes(execution).contains("ChainedInvokeSucceeded"));+ assertEquals(List.of("ExecutionStarted", "InvocationCompleted", "ChainedInvokeStarted", "ExecutionSucceeded"),+ eventTypes(execution));assertEquals(1, invoker.events.size());}+ @Test+ void aTargetThatCannotBeInvokedStartsInBatchOrderAndFailsBeforeTheInvocationEnds() {+ invoker.script(event -> {+ checkpoint(event, token(event), List.of(step("s1", DurableOperationAction.START, null, null),+ chainedStart("i1", "missing-fn", null), step("s2", DurableOperationAction.START, null, null),+ executionSucceed("\"closed\"")));+ return succeeded("");+ });++ DurableExecution execution = service.get(start("exec-1", "{}", false).getExecutionArn());++ assertEquals(List.of("ExecutionStarted", "StepStarted", "ChainedInvokeStarted", "StepStarted",+ "ChainedInvokeFailed", "InvocationCompleted", "ExecutionSucceeded"), eventTypes(execution));+ assertEquals(DurableOperationStatus.FAILED, execution.getOperations().get("i1").getStatus());+ }+@Testvoid stoppingTheParentLeavesADurableChildRunning() {invoker.childScript("child-fn", event -> {@@ -920,6 +956,7 @@ class DurableExecutionServiceTest {final Deque<Function<JsonNode, DurableInvocationResult>> scripts = new ArrayDeque<>();final List<JsonNode> events = new ArrayList<>();+ final List<String> requestIds = new ArrayList<>();/** Durable functions other than the one under test, each with its own scripts. */final Map<String, Deque<Function<JsonNode, DurableInvocationResult>>> childScripts = new HashMap<>();/** Plain functions a chained invoke may call, answering the raw payload. */@@ -956,7 +993,7 @@ class DurableExecutionServiceTest {}@Override- public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {+ public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload, String requestId) {JsonNode event;try {event = MAPPER.readTree(payload);@@ -972,6 +1009,7 @@ class DurableExecutionServiceTest {return script != null ? script.apply(event) : pending();}events.add(event);+ requestIds.add(requestId);Function<JsonNode, DurableInvocationResult> script = scripts.poll();if (script == null) {return handlerResponse("{\"Status\":\"FAILED\",\"Error\":{\"ErrorMessage\":\"no script for "diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/ScriptedDurableFunctionInvoker.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/ScriptedDurableFunctionInvoker.javaindex 268c67a6bc..7faa4294d3 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/ScriptedDurableFunctionInvoker.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/ScriptedDurableFunctionInvoker.java@@ -39,7 +39,7 @@ public class ScriptedDurableFunctionInvoker implements DurableFunctionInvoker {}@Override- public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {+ public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload, String requestId) {Invocation invocation = new Invocation(target, payload, new CompletableFuture<>());invocations.add(invocation);try {