Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): end the invocation when a checkpoint closes a durable execution (#5299)

TATSUNO “Taz” Yasuhiroebd364e23e8c
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
index 817a88910e..c5800aa1a1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
@@ -146,6 +146,10 @@ public class LambdaExecutorService implements Resettable {
return invoke(fn, payload, type, 0);
}
+ public InvokeResult invoke(LambdaFunction fn, byte[] payload, InvocationType type, String requestId) {
+ return invoke(fn, payload, type, 0, null, null, requestId);
+ }
+
/**
* Invokes {@code fn}, carrying the number of invocations that the same originating event has
* already caused. A direct invoke starts at zero; each destination delivery adds one, whether
@@ -167,8 +171,11 @@ public class LambdaExecutorService implements Resettable {
InvokeResult invoke(LambdaFunction fn, byte[] payload, InvocationType type, int chainDepth,
String invokedQualifier, String clientContext) {
- String requestId = UUID.randomUUID().toString();
+ return invoke(fn, payload, type, chainDepth, invokedQualifier, clientContext, UUID.randomUUID().toString());
+ }
+ private InvokeResult invoke(LambdaFunction fn, byte[] payload, InvocationType type, int chainDepth,
+ String invokedQualifier, String clientContext, String requestId) {
if (type == InvocationType.DryRun) {
return new InvokeResult(204, null, new byte[0], null, requestId);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java
index d49dc086f0..8982346312 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java
@@ -20,6 +20,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Consumer;
+import java.util.function.Function;
import java.util.regex.Pattern;
/**
@@ -37,18 +38,31 @@ final class DurableCheckpointApplier {
private DurableCheckpointApplier() {
}
- /** {@code chainedInvokes} names the CHAINED_INVOKE operations this batch started, for the service to run. */
- record Outcome(boolean closed, List<String> chainedInvokes) {
+ /**
+ * A non-null {@code closingStatus} means the batch closed the execution, which the service then
+ * closes. {@code chainedInvokes} names the CHAINED_INVOKE operations this batch started, for the
+ * service to run. {@code unreachableChainedInvokes} holds those whose target cannot be invoked,
+ * with the error that fails them, in batch order.
+ */
+ record Outcome(DurableExecutionStatus closingStatus, String closingResult, DurableErrorObject closingError,
+ List<String> chainedInvokes, Map<String, AwsException> unreachableChainedInvokes) {
+
+ boolean closed() {
+ return closingStatus != null;
+ }
}
- static Outcome apply(DurableExecution execution, List<DurableOperationUpdate> updates, long now) {
+ /** {@code unreachableTarget} gives the error for a chained target that cannot be invoked, or null. */
+ static Outcome apply(DurableExecution execution, List<DurableOperationUpdate> updates, long now,
+ Function<String, AwsException> unreachableTarget) {
validateBatch(updates);
- Draft draft = new Draft(execution, now);
+ Draft draft = new Draft(execution, now, unreachableTarget);
for (DurableOperationUpdate update : updates) {
draft.apply(update);
}
draft.commit();
- return new Outcome(draft.closingStatus != null, List.copyOf(draft.chainedInvokes));
+ return new Outcome(draft.closingStatus, draft.closingResult, draft.closingError,
+ List.copyOf(draft.chainedInvokes), draft.unreachableChainedInvokes);
}
/** Completes waits, step retries and callback timeouts whose time has come. The sweeper and every checkpoint call it. */
@@ -157,17 +171,20 @@ final class DurableCheckpointApplier {
private final DurableExecution execution;
private final long now;
+ private final Function<String, AwsException> unreachableTarget;
private final LinkedHashMap<String, DurableOperation> operations = new LinkedHashMap<>();
private final List<Consumer<DurableExecution>> events = new ArrayList<>();
private final List<String> chainedInvokes = new ArrayList<>();
+ private final LinkedHashMap<String, AwsException> unreachableChainedInvokes = new LinkedHashMap<>();
private long changeSequence;
private DurableExecutionStatus closingStatus;
private String closingResult;
private DurableErrorObject closingError;
- Draft(DurableExecution execution, long now) {
+ Draft(DurableExecution execution, long now, Function<String, AwsException> unreachableTarget) {
this.execution = execution;
this.now = now;
+ this.unreachableTarget = unreachableTarget;
this.changeSequence = execution.getChangeSequence();
for (Map.Entry<String, DurableOperation> entry : execution.getOperations().entrySet()) {
operations.put(entry.getKey(), entry.getValue().copy());
@@ -218,17 +235,6 @@ final class DurableCheckpointApplier {
for (Consumer<DurableExecution> event : events) {
event.accept(execution);
}
- if (closingStatus != null) {
- execution.setStatus(closingStatus);
- execution.setResult(closingResult);
- execution.setError(closingError);
- execution.setEndTimestamp(now);
- if (closingStatus == DurableExecutionStatus.SUCCEEDED) {
- DurableHistory.executionSucceeded(execution, now);
- } else {
- DurableHistory.executionEnded(execution, "ExecutionFailed", now);
- }
- }
}
private void applyExecution(DurableOperationUpdate update) {
@@ -245,12 +251,6 @@ final class DurableCheckpointApplier {
closingStatus = succeeded ? DurableExecutionStatus.SUCCEEDED : DurableExecutionStatus.FAILED;
closingResult = succeeded ? update.payload() : null;
closingError = succeeded ? null : update.error();
- DurableOperation operation = operations.get(execution.getExecutionId());
- if (operation != null) {
- operation.setStatus(succeeded ? DurableOperationStatus.SUCCEEDED : DurableOperationStatus.FAILED);
- operation.setEndTimestamp(now);
- touch(operation);
- }
}
private void applyContext(DurableOperationUpdate update, DurableOperation existing) {
@@ -393,7 +393,19 @@ final class DurableCheckpointApplier {
operation.setChainedFunctionName(update.chainedFunctionName());
operation.setChainedTenantId(update.chainedTenantId());
operation.setInputPayload(update.payload());
- chainedInvokes.add(update.id());
+ AwsException unreachable = unreachableTarget.apply(update.chainedFunctionName());
+ if (unreachable == null) {
+ chainedInvokes.add(update.id());
+ return;
+ }
+ // AWS logs the start of a target it cannot invoke in batch order, naming only the function.
+ Map<String, Object> details = new LinkedHashMap<>();
+ details.put("FunctionName", update.chainedFunctionName());
+ if (update.chainedTenantId() != null) {
+ details.put("TenantId", update.chainedTenantId());
+ }
+ event(operation, "ChainedInvokeStarted", details);
+ unreachableChainedInvokes.put(update.id(), unreachable);
}
/** A name that does not parse is left to the target lookup, which fails the operation. */
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java
index b792a9669e..b665e4e40f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java
@@ -35,6 +35,7 @@ import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -122,7 +123,10 @@ public class DurableExecutionService implements Resettable {
Long startedBefore, boolean reverseOrder, Integer maxItems, String marker) {
}
- /** The token is null when the batch closed the execution. The SDK reads that as "completed". */
+ /**
+ * The token is null and the operations are empty when the batch closed the execution. The SDK
+ * reads that as "completed".
+ */
public record CheckpointResult(String checkpointToken, List<DurableOperation> newExecutionState) {
}
@@ -247,11 +251,25 @@ public class DurableExecutionService implements Resettable {
}
DurableExecution execution = loadWithCurrentToken(arn, checkpointToken);
long now = clock.millis();
- DurableCheckpointApplier.Outcome outcome = DurableCheckpointApplier.apply(execution, updates, now);
+ Map<String, ResolvedDurableTarget> targets = new HashMap<>();
+ DurableCheckpointApplier.Outcome outcome = DurableCheckpointApplier.apply(execution, updates, now,
+ functionName -> unreachableTarget(execution, functionName, targets));
+ for (Map.Entry<String, AwsException> unreachable : outcome.unreachableChainedInvokes().entrySet()) {
+ completeChainedInvoke(execution, execution.getOperations().get(unreachable.getKey()),
+ ChainedOutcome.failed(unreachable.getValue()), now, effects);
+ }
+ if (outcome.closed()) {
+ // AWS ends the invocation here and ignores what the handler returns after it.
+ DurableHistory.invocationCompleted(execution, execution.getCurrentInvocationStartedAt(), now,
+ execution.getCurrentInvocationId(), null);
+ execution.setCurrentInvocationId(null);
+ }
// AWS runs a target started in the batch that closes the execution, and ignores its result.
List<DurableExecution> children = new ArrayList<>();
for (String operationId : outcome.chainedInvokes()) {
- startChainedInvoke(execution, execution.getOperations().get(operationId), now, children, effects);
+ DurableOperation operation = execution.getOperations().get(operationId);
+ startChainedInvoke(execution, operation, targets.get(operation.getChainedFunctionName()), now,
+ children, effects);
}
if (!outcome.closed()) {
DurableCheckpointApplier.fireDueTimers(execution, now);
@@ -259,7 +277,8 @@ public class DurableExecutionService implements Resettable {
List<DurableOperation> changed = unseenOperations(execution);
String nextToken = null;
if (outcome.closed()) {
- closeBookkeeping(execution, now, effects);
+ close(execution, outcome.closingStatus(), outcome.closingResult(), outcome.closingError(), now,
+ effects);
} else {
execution.setCheckpointSequence(execution.getCheckpointSequence() + 1);
nextToken = DurableTokens.checkpointToken(execution.getExecutionArn(),
@@ -274,7 +293,7 @@ public class DurableExecutionService implements Resettable {
for (DurableExecution child : children) {
save(child);
}
- result = new CheckpointResult(nextToken, changed);
+ result = new CheckpointResult(nextToken, outcome.closed() ? List.of() : changed);
}
runEffects(effects);
return result;
@@ -537,6 +556,7 @@ public class DurableExecutionService implements Resettable {
return;
}
startedAt = clock.millis();
+ execution.setCurrentInvocationStartedAt(startedAt);
String token = DurableTokens.checkpointToken(execution.getExecutionArn(), invocationId,
execution.getCheckpointSequence());
List<String> updatedOperationIds = unseenOperations(execution).stream().map(DurableOperation::getId).toList();
@@ -558,7 +578,7 @@ public class DurableExecutionService implements Resettable {
result = new DurableInvocationResult(null, null, "ResourceNotFoundException");
} else {
try {
- result = invoker.invoke(target, payload);
+ result = invoker.invoke(target, payload, invocationId);
} catch (AwsException e) {
result = new DurableInvocationResult(null, DurableWire.functionErrorPayload(
DurableErrorObject.of(e.getMessage(), e.getErrorCode())),
@@ -719,13 +739,17 @@ public class DurableExecutionService implements Resettable {
root.setEndTimestamp(now);
root.setChangeSequence(execution.nextChangeSequence());
}
- switch (status) {
+ closeEvent(execution, now);
+ closeBookkeeping(execution, now, effects);
+ }
+
+ private static void closeEvent(DurableExecution execution, long now) {
+ switch (execution.getStatus()) {
case SUCCEEDED -> DurableHistory.executionSucceeded(execution, now);
case TIMED_OUT -> DurableHistory.executionTimedOut(execution, now);
case STOPPED -> DurableHistory.executionEnded(execution, "ExecutionStopped", now);
default -> DurableHistory.executionEnded(execution, "ExecutionFailed", now);
}
- closeBookkeeping(execution, now, effects);
}
/** What every close shares, whether the applier or the service decided it. */
@@ -765,25 +789,25 @@ public class DurableExecutionService implements Resettable {
effects.add(() -> finishChainedInvoke(parent.accountId(), parent.storeKey(), operationId, childArn, outcome));
}
- /**
- * Resolves the target of a CHAINED_INVOKE this checkpoint started. A target that cannot be
- * invoked fails the operation at once, and ChainedInvokeStarted then names only the function.
- */
- private void startChainedInvoke(DurableExecution execution, DurableOperation operation, long now,
- List<DurableExecution> children, List<Runnable> effects) {
+ /** Null when the target can be invoked. The resolved target is kept in {@code targets} for the start. */
+ private AwsException unreachableTarget(DurableExecution execution, String functionName,
+ Map<String, ResolvedDurableTarget> targets) {
+ try {
+ targets.put(functionName, resolveChainedTarget(execution, functionName));
+ return null;
+ } catch (AwsException e) {
+ return e;
+ }
+ }
+
+ private void startChainedInvoke(DurableExecution execution, DurableOperation operation,
+ ResolvedDurableTarget target, long now, List<DurableExecution> children,
+ List<Runnable> effects) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("FunctionName", operation.getChainedFunctionName());
if (operation.getChainedTenantId() != null) {
details.put("TenantId", operation.getChainedTenantId());
}
- ResolvedDurableTarget target;
- try {
- target = resolveChainedTarget(execution, operation.getChainedFunctionName());
- } catch (AwsException e) {
- DurableHistory.operationEvent(execution, operation, "ChainedInvokeStarted", now, details);
- completeChainedInvoke(execution, operation, ChainedOutcome.failed(e), now, effects);
- return;
- }
details.put("Input", DurableHistory.payloadWrapper(operation.getInputPayload()));
details.put("ExecutedVersion", target.version());
if (target.durable()) {
@@ -854,7 +878,7 @@ public class DurableExecutionService implements Resettable {
private ChainedOutcome invokePlainFunction(ResolvedDurableTarget target, byte[] payload) {
DurableInvocationResult result;
try {
- result = invoker.invoke(target, payload);
+ result = invoker.invoke(target, payload, UUID.randomUUID().toString());
} catch (AwsException e) {
return ChainedOutcome.failed(e);
} catch (RuntimeException e) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableFunctionInvoker.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableFunctionInvoker.java
index c0e1645de5..1522d80597 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableFunctionInvoker.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableFunctionInvoker.java
@@ -10,7 +10,7 @@ public interface DurableFunctionInvoker {
/** The function a durable execution runs on, or an AwsException when the name does not resolve. */
ResolvedDurableTarget resolve(String accountId, String region, String functionName, String qualifier);
- DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload);
+ DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload, String requestId);
record ResolvedDurableTarget(String accountId,
String region,
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionInvoker.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionInvoker.java
index 3fbaddc0c5..b5462b9549 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionInvoker.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionInvoker.java
@@ -38,11 +38,11 @@ public class LambdaDurableFunctionInvoker implements DurableFunctionInvoker {
* {@code $LATEST} execution resumes on whatever code $LATEST holds now, as on AWS.
*/
@Override
- public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {
+ public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload, String requestId) {
LambdaFunction fn = targetResolver.resolveInvokeTargetForAccount(target.accountId(), target.region(),
target.functionName(), target.version());
InvokeResult result = RequestScopes.callAs(target.accountId(),
- () -> executorService.invoke(fn, payload, InvocationType.RequestResponse));
+ () -> executorService.invoke(fn, payload, InvocationType.RequestResponse, requestId));
return new DurableInvocationResult(result.getRequestId(), result.getPayload(), result.getFunctionError());
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java
index 9ab5b422f4..e57b7495bd 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java
@@ -48,6 +48,7 @@ public class DurableExecution {
private long seenSequence;
private long checkpointSequence;
private String currentInvocationId;
+ private Long currentInvocationStartedAt;
private boolean reinvokeRequested;
private int consecutiveInvocationFailures;
private Long nextInvocationAttemptAt;
@@ -189,6 +190,11 @@ public class DurableExecution {
public String getCurrentInvocationId() { return currentInvocationId; }
public void setCurrentInvocationId(String currentInvocationId) { this.currentInvocationId = currentInvocationId; }
+ public Long getCurrentInvocationStartedAt() { return currentInvocationStartedAt; }
+ public void setCurrentInvocationStartedAt(Long currentInvocationStartedAt) {
+ this.currentInvocationStartedAt = currentInvocationStartedAt;
+ }
+
public boolean isReinvokeRequested() { return reinvokeRequested; }
public void setReinvokeRequested(boolean reinvokeRequested) { this.reinvokeRequested = reinvokeRequested; }
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java
index adf25fb43f..699da6574b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java
@@ -12,6 +12,7 @@ import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -20,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
class DurableCheckpointApplierTest {
private static final long NOW = 1_700_000_000_000L;
+ private static final Function<String, AwsException> REACHABLE = functionName -> null;
private static final String EXECUTION_ARN =
"arn:aws:lambda:us-east-1:000000000000:function:fn:1/durable-execution/exec/exec-id";
@@ -56,7 +58,7 @@ class DurableCheckpointApplierTest {
"Cannot update the same operation twice in a single request.");
DurableExecution execution = execution();
DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null), update("s1", null,
- DurableOperationType.STEP, DurableOperationAction.SUCCEED, "1")), NOW);
+ DurableOperationType.STEP, DurableOperationAction.SUCCEED, "1")), NOW, REACHABLE);
assertEquals(DurableOperationStatus.SUCCEEDED, execution.getOperations().get("s1").getStatus());
}
@@ -64,7 +66,7 @@ class DurableCheckpointApplierTest {
void parentMustBeAContextAndMetadataMustStayConsistent() {
assertRejected(List.of(stepStart("s1", "missing")), "Invalid parent operation id.");
DurableExecution execution = execution();
- DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null)), NOW);
+ DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null)), NOW, REACHABLE);
assertEquals("Inconsistent operation type.", reject(execution, List.of(waitStart("s1"))));
assertEquals("Inconsistent parent operation id.", reject(execution, List.of(stepStart("ctx", null),
update("s1", "ctx", DurableOperationType.STEP, DurableOperationAction.SUCCEED, "1"))));
@@ -73,7 +75,7 @@ class DurableCheckpointApplierTest {
@Test
void stepTransitionsAreGuarded() {
DurableExecution execution = execution();
- DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null)), NOW);
+ DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null)), NOW, REACHABLE);
assertEquals("Invalid current STEP state to start.", reject(execution, List.of(stepStart("s1", null))));
assertEquals("Cannot provide an Error for SUCCEED action.", reject(execution, List.of(
new DurableOperationUpdate("s1", null, null, DurableOperationType.STEP, null,
@@ -81,7 +83,7 @@ class DurableCheckpointApplierTest {
assertEquals("Invalid StepOptions for the given action.", reject(execution, List.of(
update("s1", null, DurableOperationType.STEP, DurableOperationAction.RETRY, null))));
DurableCheckpointApplier.apply(execution, List.of(update("s1", null, DurableOperationType.STEP,
- DurableOperationAction.FAIL, null)), NOW);
+ DurableOperationAction.FAIL, null)), NOW, REACHABLE);
assertEquals("Invalid current STEP state to close.", reject(execution, List.of(
update("s1", null, DurableOperationType.STEP, DurableOperationAction.SUCCEED, "1"))));
}
@@ -93,10 +95,10 @@ class DurableCheckpointApplierTest {
assertRejected(List.of(update("w1", null, DurableOperationType.WAIT, DurableOperationAction.CANCEL, null)),
"Cannot cancel a WAIT that does not exist or has already completed.");
DurableExecution execution = execution();
- DurableCheckpointApplier.apply(execution, List.of(waitStart("w1")), NOW);
+ DurableCheckpointApplier.apply(execution, List.of(waitStart("w1")), NOW, REACHABLE);
assertEquals("Cannot start a WAIT that already exist.", reject(execution, List.of(waitStart("w1"))));
DurableCheckpointApplier.apply(execution, List.of(update("w1", null, DurableOperationType.WAIT,
- DurableOperationAction.CANCEL, null)), NOW);
+ DurableOperationAction.CANCEL, null)), NOW, REACHABLE);
assertEquals(DurableOperationStatus.CANCELLED, execution.getOperations().get("w1").getStatus());
}
@@ -131,7 +133,7 @@ class DurableCheckpointApplierTest {
DurableExecution execution = execution();
DurableCheckpointApplier.Outcome outcome = DurableCheckpointApplier.apply(execution,
List.of(chainedStart("i1", "arn:aws:lambda:us-east-1:000000000000:function:target-fn",
- "\"" + "x".repeat(300 * 1024) + "\"")), NOW);
+ "\"" + "x".repeat(300 * 1024) + "\"")), NOW, REACHABLE);
assertEquals(List.of("i1"), outcome.chainedInvokes(), "the service runs what the batch started");
DurableOperation invoke = execution.getOperations().get("i1");
assertEquals(DurableOperationStatus.STARTED, invoke.getStatus());
@@ -143,7 +145,7 @@ class DurableCheckpointApplierTest {
@Test
void aCallbackStartsWithAnIdThatNamesItsExecution() {
DurableExecution execution = execution();
- DurableCheckpointApplier.apply(execution, List.of(callbackStart("c1", 30, 10)), NOW);
+ DurableCheckpointApplier.apply(execution, List.of(callbackStart("c1", 30, 10)), NOW, REACHABLE);
DurableOperation callback = execution.getOperations().get("c1");
assertEquals(DurableOperationStatus.STARTED, callback.getStatus());
@@ -162,7 +164,7 @@ class DurableCheckpointApplierTest {
void aCallbackTimesOutOnTheDeadlineItMissesFirst() {
DurableExecution execution = execution();
DurableCheckpointApplier.apply(execution, List.of(callbackStart("slow", 5, 0), callbackStart("silent", 60, 3)),
- NOW);
+ NOW, REACHABLE);
DurableCheckpointApplier.fireDueTimers(execution, NOW + 3_000);
DurableOperation silent = execution.getOperations().get("silent");
@@ -197,7 +199,7 @@ class DurableCheckpointApplierTest {
private static String reject(DurableExecution execution, List<DurableOperationUpdate> updates) {
AwsException rejected = assertThrows(AwsException.class,
- () -> DurableCheckpointApplier.apply(execution, updates, NOW));
+ () -> DurableCheckpointApplier.apply(execution, updates, NOW, REACHABLE));
assertEquals(400, rejected.getHttpStatus());
assertEquals("InvalidParameterValueException", rejected.getErrorCode());
return rejected.getMessage();
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java
index 291dded476..a3e44db74b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java
@@ -373,7 +373,7 @@ class DurableExecutionServicePersistenceTest {
}
@Override
- public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {
+ public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload, String requestId) {
if (!target.durable()) {
plainPayloads.add(new String(payload, StandardCharsets.UTF_8));
return new DurableInvocationResult("req", plainOutput.getBytes(StandardCharsets.UTF_8), null);
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java
index db60cd679f..128ceed0eb 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java
@@ -207,6 +207,26 @@ class DurableExecutionServiceTest {
assertEquals("\"r\"", execution.getResult(), "the checkpointed result wins over the handler return");
}
+ @Test
+ void aClosingCheckpointEndsTheInvocationAndTheHandlerReturnIsIgnored() {
+ List<CheckpointResult> closing = new ArrayList<>();
+ invoker.script(event -> {
+ closing.add(checkpoint(event, token(event), List.of(step("s1", DurableOperationAction.START, null, null),
+ executionSucceed("\"closed\""))));
+ return functionError("{\"errorMessage\":\"after close\"}");
+ });
+
+ DurableExecution execution = service.get(start("exec-1", "{}", false).getExecutionArn());
+
+ assertEquals(DurableExecutionStatus.SUCCEEDED, execution.getStatus());
+ assertEquals("\"closed\"", execution.getResult());
+ assertEquals(List.of("ExecutionStarted", "StepStarted", "InvocationCompleted", "ExecutionSucceeded"),
+ eventTypes(execution));
+ Map<?, ?> completed = (Map<?, ?>) execution.getHistory().get(2).getDetails().get("InvocationCompletedDetails");
+ assertEquals(invoker.requestIds.get(0), completed.get("RequestId"));
+ assertTrue(closing.get(0).newExecutionState().isEmpty());
+ }
+
@Test
void aRetriedCheckpointWithTheSameClientTokenGetsTheSameAnswer() {
invoker.script(event -> {
@@ -794,11 +814,27 @@ class DurableExecutionServiceTest {
assertEquals(List.of("{\"n\":1}"), received);
assertEquals("\"closed\"", execution.getResult());
- assertTrue(eventTypes(execution).contains("ChainedInvokeStarted"));
- assertFalse(eventTypes(execution).contains("ChainedInvokeSucceeded"));
+ assertEquals(List.of("ExecutionStarted", "InvocationCompleted", "ChainedInvokeStarted", "ExecutionSucceeded"),
+ eventTypes(execution));
assertEquals(1, invoker.events.size());
}
+ @Test
+ void aTargetThatCannotBeInvokedStartsInBatchOrderAndFailsBeforeTheInvocationEnds() {
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(step("s1", DurableOperationAction.START, null, null),
+ chainedStart("i1", "missing-fn", null), step("s2", DurableOperationAction.START, null, null),
+ executionSucceed("\"closed\"")));
+ return succeeded("");
+ });
+
+ DurableExecution execution = service.get(start("exec-1", "{}", false).getExecutionArn());
+
+ assertEquals(List.of("ExecutionStarted", "StepStarted", "ChainedInvokeStarted", "StepStarted",
+ "ChainedInvokeFailed", "InvocationCompleted", "ExecutionSucceeded"), eventTypes(execution));
+ assertEquals(DurableOperationStatus.FAILED, execution.getOperations().get("i1").getStatus());
+ }
+
@Test
void stoppingTheParentLeavesADurableChildRunning() {
invoker.childScript("child-fn", event -> {
@@ -920,6 +956,7 @@ class DurableExecutionServiceTest {
final Deque<Function<JsonNode, DurableInvocationResult>> scripts = new ArrayDeque<>();
final List<JsonNode> events = new ArrayList<>();
+ final List<String> requestIds = new ArrayList<>();
/** Durable functions other than the one under test, each with its own scripts. */
final Map<String, Deque<Function<JsonNode, DurableInvocationResult>>> childScripts = new HashMap<>();
/** Plain functions a chained invoke may call, answering the raw payload. */
@@ -956,7 +993,7 @@ class DurableExecutionServiceTest {
}
@Override
- public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {
+ public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload, String requestId) {
JsonNode event;
try {
event = MAPPER.readTree(payload);
@@ -972,6 +1009,7 @@ class DurableExecutionServiceTest {
return script != null ? script.apply(event) : pending();
}
events.add(event);
+ requestIds.add(requestId);
Function<JsonNode, DurableInvocationResult> script = scripts.poll();
if (script == null) {
return handlerResponse("{\"Status\":\"FAILED\",\"Error\":{\"ErrorMessage\":\"no script for "
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/ScriptedDurableFunctionInvoker.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/ScriptedDurableFunctionInvoker.java
index 268c67a6bc..7faa4294d3 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/ScriptedDurableFunctionInvoker.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/ScriptedDurableFunctionInvoker.java
@@ -39,7 +39,7 @@ public class ScriptedDurableFunctionInvoker implements DurableFunctionInvoker {
}
@Override
- public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {
+ public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload, String requestId) {
Invocation invocation = new Invocation(target, payload, new CompletableFuture<>());
invocations.add(invocation);
try {