Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(lambda): durable function chained invokes (#5164)

TATSUNO “Taz” Yasuhiro6e6ce61eea6a
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java
index 5be56bd05a..20a14d167e 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java
@@ -23,6 +23,8 @@ class LambdaDurableFunctionsTest {
private static final String ROLE = "arn:aws:iam::000000000000:role/lambda-role";
private static final String FN = TestFixtures.uniqueName("fn-durable-exec");
private static final String CALLBACK_FN = TestFixtures.uniqueName("fn-durable-callback");
+ private static final String CHAIN_FN = TestFixtures.uniqueName("fn-durable-chain");
+ private static final String CHAIN_TARGET_FN = TestFixtures.uniqueName("fn-durable-chain-target");
private static LambdaClient lambda;
@@ -34,7 +36,7 @@ class LambdaDurableFunctionsTest {
@AfterAll
static void cleanup() {
if (lambda != null) {
- for (String name : new String[] {FN, CALLBACK_FN}) {
+ for (String name : new String[] {FN, CALLBACK_FN, CHAIN_FN, CHAIN_TARGET_FN}) {
try {
lambda.deleteFunction(DeleteFunctionRequest.builder().functionName(name).build());
} catch (Exception ignored) {
@@ -179,4 +181,45 @@ class LambdaDurableFunctionsTest {
SendDurableExecutionCallbackSuccessRequest.builder().callbackId(id).build()))
.isInstanceOf(CallbackTimeoutException.class);
}
+
+ @Test
+ @DisplayName("a durable function invokes another function and receives its result")
+ void durableChainedInvokeReturnsTheTargetResult() {
+ Assumptions.assumeTrue(TestFixtures.isLambdaDispatchAvailable(),
+ "skipping: Lambda dispatch (Docker) not available in this environment");
+
+ lambda.createFunction(CreateFunctionRequest.builder()
+ .functionName(CHAIN_TARGET_FN)
+ .runtime(Runtime.NODEJS20_X)
+ .role(ROLE)
+ .handler("index.handler")
+ .code(FunctionCode.builder().zipFile(SdkBytes.fromByteArray(LambdaUtils.handlerZip())).build())
+ .build());
+ lambda.createFunction(CreateFunctionRequest.builder()
+ .functionName(CHAIN_FN)
+ .runtime(Runtime.PYTHON3_14)
+ .role(ROLE)
+ .handler("lambda_function.handler")
+ .timeout(30)
+ .durableConfig(DurableConfig.builder().executionTimeout(120).retentionPeriodInDays(1).build())
+ .code(FunctionCode.builder().zipFile(SdkBytes.fromByteArray(LambdaUtils.durablePythonZip())).build())
+ .build());
+
+ InvokeResponse invoked = lambda.invoke(InvokeRequest.builder()
+ .functionName(CHAIN_FN + ":$LATEST")
+ .payload(SdkBytes.fromUtf8String("{\"chain\": \"" + CHAIN_TARGET_FN + "\"}"))
+ .build());
+
+ assertThat(invoked.functionError()).isNull();
+ assertThat(invoked.payload().asUtf8String()).contains("Hello, Durable!");
+ GetDurableExecutionHistoryResponse history = lambda.getDurableExecutionHistory(
+ GetDurableExecutionHistoryRequest.builder().durableExecutionArn(invoked.durableExecutionArn()).build());
+ assertThat(history.events()).extracting(Event::eventType)
+ .contains(EventType.CHAINED_INVOKE_STARTED, EventType.CHAINED_INVOKE_SUCCEEDED);
+ Event started = history.events().stream()
+ .filter(event -> event.eventType() == EventType.CHAINED_INVOKE_STARTED)
+ .findFirst().orElseThrow();
+ assertThat(started.chainedInvokeStartedDetails().functionName()).isEqualTo(CHAIN_TARGET_FN);
+ assertThat(started.chainedInvokeStartedDetails().executedVersion()).isEqualTo("$LATEST");
+ }
}
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java
index 67d3545d50..8ec6954f46 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java
@@ -284,6 +284,22 @@ public final class LambdaUtils {
if request.get("fail"):
return {"Status": "FAILED",
"Error": {"ErrorMessage": "asked to fail", "ErrorType": "TestFailure"}}
+ if request.get("chain"):
+ if "chain" not in operations:
+ client.checkpoint_durable_execution(
+ DurableExecutionArn=event["DurableExecutionArn"],
+ CheckpointToken=event["CheckpointToken"],
+ Updates=[{"Id": "chain", "Name": "greet", "Type": "CHAINED_INVOKE",
+ "SubType": "ChainedInvoke", "Action": "START",
+ "Payload": json.dumps({"name": "Durable"}),
+ "ChainedInvokeOptions": {"FunctionName": request["chain"]}}])
+ return {"Status": "PENDING"}
+ chain = operations["chain"]
+ if chain["Status"] == "STARTED":
+ return {"Status": "PENDING"}
+ details = chain.get("ChainedInvokeDetails", {})
+ return {"Status": "SUCCEEDED",
+ "Result": details.get("Result") or json.dumps(details.get("Error"))}
if request.get("callback"):
if "callback" not in operations:
client.checkpoint_durable_execution(
diff --git a/docs/services/lambda.md b/docs/services/lambda.md
index f6a3f1f6ea..6aee0a6c8c 100644
--- a/docs/services/lambda.md
+++ b/docs/services/lambda.md
@@ -168,7 +168,7 @@ background. Invocations through an ARN (EventBridge, Scheduler, function URLs, d
durable executions the same way.
The checkpoint protocol under `/2025-12-01` is the real one, so the Durable Execution SDK works
-unchanged for steps, waits, step retries, callbacks and replay. Timers are persisted and fired by a
+unchanged for steps, waits, step retries, callbacks, chained invokes and replay. Timers are persisted and fired by a
background sweep, so an execution survives a restart of Floci. A function that returns something
other than the `{Status, Result, Error}` envelope fails its execution with
`Invalid Status in invocation output.` A function that throws is re-invoked four more times, after
@@ -182,6 +182,13 @@ timeout. A failure sent with an empty request body fails the callback but does n
function again, as on AWS, so the execution keeps waiting. The SDKs send `{}` when `Error` is
omitted, which does invoke it.
+A chained invoke (`context.invoke`) runs another function with an input of up to 1 MB. A durable
+target needs a version or alias and runs as its own durable execution, and its close completes the
+operation as `SUCCEEDED`, `FAILED`, `TIMED_OUT` or `STOPPED`. Any other function is invoked once,
+and an output over 1 MB fails the operation. A missing function or an unqualified durable one
+fails the operation at once, and a target in another account or Region is rejected at checkpoint.
+Stopping the parent leaves the child running, as on AWS.
+
`GetDurableExecutionHistory` leaves payloads out unless `IncludeExecutionData=true` is sent. The
API reference names `true` as the default, but AWS answers this way.
@@ -194,7 +201,8 @@ package.
Not emulated yet:
-- Chained invokes (`context.invoke`). They are rejected at checkpoint time.
+- The execution role's `lambda:InvokeFunction` permission before a chained invoke. `TenantId` is
+ recorded in the history only.
- Durable executions started by Step Functions or by an event source mapping. These invoke the
function as a plain one.
- Payload encryption. `KMSKeyArn` is stored and returned only.
@@ -399,7 +407,6 @@ These AWS Lambda operations have no handler in Floci. Calls will return `404` or
- Layer permissions (`AddLayerVersionPermission`, `RemoveLayerVersionPermission`, `GetLayerVersionPolicy`)
- Provisioned concurrency (`PutProvisionedConcurrencyConfig`, `GetProvisionedConcurrencyConfig`, `ListProvisionedConcurrencyConfigs`, `DeleteProvisionedConcurrencyConfig`)
- `InvokeWithResponseStream`
-- Durable function chained invokes. See Durable Functions
- Code signing enforcement. A configuration is created, read, updated, deleted and listed, and
nothing verifies a signature against it, so it never gates a deployment. Attaching one to a
function is not wired either: there is no `PutFunctionCodeSigningConfig`, so
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java
index 0051ccb852..d49dc086f0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.lambda.durable;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.lambda.LambdaArnUtils;
import io.github.hectorvent.floci.services.lambda.durable.model.DurableErrorObject;
import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecution;
import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecutionStatus;
@@ -30,12 +31,14 @@ final class DurableCheckpointApplier {
static final int MAX_OPERATION_PAYLOAD_BYTES = 256 * 1024;
static final int MAX_ERROR_BYTES = 32 * 1024;
+ static final int MAX_CHAINED_INVOKE_PAYLOAD_BYTES = 1024 * 1024;
private static final Pattern OPERATION_ID = Pattern.compile("[a-zA-Z0-9-_]{1,64}");
private DurableCheckpointApplier() {
}
- record Outcome(boolean closed) {
+ /** {@code chainedInvokes} names the CHAINED_INVOKE operations this batch started, for the service to run. */
+ record Outcome(boolean closed, List<String> chainedInvokes) {
}
static Outcome apply(DurableExecution execution, List<DurableOperationUpdate> updates, long now) {
@@ -45,7 +48,7 @@ final class DurableCheckpointApplier {
draft.apply(update);
}
draft.commit();
- return new Outcome(draft.closingStatus != null);
+ return new Outcome(draft.closingStatus != null, List.copyOf(draft.chainedInvokes));
}
/** Completes waits, step retries and callback timeouts whose time has come. The sweeper and every checkpoint call it. */
@@ -156,6 +159,7 @@ final class DurableCheckpointApplier {
private final long now;
private final LinkedHashMap<String, DurableOperation> operations = new LinkedHashMap<>();
private final List<Consumer<DurableExecution>> events = new ArrayList<>();
+ private final List<String> chainedInvokes = new ArrayList<>();
private long changeSequence;
private DurableExecutionStatus closingStatus;
private String closingResult;
@@ -189,7 +193,12 @@ final class DurableCheckpointApplier {
}
}
requireErrorSize(update);
- if (utf8Length(update.payload()) > MAX_OPERATION_PAYLOAD_BYTES) {
+ if (update.type() == DurableOperationType.CHAINED_INVOKE) {
+ if (utf8Length(update.payload()) > MAX_CHAINED_INVOKE_PAYLOAD_BYTES) {
+ throw invalid("CHAINED_INVOKE input payload size must be less than or equal to "
+ + MAX_CHAINED_INVOKE_PAYLOAD_BYTES + " bytes.");
+ }
+ } else if (utf8Length(update.payload()) > MAX_OPERATION_PAYLOAD_BYTES) {
throw invalid(update.type() + " payload size must be less than or equal to "
+ MAX_OPERATION_PAYLOAD_BYTES + " bytes.");
}
@@ -198,7 +207,7 @@ final class DurableCheckpointApplier {
case STEP -> applyStep(update, existing);
case WAIT -> applyWait(update, existing);
case CALLBACK -> applyCallback(update, existing);
- case CHAINED_INVOKE -> throw invalid(update.type() + " operations are not supported yet");
+ case CHAINED_INVOKE -> applyChainedInvoke(update, existing);
default -> throw invalid("Unknown operation type.");
}
}
@@ -368,6 +377,41 @@ final class DurableCheckpointApplier {
}
}
+ /** The service resolves and runs the target, and records ChainedInvokeStarted with what it found. */
+ private void applyChainedInvoke(DurableOperationUpdate update, DurableOperation existing) {
+ if (update.action() != DurableOperationAction.START) {
+ throw invalid("Invalid action for the given operation type.");
+ }
+ if (update.chainedFunctionName() == null) {
+ throw invalid("Update for CHAINED_INVOKE operation requires ChainedInvokeOptions.");
+ }
+ if (existing != null) {
+ throw invalid("Cannot start a CHAINED_INVOKE that already exist.");
+ }
+ requireSameAccountAndRegion(update.chainedFunctionName());
+ DurableOperation operation = create(update);
+ operation.setChainedFunctionName(update.chainedFunctionName());
+ operation.setChainedTenantId(update.chainedTenantId());
+ operation.setInputPayload(update.payload());
+ chainedInvokes.add(update.id());
+ }
+
+ /** A name that does not parse is left to the target lookup, which fails the operation. */
+ private void requireSameAccountAndRegion(String functionName) {
+ LambdaArnUtils.ResolvedFunctionRef ref;
+ try {
+ ref = LambdaArnUtils.resolve(functionName);
+ } catch (AwsException e) {
+ return;
+ }
+ if (ref.account() != null && !ref.account().equals(execution.getAccountId())) {
+ throw invalid("Cannot start a CHAINED_INVOKE on a function in another account.");
+ }
+ if (ref.region() != null && !ref.region().equals(execution.getRegion())) {
+ throw invalid("Cannot start a CHAINED_INVOKE on a function in another region.");
+ }
+ }
+
/** Only the function starts a callback. SendDurableExecutionCallback* completes it. */
private void applyCallback(DurableOperationUpdate update, DurableOperation existing) {
if (update.action() != DurableOperationAction.START) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java
index 9a482866de..b792a9669e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java
@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.lambda.durable;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.Pagination;
import io.github.hectorvent.floci.core.common.PaginatedResult;
@@ -247,6 +248,11 @@ public class DurableExecutionService implements Resettable {
DurableExecution execution = loadWithCurrentToken(arn, checkpointToken);
long now = clock.millis();
DurableCheckpointApplier.Outcome outcome = DurableCheckpointApplier.apply(execution, updates, now);
+ // AWS runs a target started in the batch that closes the execution, and ignores its result.
+ List<DurableExecution> children = new ArrayList<>();
+ for (String operationId : outcome.chainedInvokes()) {
+ startChainedInvoke(execution, execution.getOperations().get(operationId), now, children, effects);
+ }
if (!outcome.closed()) {
DurableCheckpointApplier.fireDueTimers(execution, now);
}
@@ -264,6 +270,10 @@ public class DurableExecutionService implements Resettable {
: new DurableCheckpointReplay(checkpointToken, clientToken, nextToken, changed));
execution.setSeenSequence(execution.getChangeSequence());
save(execution);
+ // After the parent, so a crash in between leaves no child the parent does not know.
+ for (DurableExecution child : children) {
+ save(child);
+ }
result = new CheckpointResult(nextToken, changed);
}
runEffects(effects);
@@ -446,17 +456,27 @@ public class DurableExecutionService implements Resettable {
*/
public void recoverAfterRestart() {
int recovered = 0;
+ List<Runnable> lateReports = new ArrayList<>();
for (AccountEntry<DurableExecution> entry : store.scanAllAccountEntries(key -> true)) {
List<Runnable> effects = new ArrayList<>();
synchronized (lockFor(entry.accountId(), entry.key())) {
DurableExecution execution = store.getForAccount(entry.accountId(), entry.key()).orElse(null);
- if (execution == null || execution.isClosed()) {
+ if (execution == null) {
+ continue;
+ }
+ if (execution.isClosed()) {
+ // A child that closed just before the restart may not have reached its parent.
+ reportToParent(execution, lateReports);
+ resumeChainedInvokes(execution, effects);
+ save(execution);
+ runEffects(effects);
continue;
}
execution.setCurrentInvocationId(null);
execution.setReinvokeRequested(false);
+ resumeChainedInvokes(execution, effects);
// A crash retry keeps its backoff, and the sweeper relaunches it when it is due.
- if (execution.getNextInvocationAttemptAt() == null) {
+ if (execution.getCurrentInvocationId() == null && execution.getNextInvocationAttemptAt() == null) {
trigger(execution, effects);
}
save(execution);
@@ -464,6 +484,7 @@ public class DurableExecutionService implements Resettable {
}
runEffects(effects);
}
+ runEffects(lateReports);
if (recovered > 0) {
LOG.infov("Resumed {0} durable execution(s) after restart", recovered);
}
@@ -647,8 +668,12 @@ public class DurableExecutionService implements Resettable {
// ──────────────────────────── state changes ────────────────────────────
private DurableExecution newExecution(StartRequest request, ResolvedDurableTarget target, String name, long now) {
+ return newExecution(request, target, name, UUID.randomUUID().toString(), now);
+ }
+
+ private DurableExecution newExecution(StartRequest request, ResolvedDurableTarget target, String name,
+ String executionId, long now) {
DurableExecution execution = new DurableExecution();
- String executionId = UUID.randomUUID().toString();
execution.setExecutionId(executionId);
execution.setName(name);
execution.setAccountId(request.accountId());
@@ -715,6 +740,230 @@ public class DurableExecutionService implements Resettable {
waiter.complete(execution);
}
});
+ reportToParent(execution, effects);
+ }
+
+ // ──────────────────────────── chained invoke ────────────────────────────
+
+ private record ChainedOutcome(DurableOperationStatus status, String result, DurableErrorObject error) {
+
+ static ChainedOutcome failed(AwsException e) {
+ return new ChainedOutcome(DurableOperationStatus.FAILED, null,
+ DurableErrorObject.of(e.getMessage(), e.getErrorCode()));
+ }
+ }
+
+ /** Call it under the child's lock. The parent is only touched under its own lock, in the effect. */
+ private void reportToParent(DurableExecution child, List<Runnable> effects) {
+ if (child.getParentExecutionArn() == null) {
+ return;
+ }
+ ArnParts parent = parseArn(child.getParentExecutionArn());
+ ChainedOutcome outcome = chainedOutcome(child);
+ String operationId = child.getParentOperationId();
+ String childArn = child.getExecutionArn();
+ effects.add(() -> finishChainedInvoke(parent.accountId(), parent.storeKey(), operationId, childArn, outcome));
+ }
+
+ /**
+ * Resolves the target of a CHAINED_INVOKE this checkpoint started. A target that cannot be
+ * invoked fails the operation at once, and ChainedInvokeStarted then names only the function.
+ */
+ private void startChainedInvoke(DurableExecution execution, DurableOperation operation, long now,
+ List<DurableExecution> children, List<Runnable> effects) {
+ Map<String, Object> details = new LinkedHashMap<>();
+ details.put("FunctionName", operation.getChainedFunctionName());
+ if (operation.getChainedTenantId() != null) {
+ details.put("TenantId", operation.getChainedTenantId());
+ }
+ ResolvedDurableTarget target;
+ try {
+ target = resolveChainedTarget(execution, operation.getChainedFunctionName());
+ } catch (AwsException e) {
+ DurableHistory.operationEvent(execution, operation, "ChainedInvokeStarted", now, details);
+ completeChainedInvoke(execution, operation, ChainedOutcome.failed(e), now, effects);
+ return;
+ }
+ details.put("Input", DurableHistory.payloadWrapper(operation.getInputPayload()));
+ details.put("ExecutedVersion", target.version());
+ if (target.durable()) {
+ DurableExecution child = newChildExecution(execution, operation, target, UUID.randomUUID().toString(),
+ UUID.randomUUID().toString(), now, effects);
+ operation.setChildExecutionArn(child.getExecutionArn());
+ children.add(child);
+ details.put("DurableExecutionArn", child.getExecutionArn());
+ } else {
+ effects.add(plainChainedInvoke(execution, operation, target));
+ }
+ DurableHistory.operationEvent(execution, operation, "ChainedInvokeStarted", now, details);
+ }
+
+ /** The checkpoint already rejected a target in another account or region. */
+ private ResolvedDurableTarget resolveChainedTarget(DurableExecution execution, String functionName) {
+ LambdaArnUtils.ResolvedFunctionRef ref = LambdaArnUtils.resolve(functionName);
+ String accountId = execution.getAccountId();
+ String region = execution.getRegion();
+ ResolvedDurableTarget target;
+ try {
+ target = invoker.resolve(accountId, region, ref.name(), ref.qualifier());
+ } catch (AwsException e) {
+ if (!"ResourceNotFoundException".equals(e.getErrorCode())) {
+ throw e;
+ }
+ String qualifier = ref.qualifier() != null ? ref.qualifier() : "$LATEST";
+ throw new AwsException("ResourceNotFoundException", "Function not found: "
+ + AwsArnUtils.Arn.of("lambda", region, accountId, "function:" + ref.name() + ":" + qualifier), 404);
+ }
+ if (target.durable() && ref.qualifier() == null) {
+ throw new AwsException("InvalidParameterValueException",
+ "You cannot invoke a durable function using an unqualified ARN.", 400);
+ }
+ return target;
+ }
+
+ /** The child gets a random name, as on AWS, and reports its close back to the parent operation. */
+ private DurableExecution newChildExecution(DurableExecution parent, DurableOperation operation,
+ ResolvedDurableTarget target, String name, String executionId, long now,
+ List<Runnable> effects) {
+ StartRequest request = new StartRequest(target.accountId(), target.region(), target.functionName(),
+ target.version(), name, operation.getInputPayload(), false);
+ DurableExecution child = newExecution(request, target, name, executionId, now);
+ child.setParentExecutionArn(parent.getExecutionArn());
+ child.setParentOperationId(operation.getId());
+ trigger(child, effects);
+ return child;
+ }
+
+ private Runnable plainChainedInvoke(DurableExecution parent, DurableOperation operation,
+ ResolvedDurableTarget target) {
+ String accountId = parent.getAccountId();
+ String key = storeKey(parent);
+ String operationId = operation.getId();
+ // A plain target gets {} when the update has no Payload. A durable child starts with no input.
+ byte[] payload = (operation.getInputPayload() != null ? operation.getInputPayload() : "{}")
+ .getBytes(StandardCharsets.UTF_8);
+ long launchGeneration = generation.get();
+ return () -> launchExecutor.execute(() -> {
+ ChainedOutcome outcome = invokePlainFunction(target, payload);
+ if (launchGeneration == generation.get()) {
+ finishChainedInvoke(accountId, key, operationId, null, outcome);
+ }
+ });
+ }
+
+ private ChainedOutcome invokePlainFunction(ResolvedDurableTarget target, byte[] payload) {
+ DurableInvocationResult result;
+ try {
+ result = invoker.invoke(target, payload);
+ } catch (AwsException e) {
+ return ChainedOutcome.failed(e);
+ } catch (RuntimeException e) {
+ LOG.warnv("Chained invoke of {0} failed: {1}", target.functionArn(), e.getMessage());
+ return new ChainedOutcome(DurableOperationStatus.FAILED, null,
+ DurableErrorObject.of(e.getMessage(), "InvocationError"));
+ }
+ if (result.functionError() != null) {
+ return new ChainedOutcome(DurableOperationStatus.FAILED, null,
+ errorFromLambdaPayload(result.payload(), result.functionError()));
+ }
+ if (result.payload() != null && DurableCheckpointApplier.MAX_CHAINED_INVOKE_PAYLOAD_BYTES < result.payload().length) {
+ return new ChainedOutcome(DurableOperationStatus.FAILED, null, DurableErrorObject.of(
+ "CHAINED_INVOKE output payload size must be less than or equal to "
+ + DurableCheckpointApplier.MAX_CHAINED_INVOKE_PAYLOAD_BYTES + " bytes.", null));
+ }
+ String output = result.payload() == null || result.payload().length == 0 ? null
+ : new String(result.payload(), StandardCharsets.UTF_8);
+ return new ChainedOutcome(DurableOperationStatus.SUCCEEDED, output, null);
+ }
+
+ /** A child's close maps onto the parent operation. A timeout reports the child's ExecutionTimeout. */
+ private static ChainedOutcome chainedOutcome(DurableExecution child) {
+ return switch (child.getStatus()) {
+ case SUCCEEDED -> new ChainedOutcome(DurableOperationStatus.SUCCEEDED, child.getResult(), null);
+ case TIMED_OUT -> new ChainedOutcome(DurableOperationStatus.TIMED_OUT, null, DurableErrorObject.of(
+ "CHAINED_INVOKE timed out after " + child.getExecutionTimeoutSeconds() + " seconds.",
+ "ChainedInvoke.Timeout"));
+ case STOPPED -> new ChainedOutcome(DurableOperationStatus.STOPPED, null, child.getError());
+ default -> new ChainedOutcome(DurableOperationStatus.FAILED, null, child.getError());
+ };
+ }
+
+ /** {@code childArn} is the reporting child, which must be the one the operation started. */
+ private void finishChainedInvoke(String accountId, String key, String operationId, String childArn,
+ ChainedOutcome outcome) {
+ List<Runnable> effects = new ArrayList<>();
+ synchronized (lockFor(accountId, key)) {
+ DurableExecution execution = store.getForAccount(accountId, key).orElse(null);
+ DurableOperation operation = execution == null ? null : execution.getOperations().get(operationId);
+ if (operation == null || operation.getStatus() != DurableOperationStatus.STARTED
+ || !Objects.equals(childArn, operation.getChildExecutionArn())) {
+ return;
+ }
+ completeChainedInvoke(execution, operation, outcome, clock.millis(), effects);
+ save(execution);
+ }
+ runEffects(effects);
+ }
+
+ /** A closed execution records the outcome without an event, so a restart does not run the target again. */
+ private void completeChainedInvoke(DurableExecution execution, DurableOperation operation, ChainedOutcome outcome,
+ long now, List<Runnable> effects) {
+ operation.setStatus(outcome.status());
+ operation.setResult(outcome.result());
+ operation.setError(outcome.error());
+ operation.setEndTimestamp(now);
+ if (execution.isClosed()) {
+ return;
+ }
+ operation.setChangeSequence(execution.nextChangeSequence());
+ Map<String, Object> details = new LinkedHashMap<>();
+ if (outcome.status() == DurableOperationStatus.SUCCEEDED) {
+ details.put("Result", DurableHistory.payloadWrapper(outcome.result()));
+ } else {
+ details.put("Error", DurableHistory.errorWrapper(outcome.error()));
+ }
+ String eventType = switch (outcome.status()) {
+ case SUCCEEDED -> "ChainedInvokeSucceeded";
+ case TIMED_OUT -> "ChainedInvokeTimedOut";
+ case STOPPED -> "ChainedInvokeStopped";
+ default -> "ChainedInvokeFailed";
+ };
+ DurableHistory.operationEvent(execution, operation, eventType, now, details);
+ trigger(execution, effects);
+ }
+
+ /**
+ * A plain chained invoke in flight was lost with the process, so it runs again. A durable child
+ * the parent recorded but the store lost is started again under the same ARN.
+ */
+ private void resumeChainedInvokes(DurableExecution execution, List<Runnable> effects) {
+ for (DurableOperation operation : execution.getOperations().values()) {
+ if (operation.getType() != DurableOperationType.CHAINED_INVOKE
+ || operation.getStatus() != DurableOperationStatus.STARTED) {
+ continue;
+ }
+ ArnParts childArn = operation.getChildExecutionArn() == null ? null
+ : parseArn(operation.getChildExecutionArn());
+ if (childArn != null && store.getForAccount(childArn.accountId(), childArn.storeKey()).isPresent()) {
+ continue;
+ }
+ ResolvedDurableTarget target;
+ try {
+ // The version in the saved ARN, since an alias may point elsewhere by now.
+ target = childArn == null ? resolveChainedTarget(execution, operation.getChainedFunctionName())
+ : invoker.resolve(childArn.accountId(), childArn.region(), childArn.functionName(),
+ childArn.version());
+ } catch (AwsException e) {
+ completeChainedInvoke(execution, operation, ChainedOutcome.failed(e), clock.millis(), effects);
+ continue;
+ }
+ if (childArn == null) {
+ effects.add(plainChainedInvoke(execution, operation, target));
+ } else {
+ save(newChildExecution(execution, operation, target, childArn.name(), childArn.executionId(),
+ clock.millis(), effects));
+ }
+ }
}
/** Null unless this is the last checkpoint again, with the same ClientToken, and its token is still current. */
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java
index 20a1196bb0..66722a0b5b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java
@@ -119,6 +119,7 @@ public final class DurableWire {
putTimestamp(details, "ScheduledEndTimestamp", operation.getScheduledEndTimestamp(), millis);
}
}
+ case CHAINED_INVOKE -> putResultAndError(node.putObject("ChainedInvokeDetails"), operation);
case CALLBACK -> {
ObjectNode details = node.putObject("CallbackDetails");
putIfPresent(details, "CallbackId", operation.getCallbackId());
@@ -264,6 +265,7 @@ public final class DurableWire {
Map<?, ?> waitOptions = structure(map, "WaitOptions");
Map<?, ?> contextOptions = structure(map, "ContextOptions");
Map<?, ?> callbackOptions = structure(map, "CallbackOptions");
+ Map<?, ?> chainedInvokeOptions = structure(map, "ChainedInvokeOptions");
return new DurableOperationUpdate(
string(map, "Id"),
string(map, "ParentId"),
@@ -278,7 +280,9 @@ public final class DurableWire {
contextOptions == null ? null : bool(contextOptions, "ReplayChildren"),
integer(callbackOptions, "TimeoutSeconds", path + "callbackOptions", 0, MAX_CALLBACK_TIMEOUT_SECONDS),
integer(callbackOptions, "HeartbeatTimeoutSeconds", path + "callbackOptions", 0,
- MAX_CALLBACK_TIMEOUT_SECONDS));
+ MAX_CALLBACK_TIMEOUT_SECONDS),
+ chainedInvokeOptions == null ? null : string(chainedInvokeOptions, "FunctionName"),
+ chainedInvokeOptions == null ? null : string(chainedInvokeOptions, "TenantId"));
}
private static <E extends Enum<E>> E enumMember(Map<?, ?> map, String member, Class<E> type) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java
index bdaa1fb5a8..9ab5b422f4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java
@@ -52,6 +52,9 @@ public class DurableExecution {
private int consecutiveInvocationFailures;
private Long nextInvocationAttemptAt;
private DurableCheckpointReplay lastCheckpoint;
+ /** Set on an execution a chained invoke started. Its close completes that operation of the parent. */
+ private String parentExecutionArn;
+ private String parentOperationId;
public DurableExecution() {
}
@@ -66,7 +69,7 @@ public class DurableExecution {
return changeSequence > seenSequence;
}
- /** True while a wait, a step retry or a callback is still pending. */
+ /** True while a wait, a step retry, a callback or a chained invoke is still pending. */
@JsonIgnore
public boolean hasPendingOperations() {
for (DurableOperation operation : operations.values()) {
@@ -83,7 +86,7 @@ public class DurableExecution {
return true;
}
}
- case CALLBACK -> {
+ case CALLBACK, CHAINED_INVOKE -> {
if (operationStatus == DurableOperationStatus.STARTED) {
return true;
}
@@ -199,6 +202,12 @@ public class DurableExecution {
this.nextInvocationAttemptAt = nextInvocationAttemptAt;
}
+ public String getParentExecutionArn() { return parentExecutionArn; }
+ public void setParentExecutionArn(String parentExecutionArn) { this.parentExecutionArn = parentExecutionArn; }
+
+ public String getParentOperationId() { return parentOperationId; }
+ public void setParentOperationId(String parentOperationId) { this.parentOperationId = parentOperationId; }
+
public DurableCheckpointReplay getLastCheckpoint() { return lastCheckpoint; }
public void setLastCheckpoint(DurableCheckpointReplay lastCheckpoint) { this.lastCheckpoint = lastCheckpoint; }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java
index 827f9e976e..090480c7b6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java
@@ -34,6 +34,11 @@ public class DurableOperation {
private Long callbackDeadline;
/** Moves forward on every SendDurableExecutionCallbackHeartbeat. */
private Long heartbeatDeadline;
+ /** The FunctionName of a chained invoke, as the function sent it. */
+ private String chainedFunctionName;
+ private String chainedTenantId;
+ /** Set when a chained invoke runs a durable function. Unset, the function runs as a plain invocation. */
+ private String childExecutionArn;
/** Position in the execution's change log. The handler is only shown operations newer than it has seen. */
private long changeSequence;
@@ -61,6 +66,9 @@ public class DurableOperation {
copy.heartbeatTimeoutSeconds = heartbeatTimeoutSeconds;
copy.callbackDeadline = callbackDeadline;
copy.heartbeatDeadline = heartbeatDeadline;
+ copy.chainedFunctionName = chainedFunctionName;
+ copy.chainedTenantId = chainedTenantId;
+ copy.childExecutionArn = childExecutionArn;
copy.changeSequence = changeSequence;
return copy;
}
@@ -124,6 +132,15 @@ public class DurableOperation {
public Long getHeartbeatDeadline() { return heartbeatDeadline; }
public void setHeartbeatDeadline(Long heartbeatDeadline) { this.heartbeatDeadline = heartbeatDeadline; }
+ public String getChainedFunctionName() { return chainedFunctionName; }
+ public void setChainedFunctionName(String chainedFunctionName) { this.chainedFunctionName = chainedFunctionName; }
+
+ public String getChainedTenantId() { return chainedTenantId; }
+ public void setChainedTenantId(String chainedTenantId) { this.chainedTenantId = chainedTenantId; }
+
+ public String getChildExecutionArn() { return childExecutionArn; }
+ public void setChildExecutionArn(String childExecutionArn) { this.childExecutionArn = childExecutionArn; }
+
public long getChangeSequence() { return changeSequence; }
public void setChangeSequence(long changeSequence) { this.changeSequence = changeSequence; }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java
index e43c4f1860..8bcb17b283 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java
@@ -16,5 +16,7 @@ public record DurableOperationUpdate(String id,
Integer waitSeconds,
Boolean replayChildren,
Integer callbackTimeoutSeconds,
- Integer callbackHeartbeatTimeoutSeconds) {
+ Integer callbackHeartbeatTimeoutSeconds,
+ String chainedFunctionName,
+ String chainedTenantId) {
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java
index 400c80d89a..adf25fb43f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java
@@ -33,10 +33,10 @@ class DurableCheckpointApplierTest {
void anExecutionCloseCarriesOnlyItsOwnResultKind() {
DurableErrorObject error = DurableErrorObject.of("m", "T");
assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,
- DurableOperationAction.SUCCEED, "\"p\"", error, null, null, null, null, null)),
+ DurableOperationAction.SUCCEED, "\"p\"", error, null, null, null, null, null, null, null)),
"Cannot provide an Error for SUCCEED action.");
assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,
- DurableOperationAction.FAIL, "\"p\"", error, null, null, null, null, null)),
+ DurableOperationAction.FAIL, "\"p\"", error, null, null, null, null, null, null, null)),
"Cannot provide a Payload for FAIL action.");
}
@@ -44,7 +44,7 @@ class DurableCheckpointApplierTest {
void anExecutionErrorHasTheSameSizeLimitAsAnOperationError() {
DurableErrorObject large = DurableErrorObject.of("x".repeat(DurableCheckpointApplier.MAX_ERROR_BYTES), "T");
assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,
- DurableOperationAction.FAIL, null, large, null, null, null, null, null)),
+ DurableOperationAction.FAIL, null, large, null, null, null, null, null, null, null)),
"Error object size must be less than 32768 bytes.");
}
@@ -77,7 +77,7 @@ class DurableCheckpointApplierTest {
assertEquals("Invalid current STEP state to start.", reject(execution, List.of(stepStart("s1", null))));
assertEquals("Cannot provide an Error for SUCCEED action.", reject(execution, List.of(
new DurableOperationUpdate("s1", null, null, DurableOperationType.STEP, null,
- DurableOperationAction.SUCCEED, null, DurableErrorObject.of("x", "y"), null, null, null, null, null))));
+ DurableOperationAction.SUCCEED, null, DurableErrorObject.of("x", "y"), null, null, null, null, null, null, null))));
assertEquals("Invalid StepOptions for the given action.", reject(execution, List.of(
update("s1", null, DurableOperationType.STEP, DurableOperationAction.RETRY, null))));
DurableCheckpointApplier.apply(execution, List.of(update("s1", null, DurableOperationType.STEP,
@@ -118,9 +118,26 @@ class DurableCheckpointApplierTest {
}
@Test
- void chainedInvokesAreNotSupportedYet() {
+ void aChainedInvokeNeedsItsOptionsAndAnInputOfAtMostOneMegabyte() {
assertRejected(List.of(update("i1", null, DurableOperationType.CHAINED_INVOKE, DurableOperationAction.START,
- null)), "CHAINED_INVOKE operations are not supported yet");
+ null)), "Update for CHAINED_INVOKE operation requires ChainedInvokeOptions.");
+ assertRejected(List.of(chainedStart("i1", "\"" + "x".repeat(1024 * 1024) + "\"")),
+ "CHAINED_INVOKE input payload size must be less than or equal to 1048576 bytes.");
+ assertRejected(List.of(chainedStart("i1", "arn:aws:lambda:us-east-1:111111111111:function:fn", null)),
+ "Cannot start a CHAINED_INVOKE on a function in another account.");
+ assertRejected(List.of(chainedStart("i1", "arn:aws:lambda:eu-west-1:000000000000:function:fn", null)),
+ "Cannot start a CHAINED_INVOKE on a function in another region.");
+
+ DurableExecution execution = execution();
+ DurableCheckpointApplier.Outcome outcome = DurableCheckpointApplier.apply(execution,
+ List.of(chainedStart("i1", "arn:aws:lambda:us-east-1:000000000000:function:target-fn",
+ "\"" + "x".repeat(300 * 1024) + "\"")), NOW);
+ assertEquals(List.of("i1"), outcome.chainedInvokes(), "the service runs what the batch started");
+ DurableOperation invoke = execution.getOperations().get("i1");
+ assertEquals(DurableOperationStatus.STARTED, invoke.getStatus());
+ assertEquals("arn:aws:lambda:us-east-1:000000000000:function:target-fn", invoke.getChainedFunctionName());
+ assertEquals("Invalid action for the given operation type.", reject(execution, List.of(
+ update("i1", null, DurableOperationType.CHAINED_INVOKE, DurableOperationAction.SUCCEED, "1"))));
}
@Test
@@ -190,6 +207,8 @@ class DurableCheckpointApplierTest {
DurableExecution execution = new DurableExecution();
execution.setExecutionId("exec-id");
execution.setExecutionArn(EXECUTION_ARN);
+ execution.setAccountId("000000000000");
+ execution.setRegion("us-east-1");
execution.setName("exec");
execution.setMaxResultBytes(DurableExecutionService.ASYNC_PAYLOAD_LIMIT);
DurableOperation root = new DurableOperation();
@@ -208,12 +227,21 @@ class DurableCheckpointApplierTest {
private static DurableOperationUpdate waitStart(String id) {
return new DurableOperationUpdate(id, null, null, DurableOperationType.WAIT, null, DurableOperationAction.START,
- null, null, null, 5, null, null, null);
+ null, null, null, 5, null, null, null, null, null);
}
private static DurableOperationUpdate callbackStart(String id, int timeoutSeconds, int heartbeatSeconds) {
return new DurableOperationUpdate(id, null, null, DurableOperationType.CALLBACK, "Callback",
- DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, heartbeatSeconds);
+ DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, heartbeatSeconds, null, null);
+ }
+
+ private static DurableOperationUpdate chainedStart(String id, String payload) {
+ return chainedStart(id, "target-fn", payload);
+ }
+
+ private static DurableOperationUpdate chainedStart(String id, String functionName, String payload) {
+ return new DurableOperationUpdate(id, null, null, DurableOperationType.CHAINED_INVOKE, "ChainedInvoke",
+ DurableOperationAction.START, payload, null, null, null, null, null, null, functionName, null);
}
private static DurableOperationUpdate executionSucceed() {
@@ -223,6 +251,6 @@ class DurableCheckpointApplierTest {
private static DurableOperationUpdate update(String id, String parentId, DurableOperationType type,
DurableOperationAction action, String payload) {
return new DurableOperationUpdate(id, parentId, null, type, null, action, payload, null, null, null, null, null,
- null);
+ null, null, null);
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java
index 45642c4f6b..291dded476 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java
@@ -31,6 +31,7 @@ import java.util.Deque;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Executor;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -130,8 +131,182 @@ class DurableExecutionServicePersistenceTest {
assertEquals(3, after.events.size());
}
+ @Test
+ void aPlainChainedInvokeLostWithTheProcessRunsAgainAfterARestart() {
+ ScriptedInvoker before = new ScriptedInvoker();
+ PersistentStorageFactory firstStorage = new PersistentStorageFactory(directory);
+ // Runs the parent's invocation and drops the chained invoke it starts, as a crash would.
+ int[] launched = {0};
+ DurableExecutionService first = newService(firstStorage, before, task -> {
+ if (launched[0]++ == 0) {
+ task.run();
+ }
+ });
+ before.script((service, event) -> {
+ service.checkpoint(arn(event), token(event), null, List.of(new DurableOperationUpdate("i1", null, null,
+ DurableOperationType.CHAINED_INVOKE, "ChainedInvoke", DurableOperationAction.START, "{\"n\":1}",
+ null, null, null, null, null, null, "plain-fn", null)));
+ return "{\"Status\":\"PENDING\"}";
+ });
+ String executionArn = first.start(new StartRequest(ACCOUNT, "us-east-1", "durable-fn", "1", "exec-1", "{}",
+ false)).getExecutionArn();
+ assertEquals("STARTED", first.get(executionArn).getOperations().get("i1").getStatus().name());
+ firstStorage.flushAll();
+
+ ScriptedInvoker after = new ScriptedInvoker();
+ after.plainOutput = "{\"ran\":true}";
+ DurableExecutionService restarted = newService(new PersistentStorageFactory(directory), after);
+ after.script((service, event) -> {
+ assertEquals("{\"ran\":true}", operation(event, "i1").at("/ChainedInvokeDetails/Result").asText());
+ return "{\"Status\":\"SUCCEEDED\",\"Result\":\"\\\"done\\\"\"}";
+ });
+ restarted.recoverAfterRestart();
+
+ assertEquals(DurableExecutionStatus.SUCCEEDED, restarted.get(executionArn).getStatus());
+ assertEquals(List.of("{\"n\":1}"), after.plainPayloads);
+ }
+
+ @Test
+ void aDurableChildThatClosedBeforeItsReportCompletesTheParentAfterARestart() {
+ ScriptedInvoker before = new ScriptedInvoker();
+ PersistentStorageFactory firstStorage = new PersistentStorageFactory(directory);
+ // Runs the parent's invocation and drops the child's, so the child stays RUNNING on disk.
+ int[] launched = {0};
+ DurableExecutionService first = newService(firstStorage, before, task -> {
+ if (launched[0]++ == 0) {
+ task.run();
+ }
+ });
+ before.script((service, event) -> {
+ service.checkpoint(arn(event), token(event), null, List.of(new DurableOperationUpdate("i1", null, null,
+ DurableOperationType.CHAINED_INVOKE, "ChainedInvoke", DurableOperationAction.START, null,
+ null, null, null, null, null, null, "child-fn:1", null)));
+ return "{\"Status\":\"PENDING\"}";
+ });
+ String executionArn = first.start(new StartRequest(ACCOUNT, "us-east-1", "durable-fn", "1", "exec-1", "{}",
+ false)).getExecutionArn();
+ String childArn = first.get(executionArn).getOperations().get("i1").getChildExecutionArn();
+ firstStorage.flushAll();
+
+ // The child closed, and the process stopped before its parent heard of it.
+ PersistentStorageFactory crashedStorage = new PersistentStorageFactory(directory);
+ AccountAwareStorageBackend<DurableExecution> store = crashedStorage.create("lambda",
+ "lambda-durable-executions.json", new TypeReference<Map<String, DurableExecution>>() {});
+ String childKey = DurableExecutionService.parseArn(childArn).storeKey();
+ DurableExecution child = store.getForAccount(ACCOUNT, childKey).orElseThrow();
+ child.setStatus(DurableExecutionStatus.SUCCEEDED);
+ child.setResult("\"child done\"");
+ store.putForAccount(ACCOUNT, childKey, child);
+ crashedStorage.flushAll();
+
+ ScriptedInvoker after = new ScriptedInvoker();
+ DurableExecutionService restarted = newService(new PersistentStorageFactory(directory), after);
+ Script parent = (service, event) -> {
+ JsonNode invoke = operation(event, "i1");
+ if ("STARTED".equals(invoke.get("Status").asText())) {
+ return "{\"Status\":\"PENDING\"}";
+ }
+ assertEquals("\"child done\"", invoke.at("/ChainedInvokeDetails/Result").asText());
+ return "{\"Status\":\"SUCCEEDED\",\"Result\":\"\\\"done\\\"\"}";
+ };
+ after.script(parent);
+ after.script(parent);
+ restarted.recoverAfterRestart();
+
+ assertEquals(DurableExecutionStatus.SUCCEEDED, restarted.get(executionArn).getStatus());
+ }
+
+ @Test
+ void aDurableChildTheStoreLostIsStartedAgainUnderItsArnEvenIfItsAliasMoved() {
+ ScriptedInvoker before = new ScriptedInvoker();
+ PersistentStorageFactory firstStorage = new PersistentStorageFactory(directory);
+ DurableExecutionService first = newService(firstStorage, before, firstTaskOnly());
+ before.script((service, event) -> {
+ service.checkpoint(arn(event), token(event), null, List.of(chainedStart("i1", "child-fn:live", "{\"n\":1}")));
+ return "{\"Status\":\"PENDING\"}";
+ });
+ String parentArn = first.start(new StartRequest(ACCOUNT, "us-east-1", "durable-fn", "1", "exec-1", "{}",
+ false)).getExecutionArn();
+ String childArn = first.get(parentArn).getOperations().get("i1").getChildExecutionArn();
+ executions(firstStorage).deleteForAccount(ACCOUNT,
+ "us-east-1/" + DurableExecutionService.parseArn(childArn).executionId());
+ firstStorage.flushAll();
+
+ ScriptedInvoker after = new ScriptedInvoker();
+ after.liveVersion = "2";
+ DurableExecutionService restarted = newService(new PersistentStorageFactory(directory), after);
+ Script dispatch = (service, event) -> {
+ if (arn(event).equals(childArn)) {
+ assertEquals("{\"n\":1}", event.at("/InitialExecutionState/Operations/0/ExecutionDetails/InputPayload")
+ .asText());
+ return "{\"Status\":\"SUCCEEDED\",\"Result\":\"\\\"child\\\"\"}";
+ }
+ if ("SUCCEEDED".equals(operation(event, "i1").get("Status").asText())) {
+ return "{\"Status\":\"SUCCEEDED\",\"Result\":\"\\\"parent\\\"\"}";
+ }
+ return "{\"Status\":\"PENDING\"}";
+ };
+ after.script(dispatch);
+ after.script(dispatch);
+ restarted.recoverAfterRestart();
+
+ assertEquals(DurableExecutionStatus.SUCCEEDED, restarted.get(childArn).getStatus());
+ assertEquals(DurableExecutionStatus.SUCCEEDED, restarted.get(parentArn).getStatus());
+ assertEquals("\"child\"", restarted.get(parentArn).getOperations().get("i1").getResult());
+ }
+
+ @Test
+ void aTargetStartedByTheClosingCheckpointRunsOnceAfterARestart() {
+ ScriptedInvoker before = new ScriptedInvoker();
+ PersistentStorageFactory firstStorage = new PersistentStorageFactory(directory);
+ DurableExecutionService first = newService(firstStorage, before, firstTaskOnly());
+ before.script((service, event) -> {
+ service.checkpoint(arn(event), token(event), null, List.of(chainedStart("i1", "plain-fn", "{\"n\":1}"),
+ new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,
+ DurableOperationAction.SUCCEED, "\"closed\"", null, null, null, null, null, null, null,
+ null)));
+ return "{\"Status\":\"SUCCEEDED\",\"Result\":\"\"}";
+ });
+ first.start(new StartRequest(ACCOUNT, "us-east-1", "durable-fn", "1", "exec-1", "{}", false));
+ firstStorage.flushAll();
+
+ ScriptedInvoker second = new ScriptedInvoker();
+ PersistentStorageFactory secondStorage = new PersistentStorageFactory(directory);
+ newService(secondStorage, second).recoverAfterRestart();
+ secondStorage.flushAll();
+ ScriptedInvoker third = new ScriptedInvoker();
+ newService(new PersistentStorageFactory(directory), third).recoverAfterRestart();
+
+ assertEquals(List.of("{\"n\":1}"), second.plainPayloads);
+ assertEquals(List.of(), third.plainPayloads, "a completed target does not run again");
+ }
+
+ /** Runs the first launch, the parent's invocation, and drops what it starts, as a crash would. */
+ private static Executor firstTaskOnly() {
+ int[] launched = {0};
+ return task -> {
+ if (launched[0]++ == 0) {
+ task.run();
+ }
+ };
+ }
+
+ private static AccountAwareStorageBackend<DurableExecution> executions(StorageFactory storage) {
+ return storage.create("lambda", "lambda-durable-executions.json",
+ new TypeReference<Map<String, DurableExecution>>() {});
+ }
+
+ private static DurableOperationUpdate chainedStart(String id, String functionName, String payload) {
+ return new DurableOperationUpdate(id, null, null, DurableOperationType.CHAINED_INVOKE, "ChainedInvoke",
+ DurableOperationAction.START, payload, null, null, null, null, null, null, functionName, null);
+ }
+
private DurableExecutionService newService(StorageFactory storage, ScriptedInvoker invoker) {
- DurableExecutionService service = new DurableExecutionService(storage, MAPPER, clock, invoker, Runnable::run,
+ return newService(storage, invoker, Runnable::run);
+ }
+
+ private DurableExecutionService newService(StorageFactory storage, ScriptedInvoker invoker, Executor executor) {
+ DurableExecutionService service = new DurableExecutionService(storage, MAPPER, clock, invoker, executor,
Duration.ZERO);
invoker.service = service;
return service;
@@ -139,12 +314,12 @@ class DurableExecutionServicePersistenceTest {
private static DurableOperationUpdate update(String id, DurableOperationType type, DurableOperationAction action,
String payload, Integer waitSeconds) {
- return new DurableOperationUpdate(id, null, null, type, null, action, payload, null, null, waitSeconds, null, null, null);
+ return new DurableOperationUpdate(id, null, null, type, null, action, payload, null, null, waitSeconds, null, null, null, null, null);
}
private static DurableOperationUpdate callbackStart(String id, int timeoutSeconds) {
return new DurableOperationUpdate(id, null, null, DurableOperationType.CALLBACK, "Callback",
- DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, null);
+ DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, null, null, null);
}
private static String arn(JsonNode event) {
@@ -172,6 +347,10 @@ class DurableExecutionServicePersistenceTest {
final Deque<Script> scripts = new ArrayDeque<>();
final List<JsonNode> events = new ArrayList<>();
+ final List<String> plainPayloads = new ArrayList<>();
+ String plainOutput = "null";
+ /** The version the alias {@code live} points to. */
+ String liveVersion = "1";
DurableExecutionService service;
void script(Script script) {
@@ -180,12 +359,25 @@ class DurableExecutionServicePersistenceTest {
@Override
public ResolvedDurableTarget resolve(String accountId, String region, String functionName, String qualifier) {
+ if ("plain-fn".equals(functionName)) {
+ return new ResolvedDurableTarget(accountId, region, functionName,
+ "arn:aws:lambda:us-east-1:000000000000:function:plain-fn", "$LATEST", false, 0, 0);
+ }
+ String version = "live".equals(qualifier) ? liveVersion : "1";
+ if (qualifier != null && !"live".equals(qualifier)) {
+ version = qualifier;
+ }
return new ResolvedDurableTarget(accountId, region, functionName,
- "arn:aws:lambda:us-east-1:000000000000:function:" + functionName + ":1", "1", true, 3600, 7);
+ "arn:aws:lambda:us-east-1:000000000000:function:" + functionName + ":" + version, version, true,
+ 3600, 7);
}
@Override
public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {
+ if (!target.durable()) {
+ plainPayloads.add(new String(payload, StandardCharsets.UTF_8));
+ return new DurableInvocationResult("req", plainOutput.getBytes(StandardCharsets.UTF_8), null);
+ }
JsonNode event;
try {
event = MAPPER.readTree(payload);
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java
index 6bdd5ced49..db60cd679f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java
@@ -28,7 +28,9 @@ import java.time.Duration;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
@@ -629,8 +631,205 @@ class DurableExecutionServiceTest {
assertCallbackClosed(() -> service.completeCallback(callbackId, ACCOUNT, REGION, true, "\"late\"", null));
}
+ @Test
+ void aChainedInvokeOfAPlainFunctionHandsItsResultToTheNextInvocation() {
+ invoker.plainFunctions.put("plain-fn", event -> handlerResponse("{\"echo\":" + event + "}"));
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(chainedStart("i1", "plain-fn", "{\"a\":1}")));
+ return pending();
+ });
+ invoker.script(event -> {
+ JsonNode invoke = operation(event, "i1");
+ assertEquals("SUCCEEDED", invoke.get("Status").asText());
+ assertEquals("{\"echo\":{\"a\":1}}", invoke.at("/ChainedInvokeDetails/Result").asText());
+ return succeeded("\"done\"");
+ });
+
+ DurableExecution execution = service.get(start("exec-1", "{}", false).getExecutionArn());
+
+ assertEquals(DurableExecutionStatus.SUCCEEDED, execution.getStatus());
+ DurableHistoryEvent started = event(execution, "ChainedInvokeStarted");
+ assertEquals(Map.of("FunctionName", "plain-fn", "Input", Map.of("Payload", "{\"a\":1}", "Truncated", false),
+ "ExecutedVersion", "$LATEST"), started.getDetails().get("ChainedInvokeStartedDetails"));
+ assertTrue(eventTypes(execution).contains("ChainedInvokeSucceeded"));
+ }
+
+ @Test
+ void aChainedInvokeOfAPlainFunctionFailsWithItsErrorOrAnOversizedOutput() {
+ invoker.plainFunctions.put("raising-fn",
+ event -> functionError("{\"errorMessage\":\"child exploded\",\"errorType\":\"ValueError\"}"));
+ invoker.plainFunctions.put("chatty-fn", event -> handlerResponse("\"" + "x".repeat(1024 * 1024) + "\""));
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(chainedStart("raise", "raising-fn", null),
+ chainedStart("chatty", "chatty-fn", null)));
+ return pending();
+ });
+ invoker.script(event -> {
+ assertEquals("ValueError", operation(event, "raise").at("/ChainedInvokeDetails/Error/ErrorType").asText());
+ JsonNode chatty = operation(event, "chatty");
+ assertEquals("FAILED", chatty.get("Status").asText());
+ assertEquals("CHAINED_INVOKE output payload size must be less than or equal to 1048576 bytes.",
+ chatty.at("/ChainedInvokeDetails/Error/ErrorMessage").asText());
+ return succeeded("\"handled\"");
+ });
+
+ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(start("exec-1", "{}", false).getExecutionArn())
+ .getStatus());
+ }
+
+ @Test
+ void aChainedInvokeThatCannotStartFailsAtOnce() {
+ invoker.childScript("child-fn", event -> succeeded("\"never\""));
+ invoker.script(event -> {
+ CheckpointResult result = checkpoint(event, token(event), List.of(chainedStart("missing", "missing-fn", null),
+ chainedStart("unqualified", "child-fn", null)));
+ assertEquals(List.of(DurableOperationStatus.FAILED, DurableOperationStatus.FAILED),
+ result.newExecutionState().stream().map(DurableOperation::getStatus).toList());
+ return pending();
+ });
+ invoker.script(event -> succeeded("\"handled\""));
+
+ DurableExecution execution = service.get(start("exec-1", "{}", false).getExecutionArn());
+
+ DurableOperation missing = execution.getOperations().get("missing");
+ assertEquals("ResourceNotFoundException", missing.getError().getErrorType());
+ assertEquals("Function not found: arn:aws:lambda:us-east-1:000000000000:function:missing-fn:$LATEST",
+ missing.getError().getErrorMessage());
+ assertEquals("You cannot invoke a durable function using an unqualified ARN.",
+ execution.getOperations().get("unqualified").getError().getErrorMessage());
+ assertEquals(Map.of("FunctionName", "missing-fn"),
+ event(execution, "ChainedInvokeStarted").getDetails().get("ChainedInvokeStartedDetails"));
+ assertEquals(DurableExecutionStatus.SUCCEEDED, execution.getStatus());
+ }
+
+ @Test
+ void aDurableChildReportsHowItClosedToTheParentOperation() {
+ invoker.childScript("child-fn", event -> succeeded("\"child result\""));
+ invoker.childScript("child-fn", event -> handlerResponse(
+ "{\"Status\":\"FAILED\",\"Error\":{\"ErrorMessage\":\"child failed\",\"ErrorType\":\"ChildError\"}}"));
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(chainedStart("ok", "child-fn:1", "{\"x\":1}"),
+ chainedStart("bad", "child-fn:1", null)));
+ return pending();
+ });
+ invoker.script(event -> {
+ assertEquals("\"child result\"", operation(event, "ok").at("/ChainedInvokeDetails/Result").asText());
+ assertEquals("ChildError", operation(event, "bad").at("/ChainedInvokeDetails/Error/ErrorType").asText());
+ return succeeded("\"done\"");
+ });
+
+ DurableExecution parent = service.get(start("exec-1", "{}", false).getExecutionArn());
+
+ assertEquals(DurableExecutionStatus.SUCCEEDED, parent.getStatus());
+ String childArn = parent.getOperations().get("ok").getChildExecutionArn();
+ DurableExecution child = service.get(childArn);
+ assertEquals("{\"x\":1}", child.getInputPayload());
+ assertNull(service.get(parent.getOperations().get("bad").getChildExecutionArn()).getInputPayload(),
+ "a missing Payload starts the child with no input");
+ assertEquals(parent.getExecutionArn(), child.getParentExecutionArn());
+ assertEquals(childArn, ((Map<?, ?>) event(parent, "ChainedInvokeStarted").getDetails()
+ .get("ChainedInvokeStartedDetails")).get("DurableExecutionArn"));
+ }
+
+ @Test
+ void aDurableChildThatTimesOutOrIsStoppedClosesTheParentOperationTheSameWay() {
+ invoker.childExecutionTimeoutSeconds = 60;
+ invoker.childScript("child-fn", event -> {
+ checkpoint(event, token(event), List.of(waitStart("w1", 600)));
+ return pending();
+ });
+ invoker.childScript("child-fn", event -> {
+ checkpoint(event, token(event), List.of(waitStart("w1", 600)));
+ return pending();
+ });
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(chainedStart("slow", "child-fn:1", null),
+ chainedStart("stopped", "child-fn:1", null)));
+ return pending();
+ });
+ invoker.script(event -> {
+ assertEquals("STOPPED", operation(event, "stopped").get("Status").asText());
+ assertEquals("stopped by test", operation(event, "stopped").at("/ChainedInvokeDetails/Error/ErrorMessage")
+ .asText());
+ return pending();
+ });
+ invoker.script(event -> {
+ JsonNode slow = operation(event, "slow");
+ assertEquals("TIMED_OUT", slow.get("Status").asText());
+ assertEquals("CHAINED_INVOKE timed out after 60 seconds.",
+ slow.at("/ChainedInvokeDetails/Error/ErrorMessage").asText());
+ assertEquals("ChainedInvoke.Timeout", slow.at("/ChainedInvokeDetails/Error/ErrorType").asText());
+ return succeeded("\"done\"");
+ });
+ String parentArn = start("exec-1", "{}", false).getExecutionArn();
+
+ service.stop(service.get(parentArn).getOperations().get("stopped").getChildExecutionArn(),
+ DurableErrorObject.of("stopped by test", "Test"));
+ assertEquals(2, invoker.events.size(), "the stopped child woke the parent");
+
+ clock.advance(Duration.ofSeconds(59));
+ service.sweep();
+ assertEquals(2, invoker.events.size(), "the child has not timed out yet");
+ clock.advance(Duration.ofSeconds(1));
+ service.sweep();
+
+ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(parentArn).getStatus());
+ assertEquals(3, invoker.events.size());
+ }
+
+ @Test
+ void aChainedInvokeInTheBatchThatClosesTheExecutionStillRuns() {
+ List<String> received = new ArrayList<>();
+ invoker.plainFunctions.put("plain-fn", event -> {
+ received.add(event.toString());
+ return handlerResponse("\"ignored\"");
+ });
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(chainedStart("i1", "plain-fn", "{\"n\":1}"),
+ executionSucceed("\"closed\"")));
+ return succeeded("");
+ });
+
+ DurableExecution execution = service.get(start("exec-1", "{}", false).getExecutionArn());
+
+ assertEquals(List.of("{\"n\":1}"), received);
+ assertEquals("\"closed\"", execution.getResult());
+ assertTrue(eventTypes(execution).contains("ChainedInvokeStarted"));
+ assertFalse(eventTypes(execution).contains("ChainedInvokeSucceeded"));
+ assertEquals(1, invoker.events.size());
+ }
+
+ @Test
+ void stoppingTheParentLeavesADurableChildRunning() {
+ invoker.childScript("child-fn", event -> {
+ checkpoint(event, token(event), List.of(waitStart("w1", 600)));
+ return pending();
+ });
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(chainedStart("i1", "child-fn:1", null)));
+ return pending();
+ });
+ String parentArn = start("exec-1", "{}", false).getExecutionArn();
+ String childArn = service.get(parentArn).getOperations().get("i1").getChildExecutionArn();
+
+ service.stop(parentArn, null);
+
+ assertEquals(DurableExecutionStatus.RUNNING, service.get(childArn).getStatus());
+ assertFalse(eventTypes(service.get(parentArn)).contains("ChainedInvokeStopped"));
+ }
+
// ──────────────────────────── helpers ────────────────────────────
+ private static DurableHistoryEvent event(DurableExecution execution, String eventType) {
+ return execution.getHistory().stream().filter(event -> eventType.equals(event.getEventType())).findFirst()
+ .orElseThrow(() -> new AssertionError("no " + eventType + " in " + eventTypes(execution)));
+ }
+
+ private static DurableOperationUpdate chainedStart(String id, String functionName, String payload) {
+ return new DurableOperationUpdate(id, null, null, DurableOperationType.CHAINED_INVOKE, "ChainedInvoke",
+ DurableOperationAction.START, payload, null, null, null, null, null, null, functionName, null);
+ }
+
private static void assertCallbackClosed(Executable call) {
AwsException rejected = assertThrows(AwsException.class, call);
assertEquals("CallbackTimeoutException", rejected.getErrorCode());
@@ -676,27 +875,27 @@ class DurableExecutionServiceTest {
private static DurableOperationUpdate step(String id, DurableOperationAction action, String payload,
DurableErrorObject error) {
return new DurableOperationUpdate(id, null, null, DurableOperationType.STEP, "Step", action, payload, error,
- null, null, null, null, null);
+ null, null, null, null, null, null, null);
}
private static DurableOperationUpdate stepRetry(String id, int delaySeconds, DurableErrorObject error) {
return new DurableOperationUpdate(id, null, null, DurableOperationType.STEP, "Step",
- DurableOperationAction.RETRY, null, error, delaySeconds, null, null, null, null);
+ DurableOperationAction.RETRY, null, error, delaySeconds, null, null, null, null, null, null);
}
private static DurableOperationUpdate callbackStart(String id, int timeoutSeconds, int heartbeatSeconds) {
return new DurableOperationUpdate(id, null, null, DurableOperationType.CALLBACK, "Callback",
- DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, heartbeatSeconds);
+ DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, heartbeatSeconds, null, null);
}
private static DurableOperationUpdate waitStart(String id, int seconds) {
return new DurableOperationUpdate(id, null, null, DurableOperationType.WAIT, "Wait",
- DurableOperationAction.START, null, null, null, seconds, null, null, null);
+ DurableOperationAction.START, null, null, null, seconds, null, null, null, null, null);
}
private static DurableOperationUpdate executionSucceed(String payload) {
return new DurableOperationUpdate("execution-result", null, null, DurableOperationType.EXECUTION, null,
- DurableOperationAction.SUCCEED, payload, null, null, null, null, null, null);
+ DurableOperationAction.SUCCEED, payload, null, null, null, null, null, null, null, null);
}
private static DurableFunctionInvoker.DurableInvocationResult succeeded(String result) {
@@ -721,7 +920,12 @@ class DurableExecutionServiceTest {
final Deque<Function<JsonNode, DurableInvocationResult>> scripts = new ArrayDeque<>();
final List<JsonNode> events = new ArrayList<>();
+ /** Durable functions other than the one under test, each with its own scripts. */
+ final Map<String, Deque<Function<JsonNode, DurableInvocationResult>>> childScripts = new HashMap<>();
+ /** Plain functions a chained invoke may call, answering the raw payload. */
+ final Map<String, Function<JsonNode, DurableInvocationResult>> plainFunctions = new HashMap<>();
int executionTimeoutSeconds = 3600;
+ int childExecutionTimeoutSeconds = 3600;
String functionName = FUNCTION;
int retentionPeriodInDays = 7;
@@ -729,12 +933,26 @@ class DurableExecutionServiceTest {
scripts.add(script);
}
+ void childScript(String function, Function<JsonNode, DurableInvocationResult> script) {
+ childScripts.computeIfAbsent(function, ignored -> new ArrayDeque<>()).add(script);
+ }
+
@Override
public ResolvedDurableTarget resolve(String accountId, String region, String functionName, String qualifier) {
- String version = qualifier == null ? "1" : qualifier;
- return new ResolvedDurableTarget(accountId, region, this.functionName,
- "arn:aws:lambda:us-east-1:000000000000:function:" + this.functionName + ":" + version, version, true,
- executionTimeoutSeconds, retentionPeriodInDays);
+ String version = qualifier == null ? "$LATEST" : qualifier;
+ if (plainFunctions.containsKey(functionName)) {
+ return new ResolvedDurableTarget(accountId, region, functionName,
+ "arn:aws:lambda:us-east-1:000000000000:function:" + functionName, version, false, 0, 0);
+ }
+ String name = FUNCTION.equals(functionName) ? this.functionName : functionName;
+ if (!name.equals(this.functionName) && !childScripts.containsKey(name)) {
+ throw new AwsException("ResourceNotFoundException", "Function not found: " + name, 404);
+ }
+ String durableVersion = qualifier == null ? "1" : qualifier;
+ return new ResolvedDurableTarget(accountId, region, name,
+ "arn:aws:lambda:us-east-1:000000000000:function:" + name + ":" + durableVersion, durableVersion,
+ true, name.equals(this.functionName) ? executionTimeoutSeconds : childExecutionTimeoutSeconds,
+ retentionPeriodInDays);
}
@Override
@@ -745,6 +963,14 @@ class DurableExecutionServiceTest {
} catch (IOException e) {
throw new UncheckedIOException(e);
}
+ if (!target.durable()) {
+ return plainFunctions.get(target.functionName()).apply(event);
+ }
+ Deque<Function<JsonNode, DurableInvocationResult>> queue = childScripts.get(target.functionName());
+ if (queue != null) {
+ Function<JsonNode, DurableInvocationResult> script = queue.poll();
+ return script != null ? script.apply(event) : pending();
+ }
events.add(event);
Function<JsonNode, DurableInvocationResult> script = scripts.poll();
if (script == null) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java
index cc0a3ab4fa..c026280b73 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java
@@ -441,6 +441,53 @@ class LambdaDurableProtocolIntegrationTest {
.body("Events[3].CallbackSucceededDetails.Result.Payload", equalTo("{\"approved\":true}"));
}
+ @Test
+ void aChainedInvokeOfAnUnqualifiedDurableFunctionFailsInTheCheckpointResponse() throws Exception {
+ Response accepted = given()
+ .header("X-Amz-Invocation-Type", "Event")
+ .header("X-Amz-Durable-Execution-Name", "chained-1")
+ .body("{}")
+ .post(LAMBDA + "/functions/" + FUNCTION + ":$LATEST/invocations");
+ String arn = accepted.getHeader("X-Amz-Durable-Execution-Arn");
+
+ Invocation first = invoker.awaitInvocation(Duration.ofSeconds(30));
+ JsonNode event = MAPPER.readTree(first.payload());
+ given()
+ .urlEncodingEnabled(false)
+ .contentType("application/json")
+ .body("""
+ {
+ "CheckpointToken": "%s",
+ "Updates": [{"Id": "i1", "Name": "greet", "Type": "CHAINED_INVOKE", "SubType": "ChainedInvoke",
+ "Action": "START", "ChainedInvokeOptions": {"FunctionName": "%s"}}]
+ }
+ """.formatted(event.get("CheckpointToken").asText(), FUNCTION))
+ .when()
+ .post(DURABLE + "/durable-executions/" + arn + "/checkpoint")
+ .then()
+ .statusCode(200)
+ .body("NewExecutionState.Operations[0].Type", equalTo("CHAINED_INVOKE"))
+ .body("NewExecutionState.Operations[0].Status", equalTo("FAILED"))
+ .body("NewExecutionState.Operations[0].ChainedInvokeDetails.Error.ErrorType",
+ equalTo("InvalidParameterValueException"))
+ .body("NewExecutionState.Operations[0].ChainedInvokeDetails.Error.ErrorMessage",
+ equalTo("You cannot invoke a durable function using an unqualified ARN."));
+ first.response().complete(handlerResponse("{\"Status\": \"PENDING\"}"));
+
+ Invocation second = invoker.awaitInvocation(Duration.ofSeconds(30));
+ given()
+ .urlEncodingEnabled(false)
+ .when()
+ .get(DURABLE + "/durable-executions/" + arn + "/history")
+ .then()
+ .statusCode(200)
+ .body("Events.EventType", equalTo(List.of("ExecutionStarted", "ChainedInvokeStarted",
+ "ChainedInvokeFailed", "InvocationCompleted")))
+ .body("Events[1].ChainedInvokeStartedDetails.FunctionName", equalTo(FUNCTION))
+ .body("Events[2].ChainedInvokeFailedDetails.Error.Truncated", equalTo(true));
+ second.response().complete(handlerResponse("{\"Status\": \"SUCCEEDED\", \"Result\": \"\\\"ok\\\"\"}"));
+ }
+
private static DurableInvocationResult handlerResponse(String json) {
return new DurableInvocationResult("req-" + System.nanoTime(), json.getBytes(StandardCharsets.UTF_8), null);
}