Light, fluffy, and always free - The AWS Local Emulator alternative
feat(lambda): durable function chained invokes (#5164)
TATSUNO “Taz” Yasuhiro6e6ce61eea6a
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.javaindex 5be56bd05a..20a14d167e 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java@@ -23,6 +23,8 @@ class LambdaDurableFunctionsTest {private static final String ROLE = "arn:aws:iam::000000000000:role/lambda-role";private static final String FN = TestFixtures.uniqueName("fn-durable-exec");private static final String CALLBACK_FN = TestFixtures.uniqueName("fn-durable-callback");+ private static final String CHAIN_FN = TestFixtures.uniqueName("fn-durable-chain");+ private static final String CHAIN_TARGET_FN = TestFixtures.uniqueName("fn-durable-chain-target");private static LambdaClient lambda;@@ -34,7 +36,7 @@ class LambdaDurableFunctionsTest {@AfterAllstatic void cleanup() {if (lambda != null) {- for (String name : new String[] {FN, CALLBACK_FN}) {+ for (String name : new String[] {FN, CALLBACK_FN, CHAIN_FN, CHAIN_TARGET_FN}) {try {lambda.deleteFunction(DeleteFunctionRequest.builder().functionName(name).build());} catch (Exception ignored) {@@ -179,4 +181,45 @@ class LambdaDurableFunctionsTest {SendDurableExecutionCallbackSuccessRequest.builder().callbackId(id).build())).isInstanceOf(CallbackTimeoutException.class);}++ @Test+ @DisplayName("a durable function invokes another function and receives its result")+ void durableChainedInvokeReturnsTheTargetResult() {+ Assumptions.assumeTrue(TestFixtures.isLambdaDispatchAvailable(),+ "skipping: Lambda dispatch (Docker) not available in this environment");++ lambda.createFunction(CreateFunctionRequest.builder()+ .functionName(CHAIN_TARGET_FN)+ .runtime(Runtime.NODEJS20_X)+ .role(ROLE)+ .handler("index.handler")+ .code(FunctionCode.builder().zipFile(SdkBytes.fromByteArray(LambdaUtils.handlerZip())).build())+ .build());+ lambda.createFunction(CreateFunctionRequest.builder()+ .functionName(CHAIN_FN)+ .runtime(Runtime.PYTHON3_14)+ .role(ROLE)+ .handler("lambda_function.handler")+ .timeout(30)+ .durableConfig(DurableConfig.builder().executionTimeout(120).retentionPeriodInDays(1).build())+ .code(FunctionCode.builder().zipFile(SdkBytes.fromByteArray(LambdaUtils.durablePythonZip())).build())+ .build());++ InvokeResponse invoked = lambda.invoke(InvokeRequest.builder()+ .functionName(CHAIN_FN + ":$LATEST")+ .payload(SdkBytes.fromUtf8String("{\"chain\": \"" + CHAIN_TARGET_FN + "\"}"))+ .build());++ assertThat(invoked.functionError()).isNull();+ assertThat(invoked.payload().asUtf8String()).contains("Hello, Durable!");+ GetDurableExecutionHistoryResponse history = lambda.getDurableExecutionHistory(+ GetDurableExecutionHistoryRequest.builder().durableExecutionArn(invoked.durableExecutionArn()).build());+ assertThat(history.events()).extracting(Event::eventType)+ .contains(EventType.CHAINED_INVOKE_STARTED, EventType.CHAINED_INVOKE_SUCCEEDED);+ Event started = history.events().stream()+ .filter(event -> event.eventType() == EventType.CHAINED_INVOKE_STARTED)+ .findFirst().orElseThrow();+ assertThat(started.chainedInvokeStartedDetails().functionName()).isEqualTo(CHAIN_TARGET_FN);+ assertThat(started.chainedInvokeStartedDetails().executedVersion()).isEqualTo("$LATEST");+ }}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.javaindex 67d3545d50..8ec6954f46 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java@@ -284,6 +284,22 @@ public final class LambdaUtils {if request.get("fail"):return {"Status": "FAILED","Error": {"ErrorMessage": "asked to fail", "ErrorType": "TestFailure"}}+ if request.get("chain"):+ if "chain" not in operations:+ client.checkpoint_durable_execution(+ DurableExecutionArn=event["DurableExecutionArn"],+ CheckpointToken=event["CheckpointToken"],+ Updates=[{"Id": "chain", "Name": "greet", "Type": "CHAINED_INVOKE",+ "SubType": "ChainedInvoke", "Action": "START",+ "Payload": json.dumps({"name": "Durable"}),+ "ChainedInvokeOptions": {"FunctionName": request["chain"]}}])+ return {"Status": "PENDING"}+ chain = operations["chain"]+ if chain["Status"] == "STARTED":+ return {"Status": "PENDING"}+ details = chain.get("ChainedInvokeDetails", {})+ return {"Status": "SUCCEEDED",+ "Result": details.get("Result") or json.dumps(details.get("Error"))}if request.get("callback"):if "callback" not in operations:client.checkpoint_durable_execution(diff --git a/docs/services/lambda.md b/docs/services/lambda.mdindex f6a3f1f6ea..6aee0a6c8c 100644--- a/docs/services/lambda.md+++ b/docs/services/lambda.md@@ -168,7 +168,7 @@ background. Invocations through an ARN (EventBridge, Scheduler, function URLs, ddurable executions the same way.The checkpoint protocol under `/2025-12-01` is the real one, so the Durable Execution SDK works-unchanged for steps, waits, step retries, callbacks and replay. Timers are persisted and fired by a+unchanged for steps, waits, step retries, callbacks, chained invokes and replay. Timers are persisted and fired by abackground sweep, so an execution survives a restart of Floci. A function that returns somethingother than the `{Status, Result, Error}` envelope fails its execution with`Invalid Status in invocation output.` A function that throws is re-invoked four more times, after@@ -182,6 +182,13 @@ timeout. A failure sent with an empty request body fails the callback but does nfunction again, as on AWS, so the execution keeps waiting. The SDKs send `{}` when `Error` isomitted, which does invoke it.+A chained invoke (`context.invoke`) runs another function with an input of up to 1 MB. A durable+target needs a version or alias and runs as its own durable execution, and its close completes the+operation as `SUCCEEDED`, `FAILED`, `TIMED_OUT` or `STOPPED`. Any other function is invoked once,+and an output over 1 MB fails the operation. A missing function or an unqualified durable one+fails the operation at once, and a target in another account or Region is rejected at checkpoint.+Stopping the parent leaves the child running, as on AWS.+`GetDurableExecutionHistory` leaves payloads out unless `IncludeExecutionData=true` is sent. TheAPI reference names `true` as the default, but AWS answers this way.@@ -194,7 +201,8 @@ package.Not emulated yet:-- Chained invokes (`context.invoke`). They are rejected at checkpoint time.+- The execution role's `lambda:InvokeFunction` permission before a chained invoke. `TenantId` is+ recorded in the history only.- Durable executions started by Step Functions or by an event source mapping. These invoke thefunction as a plain one.- Payload encryption. `KMSKeyArn` is stored and returned only.@@ -399,7 +407,6 @@ These AWS Lambda operations have no handler in Floci. Calls will return `404` or- Layer permissions (`AddLayerVersionPermission`, `RemoveLayerVersionPermission`, `GetLayerVersionPolicy`)- Provisioned concurrency (`PutProvisionedConcurrencyConfig`, `GetProvisionedConcurrencyConfig`, `ListProvisionedConcurrencyConfigs`, `DeleteProvisionedConcurrencyConfig`)- `InvokeWithResponseStream`-- Durable function chained invokes. See Durable Functions- Code signing enforcement. A configuration is created, read, updated, deleted and listed, andnothing verifies a signature against it, so it never gates a deployment. Attaching one to afunction is not wired either: there is no `PutFunctionCodeSigningConfig`, sodiff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.javaindex 0051ccb852..d49dc086f0 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.lambda.durable;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.lambda.LambdaArnUtils;import io.github.hectorvent.floci.services.lambda.durable.model.DurableErrorObject;import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecution;import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecutionStatus;@@ -30,12 +31,14 @@ final class DurableCheckpointApplier {static final int MAX_OPERATION_PAYLOAD_BYTES = 256 * 1024;static final int MAX_ERROR_BYTES = 32 * 1024;+ static final int MAX_CHAINED_INVOKE_PAYLOAD_BYTES = 1024 * 1024;private static final Pattern OPERATION_ID = Pattern.compile("[a-zA-Z0-9-_]{1,64}");private DurableCheckpointApplier() {}- record Outcome(boolean closed) {+ /** {@code chainedInvokes} names the CHAINED_INVOKE operations this batch started, for the service to run. */+ record Outcome(boolean closed, List<String> chainedInvokes) {}static Outcome apply(DurableExecution execution, List<DurableOperationUpdate> updates, long now) {@@ -45,7 +48,7 @@ final class DurableCheckpointApplier {draft.apply(update);}draft.commit();- return new Outcome(draft.closingStatus != null);+ return new Outcome(draft.closingStatus != null, List.copyOf(draft.chainedInvokes));}/** Completes waits, step retries and callback timeouts whose time has come. The sweeper and every checkpoint call it. */@@ -156,6 +159,7 @@ final class DurableCheckpointApplier {private final long now;private final LinkedHashMap<String, DurableOperation> operations = new LinkedHashMap<>();private final List<Consumer<DurableExecution>> events = new ArrayList<>();+ private final List<String> chainedInvokes = new ArrayList<>();private long changeSequence;private DurableExecutionStatus closingStatus;private String closingResult;@@ -189,7 +193,12 @@ final class DurableCheckpointApplier {}}requireErrorSize(update);- if (utf8Length(update.payload()) > MAX_OPERATION_PAYLOAD_BYTES) {+ if (update.type() == DurableOperationType.CHAINED_INVOKE) {+ if (utf8Length(update.payload()) > MAX_CHAINED_INVOKE_PAYLOAD_BYTES) {+ throw invalid("CHAINED_INVOKE input payload size must be less than or equal to "+ + MAX_CHAINED_INVOKE_PAYLOAD_BYTES + " bytes.");+ }+ } else if (utf8Length(update.payload()) > MAX_OPERATION_PAYLOAD_BYTES) {throw invalid(update.type() + " payload size must be less than or equal to "+ MAX_OPERATION_PAYLOAD_BYTES + " bytes.");}@@ -198,7 +207,7 @@ final class DurableCheckpointApplier {case STEP -> applyStep(update, existing);case WAIT -> applyWait(update, existing);case CALLBACK -> applyCallback(update, existing);- case CHAINED_INVOKE -> throw invalid(update.type() + " operations are not supported yet");+ case CHAINED_INVOKE -> applyChainedInvoke(update, existing);default -> throw invalid("Unknown operation type.");}}@@ -368,6 +377,41 @@ final class DurableCheckpointApplier {}}+ /** The service resolves and runs the target, and records ChainedInvokeStarted with what it found. */+ private void applyChainedInvoke(DurableOperationUpdate update, DurableOperation existing) {+ if (update.action() != DurableOperationAction.START) {+ throw invalid("Invalid action for the given operation type.");+ }+ if (update.chainedFunctionName() == null) {+ throw invalid("Update for CHAINED_INVOKE operation requires ChainedInvokeOptions.");+ }+ if (existing != null) {+ throw invalid("Cannot start a CHAINED_INVOKE that already exist.");+ }+ requireSameAccountAndRegion(update.chainedFunctionName());+ DurableOperation operation = create(update);+ operation.setChainedFunctionName(update.chainedFunctionName());+ operation.setChainedTenantId(update.chainedTenantId());+ operation.setInputPayload(update.payload());+ chainedInvokes.add(update.id());+ }++ /** A name that does not parse is left to the target lookup, which fails the operation. */+ private void requireSameAccountAndRegion(String functionName) {+ LambdaArnUtils.ResolvedFunctionRef ref;+ try {+ ref = LambdaArnUtils.resolve(functionName);+ } catch (AwsException e) {+ return;+ }+ if (ref.account() != null && !ref.account().equals(execution.getAccountId())) {+ throw invalid("Cannot start a CHAINED_INVOKE on a function in another account.");+ }+ if (ref.region() != null && !ref.region().equals(execution.getRegion())) {+ throw invalid("Cannot start a CHAINED_INVOKE on a function in another region.");+ }+ }+/** Only the function starts a callback. SendDurableExecutionCallback* completes it. */private void applyCallback(DurableOperationUpdate update, DurableOperation existing) {if (update.action() != DurableOperationAction.START) {diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.javaindex 9a482866de..b792a9669e 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.lambda.durable;import com.fasterxml.jackson.core.type.TypeReference;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.Pagination;import io.github.hectorvent.floci.core.common.PaginatedResult;@@ -247,6 +248,11 @@ public class DurableExecutionService implements Resettable {DurableExecution execution = loadWithCurrentToken(arn, checkpointToken);long now = clock.millis();DurableCheckpointApplier.Outcome outcome = DurableCheckpointApplier.apply(execution, updates, now);+ // AWS runs a target started in the batch that closes the execution, and ignores its result.+ List<DurableExecution> children = new ArrayList<>();+ for (String operationId : outcome.chainedInvokes()) {+ startChainedInvoke(execution, execution.getOperations().get(operationId), now, children, effects);+ }if (!outcome.closed()) {DurableCheckpointApplier.fireDueTimers(execution, now);}@@ -264,6 +270,10 @@ public class DurableExecutionService implements Resettable {: new DurableCheckpointReplay(checkpointToken, clientToken, nextToken, changed));execution.setSeenSequence(execution.getChangeSequence());save(execution);+ // After the parent, so a crash in between leaves no child the parent does not know.+ for (DurableExecution child : children) {+ save(child);+ }result = new CheckpointResult(nextToken, changed);}runEffects(effects);@@ -446,17 +456,27 @@ public class DurableExecutionService implements Resettable {*/public void recoverAfterRestart() {int recovered = 0;+ List<Runnable> lateReports = new ArrayList<>();for (AccountEntry<DurableExecution> entry : store.scanAllAccountEntries(key -> true)) {List<Runnable> effects = new ArrayList<>();synchronized (lockFor(entry.accountId(), entry.key())) {DurableExecution execution = store.getForAccount(entry.accountId(), entry.key()).orElse(null);- if (execution == null || execution.isClosed()) {+ if (execution == null) {+ continue;+ }+ if (execution.isClosed()) {+ // A child that closed just before the restart may not have reached its parent.+ reportToParent(execution, lateReports);+ resumeChainedInvokes(execution, effects);+ save(execution);+ runEffects(effects);continue;}execution.setCurrentInvocationId(null);execution.setReinvokeRequested(false);+ resumeChainedInvokes(execution, effects);// A crash retry keeps its backoff, and the sweeper relaunches it when it is due.- if (execution.getNextInvocationAttemptAt() == null) {+ if (execution.getCurrentInvocationId() == null && execution.getNextInvocationAttemptAt() == null) {trigger(execution, effects);}save(execution);@@ -464,6 +484,7 @@ public class DurableExecutionService implements Resettable {}runEffects(effects);}+ runEffects(lateReports);if (recovered > 0) {LOG.infov("Resumed {0} durable execution(s) after restart", recovered);}@@ -647,8 +668,12 @@ public class DurableExecutionService implements Resettable {// ──────────────────────────── state changes ────────────────────────────private DurableExecution newExecution(StartRequest request, ResolvedDurableTarget target, String name, long now) {+ return newExecution(request, target, name, UUID.randomUUID().toString(), now);+ }++ private DurableExecution newExecution(StartRequest request, ResolvedDurableTarget target, String name,+ String executionId, long now) {DurableExecution execution = new DurableExecution();- String executionId = UUID.randomUUID().toString();execution.setExecutionId(executionId);execution.setName(name);execution.setAccountId(request.accountId());@@ -715,6 +740,230 @@ public class DurableExecutionService implements Resettable {waiter.complete(execution);}});+ reportToParent(execution, effects);+ }++ // ──────────────────────────── chained invoke ────────────────────────────++ private record ChainedOutcome(DurableOperationStatus status, String result, DurableErrorObject error) {++ static ChainedOutcome failed(AwsException e) {+ return new ChainedOutcome(DurableOperationStatus.FAILED, null,+ DurableErrorObject.of(e.getMessage(), e.getErrorCode()));+ }+ }++ /** Call it under the child's lock. The parent is only touched under its own lock, in the effect. */+ private void reportToParent(DurableExecution child, List<Runnable> effects) {+ if (child.getParentExecutionArn() == null) {+ return;+ }+ ArnParts parent = parseArn(child.getParentExecutionArn());+ ChainedOutcome outcome = chainedOutcome(child);+ String operationId = child.getParentOperationId();+ String childArn = child.getExecutionArn();+ effects.add(() -> finishChainedInvoke(parent.accountId(), parent.storeKey(), operationId, childArn, outcome));+ }++ /**+ * Resolves the target of a CHAINED_INVOKE this checkpoint started. A target that cannot be+ * invoked fails the operation at once, and ChainedInvokeStarted then names only the function.+ */+ private void startChainedInvoke(DurableExecution execution, DurableOperation operation, long now,+ List<DurableExecution> children, List<Runnable> effects) {+ Map<String, Object> details = new LinkedHashMap<>();+ details.put("FunctionName", operation.getChainedFunctionName());+ if (operation.getChainedTenantId() != null) {+ details.put("TenantId", operation.getChainedTenantId());+ }+ ResolvedDurableTarget target;+ try {+ target = resolveChainedTarget(execution, operation.getChainedFunctionName());+ } catch (AwsException e) {+ DurableHistory.operationEvent(execution, operation, "ChainedInvokeStarted", now, details);+ completeChainedInvoke(execution, operation, ChainedOutcome.failed(e), now, effects);+ return;+ }+ details.put("Input", DurableHistory.payloadWrapper(operation.getInputPayload()));+ details.put("ExecutedVersion", target.version());+ if (target.durable()) {+ DurableExecution child = newChildExecution(execution, operation, target, UUID.randomUUID().toString(),+ UUID.randomUUID().toString(), now, effects);+ operation.setChildExecutionArn(child.getExecutionArn());+ children.add(child);+ details.put("DurableExecutionArn", child.getExecutionArn());+ } else {+ effects.add(plainChainedInvoke(execution, operation, target));+ }+ DurableHistory.operationEvent(execution, operation, "ChainedInvokeStarted", now, details);+ }++ /** The checkpoint already rejected a target in another account or region. */+ private ResolvedDurableTarget resolveChainedTarget(DurableExecution execution, String functionName) {+ LambdaArnUtils.ResolvedFunctionRef ref = LambdaArnUtils.resolve(functionName);+ String accountId = execution.getAccountId();+ String region = execution.getRegion();+ ResolvedDurableTarget target;+ try {+ target = invoker.resolve(accountId, region, ref.name(), ref.qualifier());+ } catch (AwsException e) {+ if (!"ResourceNotFoundException".equals(e.getErrorCode())) {+ throw e;+ }+ String qualifier = ref.qualifier() != null ? ref.qualifier() : "$LATEST";+ throw new AwsException("ResourceNotFoundException", "Function not found: "+ + AwsArnUtils.Arn.of("lambda", region, accountId, "function:" + ref.name() + ":" + qualifier), 404);+ }+ if (target.durable() && ref.qualifier() == null) {+ throw new AwsException("InvalidParameterValueException",+ "You cannot invoke a durable function using an unqualified ARN.", 400);+ }+ return target;+ }++ /** The child gets a random name, as on AWS, and reports its close back to the parent operation. */+ private DurableExecution newChildExecution(DurableExecution parent, DurableOperation operation,+ ResolvedDurableTarget target, String name, String executionId, long now,+ List<Runnable> effects) {+ StartRequest request = new StartRequest(target.accountId(), target.region(), target.functionName(),+ target.version(), name, operation.getInputPayload(), false);+ DurableExecution child = newExecution(request, target, name, executionId, now);+ child.setParentExecutionArn(parent.getExecutionArn());+ child.setParentOperationId(operation.getId());+ trigger(child, effects);+ return child;+ }++ private Runnable plainChainedInvoke(DurableExecution parent, DurableOperation operation,+ ResolvedDurableTarget target) {+ String accountId = parent.getAccountId();+ String key = storeKey(parent);+ String operationId = operation.getId();+ // A plain target gets {} when the update has no Payload. A durable child starts with no input.+ byte[] payload = (operation.getInputPayload() != null ? operation.getInputPayload() : "{}")+ .getBytes(StandardCharsets.UTF_8);+ long launchGeneration = generation.get();+ return () -> launchExecutor.execute(() -> {+ ChainedOutcome outcome = invokePlainFunction(target, payload);+ if (launchGeneration == generation.get()) {+ finishChainedInvoke(accountId, key, operationId, null, outcome);+ }+ });+ }++ private ChainedOutcome invokePlainFunction(ResolvedDurableTarget target, byte[] payload) {+ DurableInvocationResult result;+ try {+ result = invoker.invoke(target, payload);+ } catch (AwsException e) {+ return ChainedOutcome.failed(e);+ } catch (RuntimeException e) {+ LOG.warnv("Chained invoke of {0} failed: {1}", target.functionArn(), e.getMessage());+ return new ChainedOutcome(DurableOperationStatus.FAILED, null,+ DurableErrorObject.of(e.getMessage(), "InvocationError"));+ }+ if (result.functionError() != null) {+ return new ChainedOutcome(DurableOperationStatus.FAILED, null,+ errorFromLambdaPayload(result.payload(), result.functionError()));+ }+ if (result.payload() != null && DurableCheckpointApplier.MAX_CHAINED_INVOKE_PAYLOAD_BYTES < result.payload().length) {+ return new ChainedOutcome(DurableOperationStatus.FAILED, null, DurableErrorObject.of(+ "CHAINED_INVOKE output payload size must be less than or equal to "+ + DurableCheckpointApplier.MAX_CHAINED_INVOKE_PAYLOAD_BYTES + " bytes.", null));+ }+ String output = result.payload() == null || result.payload().length == 0 ? null+ : new String(result.payload(), StandardCharsets.UTF_8);+ return new ChainedOutcome(DurableOperationStatus.SUCCEEDED, output, null);+ }++ /** A child's close maps onto the parent operation. A timeout reports the child's ExecutionTimeout. */+ private static ChainedOutcome chainedOutcome(DurableExecution child) {+ return switch (child.getStatus()) {+ case SUCCEEDED -> new ChainedOutcome(DurableOperationStatus.SUCCEEDED, child.getResult(), null);+ case TIMED_OUT -> new ChainedOutcome(DurableOperationStatus.TIMED_OUT, null, DurableErrorObject.of(+ "CHAINED_INVOKE timed out after " + child.getExecutionTimeoutSeconds() + " seconds.",+ "ChainedInvoke.Timeout"));+ case STOPPED -> new ChainedOutcome(DurableOperationStatus.STOPPED, null, child.getError());+ default -> new ChainedOutcome(DurableOperationStatus.FAILED, null, child.getError());+ };+ }++ /** {@code childArn} is the reporting child, which must be the one the operation started. */+ private void finishChainedInvoke(String accountId, String key, String operationId, String childArn,+ ChainedOutcome outcome) {+ List<Runnable> effects = new ArrayList<>();+ synchronized (lockFor(accountId, key)) {+ DurableExecution execution = store.getForAccount(accountId, key).orElse(null);+ DurableOperation operation = execution == null ? null : execution.getOperations().get(operationId);+ if (operation == null || operation.getStatus() != DurableOperationStatus.STARTED+ || !Objects.equals(childArn, operation.getChildExecutionArn())) {+ return;+ }+ completeChainedInvoke(execution, operation, outcome, clock.millis(), effects);+ save(execution);+ }+ runEffects(effects);+ }++ /** A closed execution records the outcome without an event, so a restart does not run the target again. */+ private void completeChainedInvoke(DurableExecution execution, DurableOperation operation, ChainedOutcome outcome,+ long now, List<Runnable> effects) {+ operation.setStatus(outcome.status());+ operation.setResult(outcome.result());+ operation.setError(outcome.error());+ operation.setEndTimestamp(now);+ if (execution.isClosed()) {+ return;+ }+ operation.setChangeSequence(execution.nextChangeSequence());+ Map<String, Object> details = new LinkedHashMap<>();+ if (outcome.status() == DurableOperationStatus.SUCCEEDED) {+ details.put("Result", DurableHistory.payloadWrapper(outcome.result()));+ } else {+ details.put("Error", DurableHistory.errorWrapper(outcome.error()));+ }+ String eventType = switch (outcome.status()) {+ case SUCCEEDED -> "ChainedInvokeSucceeded";+ case TIMED_OUT -> "ChainedInvokeTimedOut";+ case STOPPED -> "ChainedInvokeStopped";+ default -> "ChainedInvokeFailed";+ };+ DurableHistory.operationEvent(execution, operation, eventType, now, details);+ trigger(execution, effects);+ }++ /**+ * A plain chained invoke in flight was lost with the process, so it runs again. A durable child+ * the parent recorded but the store lost is started again under the same ARN.+ */+ private void resumeChainedInvokes(DurableExecution execution, List<Runnable> effects) {+ for (DurableOperation operation : execution.getOperations().values()) {+ if (operation.getType() != DurableOperationType.CHAINED_INVOKE+ || operation.getStatus() != DurableOperationStatus.STARTED) {+ continue;+ }+ ArnParts childArn = operation.getChildExecutionArn() == null ? null+ : parseArn(operation.getChildExecutionArn());+ if (childArn != null && store.getForAccount(childArn.accountId(), childArn.storeKey()).isPresent()) {+ continue;+ }+ ResolvedDurableTarget target;+ try {+ // The version in the saved ARN, since an alias may point elsewhere by now.+ target = childArn == null ? resolveChainedTarget(execution, operation.getChainedFunctionName())+ : invoker.resolve(childArn.accountId(), childArn.region(), childArn.functionName(),+ childArn.version());+ } catch (AwsException e) {+ completeChainedInvoke(execution, operation, ChainedOutcome.failed(e), clock.millis(), effects);+ continue;+ }+ if (childArn == null) {+ effects.add(plainChainedInvoke(execution, operation, target));+ } else {+ save(newChildExecution(execution, operation, target, childArn.name(), childArn.executionId(),+ clock.millis(), effects));+ }+ }}/** Null unless this is the last checkpoint again, with the same ClientToken, and its token is still current. */diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.javaindex 20a1196bb0..66722a0b5b 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java@@ -119,6 +119,7 @@ public final class DurableWire {putTimestamp(details, "ScheduledEndTimestamp", operation.getScheduledEndTimestamp(), millis);}}+ case CHAINED_INVOKE -> putResultAndError(node.putObject("ChainedInvokeDetails"), operation);case CALLBACK -> {ObjectNode details = node.putObject("CallbackDetails");putIfPresent(details, "CallbackId", operation.getCallbackId());@@ -264,6 +265,7 @@ public final class DurableWire {Map<?, ?> waitOptions = structure(map, "WaitOptions");Map<?, ?> contextOptions = structure(map, "ContextOptions");Map<?, ?> callbackOptions = structure(map, "CallbackOptions");+ Map<?, ?> chainedInvokeOptions = structure(map, "ChainedInvokeOptions");return new DurableOperationUpdate(string(map, "Id"),string(map, "ParentId"),@@ -278,7 +280,9 @@ public final class DurableWire {contextOptions == null ? null : bool(contextOptions, "ReplayChildren"),integer(callbackOptions, "TimeoutSeconds", path + "callbackOptions", 0, MAX_CALLBACK_TIMEOUT_SECONDS),integer(callbackOptions, "HeartbeatTimeoutSeconds", path + "callbackOptions", 0,- MAX_CALLBACK_TIMEOUT_SECONDS));+ MAX_CALLBACK_TIMEOUT_SECONDS),+ chainedInvokeOptions == null ? null : string(chainedInvokeOptions, "FunctionName"),+ chainedInvokeOptions == null ? null : string(chainedInvokeOptions, "TenantId"));}private static <E extends Enum<E>> E enumMember(Map<?, ?> map, String member, Class<E> type) {diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.javaindex bdaa1fb5a8..9ab5b422f4 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java@@ -52,6 +52,9 @@ public class DurableExecution {private int consecutiveInvocationFailures;private Long nextInvocationAttemptAt;private DurableCheckpointReplay lastCheckpoint;+ /** Set on an execution a chained invoke started. Its close completes that operation of the parent. */+ private String parentExecutionArn;+ private String parentOperationId;public DurableExecution() {}@@ -66,7 +69,7 @@ public class DurableExecution {return changeSequence > seenSequence;}- /** True while a wait, a step retry or a callback is still pending. */+ /** True while a wait, a step retry, a callback or a chained invoke is still pending. */@JsonIgnorepublic boolean hasPendingOperations() {for (DurableOperation operation : operations.values()) {@@ -83,7 +86,7 @@ public class DurableExecution {return true;}}- case CALLBACK -> {+ case CALLBACK, CHAINED_INVOKE -> {if (operationStatus == DurableOperationStatus.STARTED) {return true;}@@ -199,6 +202,12 @@ public class DurableExecution {this.nextInvocationAttemptAt = nextInvocationAttemptAt;}+ public String getParentExecutionArn() { return parentExecutionArn; }+ public void setParentExecutionArn(String parentExecutionArn) { this.parentExecutionArn = parentExecutionArn; }++ public String getParentOperationId() { return parentOperationId; }+ public void setParentOperationId(String parentOperationId) { this.parentOperationId = parentOperationId; }+public DurableCheckpointReplay getLastCheckpoint() { return lastCheckpoint; }public void setLastCheckpoint(DurableCheckpointReplay lastCheckpoint) { this.lastCheckpoint = lastCheckpoint; }}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.javaindex 827f9e976e..090480c7b6 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java@@ -34,6 +34,11 @@ public class DurableOperation {private Long callbackDeadline;/** Moves forward on every SendDurableExecutionCallbackHeartbeat. */private Long heartbeatDeadline;+ /** The FunctionName of a chained invoke, as the function sent it. */+ private String chainedFunctionName;+ private String chainedTenantId;+ /** Set when a chained invoke runs a durable function. Unset, the function runs as a plain invocation. */+ private String childExecutionArn;/** Position in the execution's change log. The handler is only shown operations newer than it has seen. */private long changeSequence;@@ -61,6 +66,9 @@ public class DurableOperation {copy.heartbeatTimeoutSeconds = heartbeatTimeoutSeconds;copy.callbackDeadline = callbackDeadline;copy.heartbeatDeadline = heartbeatDeadline;+ copy.chainedFunctionName = chainedFunctionName;+ copy.chainedTenantId = chainedTenantId;+ copy.childExecutionArn = childExecutionArn;copy.changeSequence = changeSequence;return copy;}@@ -124,6 +132,15 @@ public class DurableOperation {public Long getHeartbeatDeadline() { return heartbeatDeadline; }public void setHeartbeatDeadline(Long heartbeatDeadline) { this.heartbeatDeadline = heartbeatDeadline; }+ public String getChainedFunctionName() { return chainedFunctionName; }+ public void setChainedFunctionName(String chainedFunctionName) { this.chainedFunctionName = chainedFunctionName; }++ public String getChainedTenantId() { return chainedTenantId; }+ public void setChainedTenantId(String chainedTenantId) { this.chainedTenantId = chainedTenantId; }++ public String getChildExecutionArn() { return childExecutionArn; }+ public void setChildExecutionArn(String childExecutionArn) { this.childExecutionArn = childExecutionArn; }+public long getChangeSequence() { return changeSequence; }public void setChangeSequence(long changeSequence) { this.changeSequence = changeSequence; }}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.javaindex e43c4f1860..8bcb17b283 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java@@ -16,5 +16,7 @@ public record DurableOperationUpdate(String id,Integer waitSeconds,Boolean replayChildren,Integer callbackTimeoutSeconds,- Integer callbackHeartbeatTimeoutSeconds) {+ Integer callbackHeartbeatTimeoutSeconds,+ String chainedFunctionName,+ String chainedTenantId) {}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.javaindex 400c80d89a..adf25fb43f 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java@@ -33,10 +33,10 @@ class DurableCheckpointApplierTest {void anExecutionCloseCarriesOnlyItsOwnResultKind() {DurableErrorObject error = DurableErrorObject.of("m", "T");assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,- DurableOperationAction.SUCCEED, "\"p\"", error, null, null, null, null, null)),+ DurableOperationAction.SUCCEED, "\"p\"", error, null, null, null, null, null, null, null)),"Cannot provide an Error for SUCCEED action.");assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,- DurableOperationAction.FAIL, "\"p\"", error, null, null, null, null, null)),+ DurableOperationAction.FAIL, "\"p\"", error, null, null, null, null, null, null, null)),"Cannot provide a Payload for FAIL action.");}@@ -44,7 +44,7 @@ class DurableCheckpointApplierTest {void anExecutionErrorHasTheSameSizeLimitAsAnOperationError() {DurableErrorObject large = DurableErrorObject.of("x".repeat(DurableCheckpointApplier.MAX_ERROR_BYTES), "T");assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,- DurableOperationAction.FAIL, null, large, null, null, null, null, null)),+ DurableOperationAction.FAIL, null, large, null, null, null, null, null, null, null)),"Error object size must be less than 32768 bytes.");}@@ -77,7 +77,7 @@ class DurableCheckpointApplierTest {assertEquals("Invalid current STEP state to start.", reject(execution, List.of(stepStart("s1", null))));assertEquals("Cannot provide an Error for SUCCEED action.", reject(execution, List.of(new DurableOperationUpdate("s1", null, null, DurableOperationType.STEP, null,- DurableOperationAction.SUCCEED, null, DurableErrorObject.of("x", "y"), null, null, null, null, null))));+ DurableOperationAction.SUCCEED, null, DurableErrorObject.of("x", "y"), null, null, null, null, null, null, null))));assertEquals("Invalid StepOptions for the given action.", reject(execution, List.of(update("s1", null, DurableOperationType.STEP, DurableOperationAction.RETRY, null))));DurableCheckpointApplier.apply(execution, List.of(update("s1", null, DurableOperationType.STEP,@@ -118,9 +118,26 @@ class DurableCheckpointApplierTest {}@Test- void chainedInvokesAreNotSupportedYet() {+ void aChainedInvokeNeedsItsOptionsAndAnInputOfAtMostOneMegabyte() {assertRejected(List.of(update("i1", null, DurableOperationType.CHAINED_INVOKE, DurableOperationAction.START,- null)), "CHAINED_INVOKE operations are not supported yet");+ null)), "Update for CHAINED_INVOKE operation requires ChainedInvokeOptions.");+ assertRejected(List.of(chainedStart("i1", "\"" + "x".repeat(1024 * 1024) + "\"")),+ "CHAINED_INVOKE input payload size must be less than or equal to 1048576 bytes.");+ assertRejected(List.of(chainedStart("i1", "arn:aws:lambda:us-east-1:111111111111:function:fn", null)),+ "Cannot start a CHAINED_INVOKE on a function in another account.");+ assertRejected(List.of(chainedStart("i1", "arn:aws:lambda:eu-west-1:000000000000:function:fn", null)),+ "Cannot start a CHAINED_INVOKE on a function in another region.");++ DurableExecution execution = execution();+ DurableCheckpointApplier.Outcome outcome = DurableCheckpointApplier.apply(execution,+ List.of(chainedStart("i1", "arn:aws:lambda:us-east-1:000000000000:function:target-fn",+ "\"" + "x".repeat(300 * 1024) + "\"")), NOW);+ assertEquals(List.of("i1"), outcome.chainedInvokes(), "the service runs what the batch started");+ DurableOperation invoke = execution.getOperations().get("i1");+ assertEquals(DurableOperationStatus.STARTED, invoke.getStatus());+ assertEquals("arn:aws:lambda:us-east-1:000000000000:function:target-fn", invoke.getChainedFunctionName());+ assertEquals("Invalid action for the given operation type.", reject(execution, List.of(+ update("i1", null, DurableOperationType.CHAINED_INVOKE, DurableOperationAction.SUCCEED, "1"))));}@Test@@ -190,6 +207,8 @@ class DurableCheckpointApplierTest {DurableExecution execution = new DurableExecution();execution.setExecutionId("exec-id");execution.setExecutionArn(EXECUTION_ARN);+ execution.setAccountId("000000000000");+ execution.setRegion("us-east-1");execution.setName("exec");execution.setMaxResultBytes(DurableExecutionService.ASYNC_PAYLOAD_LIMIT);DurableOperation root = new DurableOperation();@@ -208,12 +227,21 @@ class DurableCheckpointApplierTest {private static DurableOperationUpdate waitStart(String id) {return new DurableOperationUpdate(id, null, null, DurableOperationType.WAIT, null, DurableOperationAction.START,- null, null, null, 5, null, null, null);+ null, null, null, 5, null, null, null, null, null);}private static DurableOperationUpdate callbackStart(String id, int timeoutSeconds, int heartbeatSeconds) {return new DurableOperationUpdate(id, null, null, DurableOperationType.CALLBACK, "Callback",- DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, heartbeatSeconds);+ DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, heartbeatSeconds, null, null);+ }++ private static DurableOperationUpdate chainedStart(String id, String payload) {+ return chainedStart(id, "target-fn", payload);+ }++ private static DurableOperationUpdate chainedStart(String id, String functionName, String payload) {+ return new DurableOperationUpdate(id, null, null, DurableOperationType.CHAINED_INVOKE, "ChainedInvoke",+ DurableOperationAction.START, payload, null, null, null, null, null, null, functionName, null);}private static DurableOperationUpdate executionSucceed() {@@ -223,6 +251,6 @@ class DurableCheckpointApplierTest {private static DurableOperationUpdate update(String id, String parentId, DurableOperationType type,DurableOperationAction action, String payload) {return new DurableOperationUpdate(id, parentId, null, type, null, action, payload, null, null, null, null, null,- null);+ null, null, null);}}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.javaindex 45642c4f6b..291dded476 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java@@ -31,6 +31,7 @@ import java.util.Deque;import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.concurrent.Executor;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertThrows;@@ -130,8 +131,182 @@ class DurableExecutionServicePersistenceTest {assertEquals(3, after.events.size());}+ @Test+ void aPlainChainedInvokeLostWithTheProcessRunsAgainAfterARestart() {+ ScriptedInvoker before = new ScriptedInvoker();+ PersistentStorageFactory firstStorage = new PersistentStorageFactory(directory);+ // Runs the parent's invocation and drops the chained invoke it starts, as a crash would.+ int[] launched = {0};+ DurableExecutionService first = newService(firstStorage, before, task -> {+ if (launched[0]++ == 0) {+ task.run();+ }+ });+ before.script((service, event) -> {+ service.checkpoint(arn(event), token(event), null, List.of(new DurableOperationUpdate("i1", null, null,+ DurableOperationType.CHAINED_INVOKE, "ChainedInvoke", DurableOperationAction.START, "{\"n\":1}",+ null, null, null, null, null, null, "plain-fn", null)));+ return "{\"Status\":\"PENDING\"}";+ });+ String executionArn = first.start(new StartRequest(ACCOUNT, "us-east-1", "durable-fn", "1", "exec-1", "{}",+ false)).getExecutionArn();+ assertEquals("STARTED", first.get(executionArn).getOperations().get("i1").getStatus().name());+ firstStorage.flushAll();++ ScriptedInvoker after = new ScriptedInvoker();+ after.plainOutput = "{\"ran\":true}";+ DurableExecutionService restarted = newService(new PersistentStorageFactory(directory), after);+ after.script((service, event) -> {+ assertEquals("{\"ran\":true}", operation(event, "i1").at("/ChainedInvokeDetails/Result").asText());+ return "{\"Status\":\"SUCCEEDED\",\"Result\":\"\\\"done\\\"\"}";+ });+ restarted.recoverAfterRestart();++ assertEquals(DurableExecutionStatus.SUCCEEDED, restarted.get(executionArn).getStatus());+ assertEquals(List.of("{\"n\":1}"), after.plainPayloads);+ }++ @Test+ void aDurableChildThatClosedBeforeItsReportCompletesTheParentAfterARestart() {+ ScriptedInvoker before = new ScriptedInvoker();+ PersistentStorageFactory firstStorage = new PersistentStorageFactory(directory);+ // Runs the parent's invocation and drops the child's, so the child stays RUNNING on disk.+ int[] launched = {0};+ DurableExecutionService first = newService(firstStorage, before, task -> {+ if (launched[0]++ == 0) {+ task.run();+ }+ });+ before.script((service, event) -> {+ service.checkpoint(arn(event), token(event), null, List.of(new DurableOperationUpdate("i1", null, null,+ DurableOperationType.CHAINED_INVOKE, "ChainedInvoke", DurableOperationAction.START, null,+ null, null, null, null, null, null, "child-fn:1", null)));+ return "{\"Status\":\"PENDING\"}";+ });+ String executionArn = first.start(new StartRequest(ACCOUNT, "us-east-1", "durable-fn", "1", "exec-1", "{}",+ false)).getExecutionArn();+ String childArn = first.get(executionArn).getOperations().get("i1").getChildExecutionArn();+ firstStorage.flushAll();++ // The child closed, and the process stopped before its parent heard of it.+ PersistentStorageFactory crashedStorage = new PersistentStorageFactory(directory);+ AccountAwareStorageBackend<DurableExecution> store = crashedStorage.create("lambda",+ "lambda-durable-executions.json", new TypeReference<Map<String, DurableExecution>>() {});+ String childKey = DurableExecutionService.parseArn(childArn).storeKey();+ DurableExecution child = store.getForAccount(ACCOUNT, childKey).orElseThrow();+ child.setStatus(DurableExecutionStatus.SUCCEEDED);+ child.setResult("\"child done\"");+ store.putForAccount(ACCOUNT, childKey, child);+ crashedStorage.flushAll();++ ScriptedInvoker after = new ScriptedInvoker();+ DurableExecutionService restarted = newService(new PersistentStorageFactory(directory), after);+ Script parent = (service, event) -> {+ JsonNode invoke = operation(event, "i1");+ if ("STARTED".equals(invoke.get("Status").asText())) {+ return "{\"Status\":\"PENDING\"}";+ }+ assertEquals("\"child done\"", invoke.at("/ChainedInvokeDetails/Result").asText());+ return "{\"Status\":\"SUCCEEDED\",\"Result\":\"\\\"done\\\"\"}";+ };+ after.script(parent);+ after.script(parent);+ restarted.recoverAfterRestart();++ assertEquals(DurableExecutionStatus.SUCCEEDED, restarted.get(executionArn).getStatus());+ }++ @Test+ void aDurableChildTheStoreLostIsStartedAgainUnderItsArnEvenIfItsAliasMoved() {+ ScriptedInvoker before = new ScriptedInvoker();+ PersistentStorageFactory firstStorage = new PersistentStorageFactory(directory);+ DurableExecutionService first = newService(firstStorage, before, firstTaskOnly());+ before.script((service, event) -> {+ service.checkpoint(arn(event), token(event), null, List.of(chainedStart("i1", "child-fn:live", "{\"n\":1}")));+ return "{\"Status\":\"PENDING\"}";+ });+ String parentArn = first.start(new StartRequest(ACCOUNT, "us-east-1", "durable-fn", "1", "exec-1", "{}",+ false)).getExecutionArn();+ String childArn = first.get(parentArn).getOperations().get("i1").getChildExecutionArn();+ executions(firstStorage).deleteForAccount(ACCOUNT,+ "us-east-1/" + DurableExecutionService.parseArn(childArn).executionId());+ firstStorage.flushAll();++ ScriptedInvoker after = new ScriptedInvoker();+ after.liveVersion = "2";+ DurableExecutionService restarted = newService(new PersistentStorageFactory(directory), after);+ Script dispatch = (service, event) -> {+ if (arn(event).equals(childArn)) {+ assertEquals("{\"n\":1}", event.at("/InitialExecutionState/Operations/0/ExecutionDetails/InputPayload")+ .asText());+ return "{\"Status\":\"SUCCEEDED\",\"Result\":\"\\\"child\\\"\"}";+ }+ if ("SUCCEEDED".equals(operation(event, "i1").get("Status").asText())) {+ return "{\"Status\":\"SUCCEEDED\",\"Result\":\"\\\"parent\\\"\"}";+ }+ return "{\"Status\":\"PENDING\"}";+ };+ after.script(dispatch);+ after.script(dispatch);+ restarted.recoverAfterRestart();++ assertEquals(DurableExecutionStatus.SUCCEEDED, restarted.get(childArn).getStatus());+ assertEquals(DurableExecutionStatus.SUCCEEDED, restarted.get(parentArn).getStatus());+ assertEquals("\"child\"", restarted.get(parentArn).getOperations().get("i1").getResult());+ }++ @Test+ void aTargetStartedByTheClosingCheckpointRunsOnceAfterARestart() {+ ScriptedInvoker before = new ScriptedInvoker();+ PersistentStorageFactory firstStorage = new PersistentStorageFactory(directory);+ DurableExecutionService first = newService(firstStorage, before, firstTaskOnly());+ before.script((service, event) -> {+ service.checkpoint(arn(event), token(event), null, List.of(chainedStart("i1", "plain-fn", "{\"n\":1}"),+ new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,+ DurableOperationAction.SUCCEED, "\"closed\"", null, null, null, null, null, null, null,+ null)));+ return "{\"Status\":\"SUCCEEDED\",\"Result\":\"\"}";+ });+ first.start(new StartRequest(ACCOUNT, "us-east-1", "durable-fn", "1", "exec-1", "{}", false));+ firstStorage.flushAll();++ ScriptedInvoker second = new ScriptedInvoker();+ PersistentStorageFactory secondStorage = new PersistentStorageFactory(directory);+ newService(secondStorage, second).recoverAfterRestart();+ secondStorage.flushAll();+ ScriptedInvoker third = new ScriptedInvoker();+ newService(new PersistentStorageFactory(directory), third).recoverAfterRestart();++ assertEquals(List.of("{\"n\":1}"), second.plainPayloads);+ assertEquals(List.of(), third.plainPayloads, "a completed target does not run again");+ }++ /** Runs the first launch, the parent's invocation, and drops what it starts, as a crash would. */+ private static Executor firstTaskOnly() {+ int[] launched = {0};+ return task -> {+ if (launched[0]++ == 0) {+ task.run();+ }+ };+ }++ private static AccountAwareStorageBackend<DurableExecution> executions(StorageFactory storage) {+ return storage.create("lambda", "lambda-durable-executions.json",+ new TypeReference<Map<String, DurableExecution>>() {});+ }++ private static DurableOperationUpdate chainedStart(String id, String functionName, String payload) {+ return new DurableOperationUpdate(id, null, null, DurableOperationType.CHAINED_INVOKE, "ChainedInvoke",+ DurableOperationAction.START, payload, null, null, null, null, null, null, functionName, null);+ }+private DurableExecutionService newService(StorageFactory storage, ScriptedInvoker invoker) {- DurableExecutionService service = new DurableExecutionService(storage, MAPPER, clock, invoker, Runnable::run,+ return newService(storage, invoker, Runnable::run);+ }++ private DurableExecutionService newService(StorageFactory storage, ScriptedInvoker invoker, Executor executor) {+ DurableExecutionService service = new DurableExecutionService(storage, MAPPER, clock, invoker, executor,Duration.ZERO);invoker.service = service;return service;@@ -139,12 +314,12 @@ class DurableExecutionServicePersistenceTest {private static DurableOperationUpdate update(String id, DurableOperationType type, DurableOperationAction action,String payload, Integer waitSeconds) {- return new DurableOperationUpdate(id, null, null, type, null, action, payload, null, null, waitSeconds, null, null, null);+ return new DurableOperationUpdate(id, null, null, type, null, action, payload, null, null, waitSeconds, null, null, null, null, null);}private static DurableOperationUpdate callbackStart(String id, int timeoutSeconds) {return new DurableOperationUpdate(id, null, null, DurableOperationType.CALLBACK, "Callback",- DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, null);+ DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, null, null, null);}private static String arn(JsonNode event) {@@ -172,6 +347,10 @@ class DurableExecutionServicePersistenceTest {final Deque<Script> scripts = new ArrayDeque<>();final List<JsonNode> events = new ArrayList<>();+ final List<String> plainPayloads = new ArrayList<>();+ String plainOutput = "null";+ /** The version the alias {@code live} points to. */+ String liveVersion = "1";DurableExecutionService service;void script(Script script) {@@ -180,12 +359,25 @@ class DurableExecutionServicePersistenceTest {@Overridepublic ResolvedDurableTarget resolve(String accountId, String region, String functionName, String qualifier) {+ if ("plain-fn".equals(functionName)) {+ return new ResolvedDurableTarget(accountId, region, functionName,+ "arn:aws:lambda:us-east-1:000000000000:function:plain-fn", "$LATEST", false, 0, 0);+ }+ String version = "live".equals(qualifier) ? liveVersion : "1";+ if (qualifier != null && !"live".equals(qualifier)) {+ version = qualifier;+ }return new ResolvedDurableTarget(accountId, region, functionName,- "arn:aws:lambda:us-east-1:000000000000:function:" + functionName + ":1", "1", true, 3600, 7);+ "arn:aws:lambda:us-east-1:000000000000:function:" + functionName + ":" + version, version, true,+ 3600, 7);}@Overridepublic DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {+ if (!target.durable()) {+ plainPayloads.add(new String(payload, StandardCharsets.UTF_8));+ return new DurableInvocationResult("req", plainOutput.getBytes(StandardCharsets.UTF_8), null);+ }JsonNode event;try {event = MAPPER.readTree(payload);diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.javaindex 6bdd5ced49..db60cd679f 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java@@ -28,7 +28,9 @@ import java.time.Duration;import java.util.ArrayDeque;import java.util.ArrayList;import java.util.Deque;+import java.util.HashMap;import java.util.List;+import java.util.Map;import java.util.Set;import java.util.concurrent.TimeUnit;import java.util.function.Function;@@ -629,8 +631,205 @@ class DurableExecutionServiceTest {assertCallbackClosed(() -> service.completeCallback(callbackId, ACCOUNT, REGION, true, "\"late\"", null));}+ @Test+ void aChainedInvokeOfAPlainFunctionHandsItsResultToTheNextInvocation() {+ invoker.plainFunctions.put("plain-fn", event -> handlerResponse("{\"echo\":" + event + "}"));+ invoker.script(event -> {+ checkpoint(event, token(event), List.of(chainedStart("i1", "plain-fn", "{\"a\":1}")));+ return pending();+ });+ invoker.script(event -> {+ JsonNode invoke = operation(event, "i1");+ assertEquals("SUCCEEDED", invoke.get("Status").asText());+ assertEquals("{\"echo\":{\"a\":1}}", invoke.at("/ChainedInvokeDetails/Result").asText());+ return succeeded("\"done\"");+ });++ DurableExecution execution = service.get(start("exec-1", "{}", false).getExecutionArn());++ assertEquals(DurableExecutionStatus.SUCCEEDED, execution.getStatus());+ DurableHistoryEvent started = event(execution, "ChainedInvokeStarted");+ assertEquals(Map.of("FunctionName", "plain-fn", "Input", Map.of("Payload", "{\"a\":1}", "Truncated", false),+ "ExecutedVersion", "$LATEST"), started.getDetails().get("ChainedInvokeStartedDetails"));+ assertTrue(eventTypes(execution).contains("ChainedInvokeSucceeded"));+ }++ @Test+ void aChainedInvokeOfAPlainFunctionFailsWithItsErrorOrAnOversizedOutput() {+ invoker.plainFunctions.put("raising-fn",+ event -> functionError("{\"errorMessage\":\"child exploded\",\"errorType\":\"ValueError\"}"));+ invoker.plainFunctions.put("chatty-fn", event -> handlerResponse("\"" + "x".repeat(1024 * 1024) + "\""));+ invoker.script(event -> {+ checkpoint(event, token(event), List.of(chainedStart("raise", "raising-fn", null),+ chainedStart("chatty", "chatty-fn", null)));+ return pending();+ });+ invoker.script(event -> {+ assertEquals("ValueError", operation(event, "raise").at("/ChainedInvokeDetails/Error/ErrorType").asText());+ JsonNode chatty = operation(event, "chatty");+ assertEquals("FAILED", chatty.get("Status").asText());+ assertEquals("CHAINED_INVOKE output payload size must be less than or equal to 1048576 bytes.",+ chatty.at("/ChainedInvokeDetails/Error/ErrorMessage").asText());+ return succeeded("\"handled\"");+ });++ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(start("exec-1", "{}", false).getExecutionArn())+ .getStatus());+ }++ @Test+ void aChainedInvokeThatCannotStartFailsAtOnce() {+ invoker.childScript("child-fn", event -> succeeded("\"never\""));+ invoker.script(event -> {+ CheckpointResult result = checkpoint(event, token(event), List.of(chainedStart("missing", "missing-fn", null),+ chainedStart("unqualified", "child-fn", null)));+ assertEquals(List.of(DurableOperationStatus.FAILED, DurableOperationStatus.FAILED),+ result.newExecutionState().stream().map(DurableOperation::getStatus).toList());+ return pending();+ });+ invoker.script(event -> succeeded("\"handled\""));++ DurableExecution execution = service.get(start("exec-1", "{}", false).getExecutionArn());++ DurableOperation missing = execution.getOperations().get("missing");+ assertEquals("ResourceNotFoundException", missing.getError().getErrorType());+ assertEquals("Function not found: arn:aws:lambda:us-east-1:000000000000:function:missing-fn:$LATEST",+ missing.getError().getErrorMessage());+ assertEquals("You cannot invoke a durable function using an unqualified ARN.",+ execution.getOperations().get("unqualified").getError().getErrorMessage());+ assertEquals(Map.of("FunctionName", "missing-fn"),+ event(execution, "ChainedInvokeStarted").getDetails().get("ChainedInvokeStartedDetails"));+ assertEquals(DurableExecutionStatus.SUCCEEDED, execution.getStatus());+ }++ @Test+ void aDurableChildReportsHowItClosedToTheParentOperation() {+ invoker.childScript("child-fn", event -> succeeded("\"child result\""));+ invoker.childScript("child-fn", event -> handlerResponse(+ "{\"Status\":\"FAILED\",\"Error\":{\"ErrorMessage\":\"child failed\",\"ErrorType\":\"ChildError\"}}"));+ invoker.script(event -> {+ checkpoint(event, token(event), List.of(chainedStart("ok", "child-fn:1", "{\"x\":1}"),+ chainedStart("bad", "child-fn:1", null)));+ return pending();+ });+ invoker.script(event -> {+ assertEquals("\"child result\"", operation(event, "ok").at("/ChainedInvokeDetails/Result").asText());+ assertEquals("ChildError", operation(event, "bad").at("/ChainedInvokeDetails/Error/ErrorType").asText());+ return succeeded("\"done\"");+ });++ DurableExecution parent = service.get(start("exec-1", "{}", false).getExecutionArn());++ assertEquals(DurableExecutionStatus.SUCCEEDED, parent.getStatus());+ String childArn = parent.getOperations().get("ok").getChildExecutionArn();+ DurableExecution child = service.get(childArn);+ assertEquals("{\"x\":1}", child.getInputPayload());+ assertNull(service.get(parent.getOperations().get("bad").getChildExecutionArn()).getInputPayload(),+ "a missing Payload starts the child with no input");+ assertEquals(parent.getExecutionArn(), child.getParentExecutionArn());+ assertEquals(childArn, ((Map<?, ?>) event(parent, "ChainedInvokeStarted").getDetails()+ .get("ChainedInvokeStartedDetails")).get("DurableExecutionArn"));+ }++ @Test+ void aDurableChildThatTimesOutOrIsStoppedClosesTheParentOperationTheSameWay() {+ invoker.childExecutionTimeoutSeconds = 60;+ invoker.childScript("child-fn", event -> {+ checkpoint(event, token(event), List.of(waitStart("w1", 600)));+ return pending();+ });+ invoker.childScript("child-fn", event -> {+ checkpoint(event, token(event), List.of(waitStart("w1", 600)));+ return pending();+ });+ invoker.script(event -> {+ checkpoint(event, token(event), List.of(chainedStart("slow", "child-fn:1", null),+ chainedStart("stopped", "child-fn:1", null)));+ return pending();+ });+ invoker.script(event -> {+ assertEquals("STOPPED", operation(event, "stopped").get("Status").asText());+ assertEquals("stopped by test", operation(event, "stopped").at("/ChainedInvokeDetails/Error/ErrorMessage")+ .asText());+ return pending();+ });+ invoker.script(event -> {+ JsonNode slow = operation(event, "slow");+ assertEquals("TIMED_OUT", slow.get("Status").asText());+ assertEquals("CHAINED_INVOKE timed out after 60 seconds.",+ slow.at("/ChainedInvokeDetails/Error/ErrorMessage").asText());+ assertEquals("ChainedInvoke.Timeout", slow.at("/ChainedInvokeDetails/Error/ErrorType").asText());+ return succeeded("\"done\"");+ });+ String parentArn = start("exec-1", "{}", false).getExecutionArn();++ service.stop(service.get(parentArn).getOperations().get("stopped").getChildExecutionArn(),+ DurableErrorObject.of("stopped by test", "Test"));+ assertEquals(2, invoker.events.size(), "the stopped child woke the parent");++ clock.advance(Duration.ofSeconds(59));+ service.sweep();+ assertEquals(2, invoker.events.size(), "the child has not timed out yet");+ clock.advance(Duration.ofSeconds(1));+ service.sweep();++ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(parentArn).getStatus());+ assertEquals(3, invoker.events.size());+ }++ @Test+ void aChainedInvokeInTheBatchThatClosesTheExecutionStillRuns() {+ List<String> received = new ArrayList<>();+ invoker.plainFunctions.put("plain-fn", event -> {+ received.add(event.toString());+ return handlerResponse("\"ignored\"");+ });+ invoker.script(event -> {+ checkpoint(event, token(event), List.of(chainedStart("i1", "plain-fn", "{\"n\":1}"),+ executionSucceed("\"closed\"")));+ return succeeded("");+ });++ DurableExecution execution = service.get(start("exec-1", "{}", false).getExecutionArn());++ assertEquals(List.of("{\"n\":1}"), received);+ assertEquals("\"closed\"", execution.getResult());+ assertTrue(eventTypes(execution).contains("ChainedInvokeStarted"));+ assertFalse(eventTypes(execution).contains("ChainedInvokeSucceeded"));+ assertEquals(1, invoker.events.size());+ }++ @Test+ void stoppingTheParentLeavesADurableChildRunning() {+ invoker.childScript("child-fn", event -> {+ checkpoint(event, token(event), List.of(waitStart("w1", 600)));+ return pending();+ });+ invoker.script(event -> {+ checkpoint(event, token(event), List.of(chainedStart("i1", "child-fn:1", null)));+ return pending();+ });+ String parentArn = start("exec-1", "{}", false).getExecutionArn();+ String childArn = service.get(parentArn).getOperations().get("i1").getChildExecutionArn();++ service.stop(parentArn, null);++ assertEquals(DurableExecutionStatus.RUNNING, service.get(childArn).getStatus());+ assertFalse(eventTypes(service.get(parentArn)).contains("ChainedInvokeStopped"));+ }+// ──────────────────────────── helpers ────────────────────────────+ private static DurableHistoryEvent event(DurableExecution execution, String eventType) {+ return execution.getHistory().stream().filter(event -> eventType.equals(event.getEventType())).findFirst()+ .orElseThrow(() -> new AssertionError("no " + eventType + " in " + eventTypes(execution)));+ }++ private static DurableOperationUpdate chainedStart(String id, String functionName, String payload) {+ return new DurableOperationUpdate(id, null, null, DurableOperationType.CHAINED_INVOKE, "ChainedInvoke",+ DurableOperationAction.START, payload, null, null, null, null, null, null, functionName, null);+ }+private static void assertCallbackClosed(Executable call) {AwsException rejected = assertThrows(AwsException.class, call);assertEquals("CallbackTimeoutException", rejected.getErrorCode());@@ -676,27 +875,27 @@ class DurableExecutionServiceTest {private static DurableOperationUpdate step(String id, DurableOperationAction action, String payload,DurableErrorObject error) {return new DurableOperationUpdate(id, null, null, DurableOperationType.STEP, "Step", action, payload, error,- null, null, null, null, null);+ null, null, null, null, null, null, null);}private static DurableOperationUpdate stepRetry(String id, int delaySeconds, DurableErrorObject error) {return new DurableOperationUpdate(id, null, null, DurableOperationType.STEP, "Step",- DurableOperationAction.RETRY, null, error, delaySeconds, null, null, null, null);+ DurableOperationAction.RETRY, null, error, delaySeconds, null, null, null, null, null, null);}private static DurableOperationUpdate callbackStart(String id, int timeoutSeconds, int heartbeatSeconds) {return new DurableOperationUpdate(id, null, null, DurableOperationType.CALLBACK, "Callback",- DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, heartbeatSeconds);+ DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, heartbeatSeconds, null, null);}private static DurableOperationUpdate waitStart(String id, int seconds) {return new DurableOperationUpdate(id, null, null, DurableOperationType.WAIT, "Wait",- DurableOperationAction.START, null, null, null, seconds, null, null, null);+ DurableOperationAction.START, null, null, null, seconds, null, null, null, null, null);}private static DurableOperationUpdate executionSucceed(String payload) {return new DurableOperationUpdate("execution-result", null, null, DurableOperationType.EXECUTION, null,- DurableOperationAction.SUCCEED, payload, null, null, null, null, null, null);+ DurableOperationAction.SUCCEED, payload, null, null, null, null, null, null, null, null);}private static DurableFunctionInvoker.DurableInvocationResult succeeded(String result) {@@ -721,7 +920,12 @@ class DurableExecutionServiceTest {final Deque<Function<JsonNode, DurableInvocationResult>> scripts = new ArrayDeque<>();final List<JsonNode> events = new ArrayList<>();+ /** Durable functions other than the one under test, each with its own scripts. */+ final Map<String, Deque<Function<JsonNode, DurableInvocationResult>>> childScripts = new HashMap<>();+ /** Plain functions a chained invoke may call, answering the raw payload. */+ final Map<String, Function<JsonNode, DurableInvocationResult>> plainFunctions = new HashMap<>();int executionTimeoutSeconds = 3600;+ int childExecutionTimeoutSeconds = 3600;String functionName = FUNCTION;int retentionPeriodInDays = 7;@@ -729,12 +933,26 @@ class DurableExecutionServiceTest {scripts.add(script);}+ void childScript(String function, Function<JsonNode, DurableInvocationResult> script) {+ childScripts.computeIfAbsent(function, ignored -> new ArrayDeque<>()).add(script);+ }+@Overridepublic ResolvedDurableTarget resolve(String accountId, String region, String functionName, String qualifier) {- String version = qualifier == null ? "1" : qualifier;- return new ResolvedDurableTarget(accountId, region, this.functionName,- "arn:aws:lambda:us-east-1:000000000000:function:" + this.functionName + ":" + version, version, true,- executionTimeoutSeconds, retentionPeriodInDays);+ String version = qualifier == null ? "$LATEST" : qualifier;+ if (plainFunctions.containsKey(functionName)) {+ return new ResolvedDurableTarget(accountId, region, functionName,+ "arn:aws:lambda:us-east-1:000000000000:function:" + functionName, version, false, 0, 0);+ }+ String name = FUNCTION.equals(functionName) ? this.functionName : functionName;+ if (!name.equals(this.functionName) && !childScripts.containsKey(name)) {+ throw new AwsException("ResourceNotFoundException", "Function not found: " + name, 404);+ }+ String durableVersion = qualifier == null ? "1" : qualifier;+ return new ResolvedDurableTarget(accountId, region, name,+ "arn:aws:lambda:us-east-1:000000000000:function:" + name + ":" + durableVersion, durableVersion,+ true, name.equals(this.functionName) ? executionTimeoutSeconds : childExecutionTimeoutSeconds,+ retentionPeriodInDays);}@Override@@ -745,6 +963,14 @@ class DurableExecutionServiceTest {} catch (IOException e) {throw new UncheckedIOException(e);}+ if (!target.durable()) {+ return plainFunctions.get(target.functionName()).apply(event);+ }+ Deque<Function<JsonNode, DurableInvocationResult>> queue = childScripts.get(target.functionName());+ if (queue != null) {+ Function<JsonNode, DurableInvocationResult> script = queue.poll();+ return script != null ? script.apply(event) : pending();+ }events.add(event);Function<JsonNode, DurableInvocationResult> script = scripts.poll();if (script == null) {diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.javaindex cc0a3ab4fa..c026280b73 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java@@ -441,6 +441,53 @@ class LambdaDurableProtocolIntegrationTest {.body("Events[3].CallbackSucceededDetails.Result.Payload", equalTo("{\"approved\":true}"));}+ @Test+ void aChainedInvokeOfAnUnqualifiedDurableFunctionFailsInTheCheckpointResponse() throws Exception {+ Response accepted = given()+ .header("X-Amz-Invocation-Type", "Event")+ .header("X-Amz-Durable-Execution-Name", "chained-1")+ .body("{}")+ .post(LAMBDA + "/functions/" + FUNCTION + ":$LATEST/invocations");+ String arn = accepted.getHeader("X-Amz-Durable-Execution-Arn");++ Invocation first = invoker.awaitInvocation(Duration.ofSeconds(30));+ JsonNode event = MAPPER.readTree(first.payload());+ given()+ .urlEncodingEnabled(false)+ .contentType("application/json")+ .body("""+ {+ "CheckpointToken": "%s",+ "Updates": [{"Id": "i1", "Name": "greet", "Type": "CHAINED_INVOKE", "SubType": "ChainedInvoke",+ "Action": "START", "ChainedInvokeOptions": {"FunctionName": "%s"}}]+ }+ """.formatted(event.get("CheckpointToken").asText(), FUNCTION))+ .when()+ .post(DURABLE + "/durable-executions/" + arn + "/checkpoint")+ .then()+ .statusCode(200)+ .body("NewExecutionState.Operations[0].Type", equalTo("CHAINED_INVOKE"))+ .body("NewExecutionState.Operations[0].Status", equalTo("FAILED"))+ .body("NewExecutionState.Operations[0].ChainedInvokeDetails.Error.ErrorType",+ equalTo("InvalidParameterValueException"))+ .body("NewExecutionState.Operations[0].ChainedInvokeDetails.Error.ErrorMessage",+ equalTo("You cannot invoke a durable function using an unqualified ARN."));+ first.response().complete(handlerResponse("{\"Status\": \"PENDING\"}"));++ Invocation second = invoker.awaitInvocation(Duration.ofSeconds(30));+ given()+ .urlEncodingEnabled(false)+ .when()+ .get(DURABLE + "/durable-executions/" + arn + "/history")+ .then()+ .statusCode(200)+ .body("Events.EventType", equalTo(List.of("ExecutionStarted", "ChainedInvokeStarted",+ "ChainedInvokeFailed", "InvocationCompleted")))+ .body("Events[1].ChainedInvokeStartedDetails.FunctionName", equalTo(FUNCTION))+ .body("Events[2].ChainedInvokeFailedDetails.Error.Truncated", equalTo(true));+ second.response().complete(handlerResponse("{\"Status\": \"SUCCEEDED\", \"Result\": \"\\\"ok\\\"\"}"));+ }+private static DurableInvocationResult handlerResponse(String json) {return new DurableInvocationResult("req-" + System.nanoTime(), json.getBytes(StandardCharsets.UTF_8), null);}