Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(lambda): run durable functions with checkpoints, waits and step retries (#5047)

TATSUNO “Taz” Yasuhiro6a3bc127695d
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java
new file mode 100644
index 0000000000..d43d0bb34f
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java
@@ -0,0 +1,120 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.core.SdkBytes;
+import software.amazon.awssdk.services.lambda.LambdaClient;
+import software.amazon.awssdk.services.lambda.model.*;
+import software.amazon.awssdk.services.lambda.model.Runtime;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * Lambda durable functions through the AWS SDK for Java v2: the Invoke extensions and the
+ * durable execution APIs, whose execution ARN travels percent-encoded in the request path.
+ */
+@DisplayName("Lambda durable functions")
+class LambdaDurableFunctionsTest {
+
+ private static final String ROLE = "arn:aws:iam::000000000000:role/lambda-role";
+ private static final String FN = TestFixtures.uniqueName("fn-durable-exec");
+
+ private static LambdaClient lambda;
+
+ @BeforeAll
+ static void setup() {
+ lambda = TestFixtures.lambdaClient();
+ }
+
+ @AfterAll
+ static void cleanup() {
+ if (lambda != null) {
+ try {
+ lambda.deleteFunction(DeleteFunctionRequest.builder().functionName(FN).build());
+ } catch (Exception ignored) {
+ // Cleanup only. A failed delete must not hide the test result.
+ }
+ lambda.close();
+ }
+ }
+
+ @Test
+ @DisplayName("a durable execution runs to completion and is readable through the execution APIs")
+ void durableExecutionRunsAndIsReadable() {
+ Assumptions.assumeTrue(TestFixtures.isLambdaDispatchAvailable(),
+ "skipping: Lambda dispatch (Docker) not available in this environment");
+
+ CreateFunctionResponse created = lambda.createFunction(CreateFunctionRequest.builder()
+ .functionName(FN)
+ .runtime(Runtime.PYTHON3_14)
+ .role(ROLE)
+ .handler("lambda_function.handler")
+ .timeout(30)
+ .publish(true)
+ .durableConfig(DurableConfig.builder().executionTimeout(120).retentionPeriodInDays(1).build())
+ .code(FunctionCode.builder().zipFile(SdkBytes.fromByteArray(LambdaUtils.durablePythonZip())).build())
+ .build());
+ assertThat(created.version()).isEqualTo("1");
+
+ InvokeResponse invoked = lambda.invoke(InvokeRequest.builder()
+ .functionName(FN + ":1")
+ .durableExecutionName("compat-sync")
+ .payload(SdkBytes.fromUtf8String("{\"wait\": 2}"))
+ .build());
+ assertThat(invoked.statusCode()).isEqualTo(200);
+ assertThat(invoked.functionError()).isNull();
+ assertThat(invoked.payload().asUtf8String()).contains("\"done\": true");
+ String arn = invoked.durableExecutionArn();
+ assertThat(arn).contains(":function:" + FN + ":1/durable-execution/compat-sync/");
+
+ GetDurableExecutionResponse execution = lambda.getDurableExecution(GetDurableExecutionRequest.builder()
+ .durableExecutionArn(arn)
+ .includeExecutionData(true)
+ .build());
+ assertThat(execution.status()).isEqualTo(ExecutionStatus.SUCCEEDED);
+ assertThat(execution.durableExecutionName()).isEqualTo("compat-sync");
+ assertThat(execution.version()).isEqualTo("1");
+ assertThat(execution.inputPayload()).isEqualTo("{\"wait\": 2}");
+ assertThat(execution.result()).contains("\"done\": true");
+ assertThat(execution.durableConfig().executionTimeout()).isEqualTo(120);
+
+ ListDurableExecutionsByFunctionResponse listed = lambda.listDurableExecutionsByFunction(
+ ListDurableExecutionsByFunctionRequest.builder().functionName(FN).qualifier("1").build());
+ assertThat(listed.durableExecutions()).extracting(Execution::durableExecutionArn).containsExactly(arn);
+ assertThat(listed.durableExecutions().get(0).status()).isEqualTo(ExecutionStatus.SUCCEEDED);
+
+ GetDurableExecutionHistoryResponse history = lambda.getDurableExecutionHistory(
+ GetDurableExecutionHistoryRequest.builder().durableExecutionArn(arn).includeExecutionData(true).build());
+ assertThat(history.events()).extracting(Event::eventType)
+ .startsWith(EventType.EXECUTION_STARTED)
+ .contains(EventType.STEP_SUCCEEDED, EventType.WAIT_STARTED, EventType.WAIT_SUCCEEDED,
+ EventType.INVOCATION_COMPLETED)
+ .endsWith(EventType.EXECUTION_SUCCEEDED);
+ assertThat(history.events().get(0).executionStartedDetails().input().payload()).isEqualTo("{\"wait\": 2}");
+
+ StopDurableExecutionResponse stopped = lambda.stopDurableExecution(StopDurableExecutionRequest.builder()
+ .durableExecutionArn(arn)
+ .build());
+ assertThat(stopped.stopTimestamp())
+ .as("stopping a finished execution reports the time it finished")
+ .isEqualTo(execution.endTimestamp());
+
+ assertThatThrownBy(() -> lambda.invoke(InvokeRequest.builder()
+ .functionName(FN + ":1")
+ .durableExecutionName("compat-sync")
+ .payload(SdkBytes.fromUtf8String("{\"wait\": 3}"))
+ .build()))
+ .isInstanceOf(DurableExecutionAlreadyStartedException.class);
+
+ assertThatThrownBy(() -> lambda.invoke(InvokeRequest.builder()
+ .functionName(FN)
+ .payload(SdkBytes.fromUtf8String("{}"))
+ .build()))
+ .isInstanceOf(InvalidParameterValueException.class)
+ .hasMessageContaining("You cannot invoke a durable function using an unqualified ARN.");
+ }
+}
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java
index fe88ca375b..e93f56cc88 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java
@@ -264,6 +264,47 @@ public final class LambdaUtils {
return createZip("index.js", code);
}
+ /**
+ * A Python durable function that speaks the checkpoint protocol through the image's boto3
+ * client: one step, one wait of {@code event.wait} seconds (default 2), then a result. An input
+ * of {@code {"fail": true}} fails the execution instead.
+ */
+ public static byte[] durablePythonZip() {
+ String code = """
+ import json
+ import boto3
+
+ client = boto3.client("lambda")
+
+
+ def handler(event, context):
+ operations = {op["Id"]: op for op in event["InitialExecutionState"]["Operations"]}
+ root = next(op for op in operations.values() if op["Type"] == "EXECUTION")
+ request = json.loads(root["ExecutionDetails"].get("InputPayload") or "{}")
+ if request.get("fail"):
+ return {"Status": "FAILED",
+ "Error": {"ErrorMessage": "asked to fail", "ErrorType": "TestFailure"}}
+ if "step" not in operations:
+ client.checkpoint_durable_execution(
+ DurableExecutionArn=event["DurableExecutionArn"],
+ CheckpointToken=event["CheckpointToken"],
+ Updates=[
+ {"Id": "step", "Name": "validate", "Type": "STEP", "SubType": "Step",
+ "Action": "START"},
+ {"Id": "step", "Name": "validate", "Type": "STEP", "SubType": "Step",
+ "Action": "SUCCEED", "Payload": json.dumps({"validated": True})},
+ {"Id": "wait", "Type": "WAIT", "SubType": "Wait", "Action": "START",
+ "WaitOptions": {"WaitSeconds": request.get("wait", 2)}},
+ ])
+ return {"Status": "PENDING"}
+ if operations["wait"]["Status"] != "SUCCEEDED":
+ return {"Status": "PENDING"}
+ step = json.loads(operations["step"]["StepDetails"]["Result"])
+ return {"Status": "SUCCEEDED", "Result": json.dumps({"done": True, "step": step})}
+ """;
+ return createZip("lambda_function.py", code);
+ }
+
private static byte[] createZip(String filename, String content) {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
diff --git a/docs/services/index.md b/docs/services/index.md
index 30cde70ff3..6aea30cf70 100644
--- a/docs/services/index.md
+++ b/docs/services/index.md
@@ -18,7 +18,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac
| [S3 Tables](s3tables.md) | `/buckets`, `/namespaces/*`, `/tables/*` | REST JSON | 25 |
| [DynamoDB](dynamodb.md) | `POST /` + `X-Amz-Target: DynamoDB_20120810.*` | JSON 1.1 | 28 |
| [DynamoDB Streams](dynamodb.md#streams) | `POST /` + `X-Amz-Target: DynamoDBStreams_20120810.*` | JSON 1.1 | 4 |
-| [Lambda](lambda.md) | `/2015-03-31/functions/...` | REST JSON | 46 |
+| [Lambda](lambda.md) | `/2015-03-31/functions/...` + `/2025-12-01/durable-executions/...` | REST JSON | 52 |
| [Lambda MicroVMs](lambda-microvms.md) | `/2025-09-09/...` + `/2026-04-04/...` | REST JSON | 22 |
| [API Gateway v1](api-gateway.md) | `/restapis/...` | REST JSON | 79 |
| [API Gateway v2](api-gateway.md#v2) | `/v2/apis/...` | REST JSON | 53 + data-plane |
diff --git a/docs/services/lambda.md b/docs/services/lambda.md
index aab408b28f..b0dbcb9c8b 100644
--- a/docs/services/lambda.md
+++ b/docs/services/lambda.md
@@ -16,7 +16,7 @@ Floci Lambda runs your function code locally inside real Docker containers - clo
| `UpdateFunctionCode` | Upload new code |
| `UpdateFunctionConfiguration` | Update runtime, handler, memory, timeout, environment, architectures, tracing, layers, and more |
| `DeleteFunction` | Remove a function |
-| `Invoke` | Invoke a function synchronously or asynchronously |
+| `Invoke` | Invoke a function synchronously or asynchronously. A durable function starts a durable execution |
| `CreateEventSourceMapping` | Connect SQS / Kinesis / DynamoDB Streams to a function |
| `GetEventSourceMapping` | Get event source mapping details |
| `ListEventSourceMappings` | List all event source mappings |
@@ -55,6 +55,12 @@ Floci Lambda runs your function code locally inside real Docker containers - clo
| `GetFunctionEventInvokeConfig` | Read the asynchronous invocation settings |
| `DeleteFunctionEventInvokeConfig` | Remove the asynchronous invocation settings |
| `ListFunctionEventInvokeConfigs` | List the asynchronous invocation settings of every version and alias of a function |
+| `CheckpointDurableExecution` | Record the operation updates of the Durable Execution SDK and return the new state |
+| `GetDurableExecutionState` | Operations of a running durable execution, for replay |
+| `GetDurableExecution` | Status, result or error and the DurableConfig of a durable execution |
+| `GetDurableExecutionHistory` | Event log of a durable execution, oldest or newest first |
+| `ListDurableExecutionsByFunction` | Durable executions of a function or version, filtered by status, name and start time |
+| `StopDurableExecution` | Stop a running durable execution with an error |
`UpdateFunctionConfiguration` validates `MemorySize` and `Timeout` as whole numbers in their
supported ranges before changing any stored settings. A rejected value returns
@@ -147,6 +153,43 @@ Two limits are worth knowing:
that fans back to the function, or a bus rule targeting it, each start a fresh chain at zero.
Avoid configuring one until this is closed.
+### Durable Functions
+
+A function created with `DurableConfig` is a durable function. Invoking it through a version or
+alias (`name:1`, `name:live`, `?Qualifier=1`) starts a durable execution. An unqualified name is
+rejected, as on AWS. `X-Amz-Durable-Execution-Name` names the execution, and Floci generates a
+UUID when it is absent. `X-Amz-Durable-Execution-Arn` returns the execution ARN. A
+`RequestResponse` invoke blocks until the execution ends, waits included, and needs an
+`ExecutionTimeout` of 15 minutes or less. `Event` answers 202 and the execution continues in the
+background. Invocations through an ARN (EventBridge, Scheduler, function URLs, destinations) start
+durable executions the same way.
+
+The checkpoint protocol under `/2025-12-01` is the real one, so the Durable Execution SDK works
+unchanged for steps, waits, step retries and replay. Timers are persisted and fired by a
+background sweep, so an execution survives a restart of Floci. A function that returns something
+other than the `{Status, Result, Error}` envelope fails its execution with
+`Invalid Status in invocation output.` A function that throws is re-invoked four more times, after
+1, 2, 4 and 8 seconds, and then fails the execution with its error. AWS does the same over a few
+minutes.
+
+`GetDurableExecutionHistory` leaves payloads out unless `IncludeExecutionData=true` is sent. The
+API reference names `true` as the default, but AWS answers this way.
+
+The public `public.ecr.aws/lambda` images do not bundle the SDK that the managed runtimes carry.
+Add `aws-durable-execution-sdk-python` or `@aws/durable-execution-sdk-js` to your deployment
+package.
+
+Not emulated yet:
+
+- Callbacks (`WaitForCallback` and the `SendDurableExecutionCallback*` APIs) and chained invokes
+ (`context.invoke`). Both are rejected at checkpoint time.
+- Durable executions started by Step Functions or by an event source mapping. These invoke the
+ function as a plain one.
+- Payload encryption. `KMSKeyArn` is stored and returned only.
+- EventBridge status-change events, the X-Ray `TraceHeader`, and a dead-letter queue or
+ destinations for durable executions.
+- The per-execution limits of 3,000 operations and 100 MB of persisted payload.
+
## Hot-Reloading via Reactive S3 Sync
Floci supports an automatic hot-reloading mechanism when functions are deployed via S3. This follows the standard AWS behavior where S3 and Lambda interact, but is optimized for a seamless local development experience.
@@ -344,6 +387,8 @@ These AWS Lambda operations have no handler in Floci. Calls will return `404` or
- Layer permissions (`AddLayerVersionPermission`, `RemoveLayerVersionPermission`, `GetLayerVersionPolicy`)
- Provisioned concurrency (`PutProvisionedConcurrencyConfig`, `GetProvisionedConcurrencyConfig`, `ListProvisionedConcurrencyConfigs`, `DeleteProvisionedConcurrencyConfig`)
- `InvokeWithResponseStream`
+- Durable function callbacks and chained invokes (`SendDurableExecutionCallbackSuccess`,
+ `SendDurableExecutionCallbackFailure`, `SendDurableExecutionCallbackHeartbeat`). See Durable Functions
- Code signing enforcement. A configuration is created, read, updated, deleted and listed, and
nothing verifies a signature against it, so it never gates a deployment. Attaching one to a
function is not wired either: there is no `PutFunctionCodeSigningConfig`, so
@@ -369,6 +414,8 @@ These AWS Lambda operations have no handler in Floci. Calls will return `404` or
| `FLOCI_SERVICES_LAMBDA_HONOUR_ARCHITECTURES` | `false` | Select each function's declared architecture for Docker image pulls and containers |
| `FLOCI_SERVICES_LAMBDA_DEFAULT_MEMORY_MB` | `128` | Default function memory (MB) |
| `FLOCI_SERVICES_LAMBDA_DEFAULT_TIMEOUT_SECONDS` | `3` | Default function timeout (seconds) |
+| `FLOCI_SERVICES_LAMBDA_DURABLE_SWEEP_ENABLED` | `true` | Fire durable execution timers (waits, step retries, execution timeouts, retention expiry) |
+| `FLOCI_SERVICES_LAMBDA_DURABLE_SWEEP_INTERVAL_SECONDS` | `1` | How often the durable sweep runs |
| `FLOCI_SERVICES_LAMBDA_RUNTIME_API_BASE_PORT` | `12000` | First port in the Lambda Runtime API range |
| `FLOCI_SERVICES_LAMBDA_RUNTIME_API_MAX_PORT` | `12499` | Last port in the Lambda Runtime API range. One port is held per running container, so the range width caps concurrent executions. Each running container also holds two Docker connections, so a wider range needs a matching `FLOCI_DOCKER_MAX_CONNECTIONS` |
| `FLOCI_SERVICES_LAMBDA_CODE_PATH` | `./data/lambda-code` | Directory where Lambda ZIP files are stored |
@@ -703,8 +750,7 @@ is a permanent diff rather than a cosmetic omission.
`DurableConfig` requires `ExecutionTimeout` on create. A durable function created without
a `Timeout` gets `min(ExecutionTimeout, 900)`. It always logs in JSON format, and
`LogFormat: Text` is rejected. On update the members are merged, and a function created
-without `DurableConfig` cannot gain one. The durable execution itself (checkpoints, waits,
-callbacks) is not emulated yet; the configuration only round-trips.
+without `DurableConfig` cannot gain one. See Durable Functions for the execution itself.
`LogGroup` is validated against AWS's documented constraint: 1-512 characters matching
`[.\-_/#A-Za-z0-9]+`. `ApplicationLogLevel` and `SystemLogLevel` are accepted with any
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 45b1965285..1f5245f6d0 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -2712,6 +2712,17 @@ public interface EmulatorConfig {
@WithDefault("3")
int defaultTimeoutSeconds();
+ /**
+ * Run the background sweep that fires durable execution timers. These are waits, step
+ * retries, execution timeouts and retention expiry. When off, timers are recorded but never fire.
+ */
+ @WithDefault("true")
+ boolean durableSweepEnabled();
+
+ /** How often the durable sweep runs. Waits are whole seconds, so 1s bounds their lateness. */
+ @WithDefault("1")
+ long durableSweepIntervalSeconds();
+
Optional<String> dockerHostOverride();
/**
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
index 6e748bb50d..a1932c75bf 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
@@ -30,6 +30,7 @@ import io.github.hectorvent.floci.services.bedrockagentcorecontrol.BedrockAgentC
import io.github.hectorvent.floci.services.bedrockagentcorecontrol.BedrockAgentCoreToolsController;
import io.github.hectorvent.floci.services.pipes.PipesController;
import io.github.hectorvent.floci.services.lambda.LambdaController;
+import io.github.hectorvent.floci.services.lambda.durable.DurableExecutionController;
import io.github.hectorvent.floci.services.lambdamicrovms.LambdaMicrovmsController;
import io.github.hectorvent.floci.services.lambdamicrovms.LambdaNetworkConnectorsController;
import io.github.hectorvent.floci.services.opensearch.OpenSearchController;
@@ -141,6 +142,7 @@ public class ResolvedServiceCatalog {
protocols(ServiceProtocol.REST_JSON),
Set.of(), Set.of("lambda"), Set.of(),
Set.of(LambdaController.class,
+ DurableExecutionController.class,
LambdaMicrovmsController.class,
LambdaNetworkConnectorsController.class)),
descriptor("apigateway", "apigateway", config.services().apigateway().enabled(), true,
diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
index 7d389b1c9f..8a3f01e277 100644
--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
@@ -26,6 +26,7 @@ import io.github.hectorvent.floci.services.dynamodb.DynamoDbRuntime;
import io.github.hectorvent.floci.services.lambda.DynamoDbStreamsEventSourcePoller;
import io.github.hectorvent.floci.services.lambda.KinesisEventSourcePoller;
import io.github.hectorvent.floci.services.lambda.SqsEventSourcePoller;
+import io.github.hectorvent.floci.services.lambda.durable.DurableExecutionService;
import io.github.hectorvent.floci.services.neptune.container.NeptuneContainerManager;
import io.github.hectorvent.floci.services.neptune.proxy.NeptuneProxyManager;
import io.github.hectorvent.floci.services.pipes.PipesService;
@@ -108,6 +109,7 @@ public class EmulatorLifecycle {
private final InitLifecycleState initLifecycleState;
private final SchemaCreationWorker schemaCreationWorker;
private final StepFunctionsService stepFunctionsService;
+ private final DurableExecutionService durableExecutionService;
private final Instance<ContainerTeardown> containerTeardowns;
private final PersistentPathValidator persistentPathValidator;
private final DynamoDbRuntime dynamoDbRuntime;
@@ -148,6 +150,7 @@ public class EmulatorLifecycle {
InitLifecycleState initLifecycleState,
SchemaCreationWorker schemaCreationWorker,
StepFunctionsService stepFunctionsService,
+ DurableExecutionService durableExecutionService,
Instance<ContainerTeardown> containerTeardowns,
PersistentPathValidator persistentPathValidator,
DynamoDbRuntime dynamoDbRuntime,
@@ -187,6 +190,7 @@ public class EmulatorLifecycle {
this.initLifecycleState = initLifecycleState;
this.schemaCreationWorker = schemaCreationWorker;
this.stepFunctionsService = stepFunctionsService;
+ this.durableExecutionService = durableExecutionService;
this.containerTeardowns = containerTeardowns;
this.persistentPathValidator = persistentPathValidator;
this.dynamoDbRuntime = dynamoDbRuntime;
@@ -235,6 +239,9 @@ public class EmulatorLifecycle {
schemaCreationWorker.recoverOrphans();
schemaCreationWorker.rehydrateSchemas();
stepFunctionsService.abortAbandonedExecutions();
+ if (config.services().lambda().enabled()) {
+ durableExecutionService.recoverAfterRestart();
+ }
// The selected DynamoDB backend is ready before any persisted stream consumer reads it.
dynamoDbRuntime.start();
diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/IamActionRegistry.java b/src/main/java/io/github/hectorvent/floci/services/iam/IamActionRegistry.java
index d9a17a8c7a..bfbc8c6321 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iam/IamActionRegistry.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iam/IamActionRegistry.java
@@ -62,6 +62,12 @@ public class IamActionRegistry {
rule("lambda", "POST", ".*/functions/[^/]+/url$", "lambda:CreateFunctionUrlConfig"),
rule("lambda", "PUT", ".*/functions/[^/]+/url$", "lambda:UpdateFunctionUrlConfig"),
rule("lambda", "DELETE", ".*/functions/[^/]+/url$", "lambda:DeleteFunctionUrlConfig"),
+ rule("lambda", "GET", ".*/functions/[^/]+/durable-executions$", "lambda:ListDurableExecutionsByFunction"),
+ rule("lambda", "POST", ".*/durable-executions/.+/checkpoint$", "lambda:CheckpointDurableExecution"),
+ rule("lambda", "GET", ".*/durable-executions/.+/state$", "lambda:GetDurableExecutionState"),
+ rule("lambda", "GET", ".*/durable-executions/.+/history$", "lambda:GetDurableExecutionHistory"),
+ rule("lambda", "POST", ".*/durable-executions/.+/stop$", "lambda:StopDurableExecution"),
+ rule("lambda", "GET", ".*/durable-executions/.+$", "lambda:GetDurableExecution"),
// ── DynamoDB (JSON 1.1, action from X-Amz-Target handled separately) ──
// Handled via Query-style action extraction in the filter
diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java b/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java
index 65524e6939..b18b26d224 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java
@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsQueryServiceResolver;
import io.github.hectorvent.floci.core.common.AwsRegions;
import io.github.hectorvent.floci.services.iam.model.ServerCertificate;
+import io.github.hectorvent.floci.services.lambda.LambdaArnUtils;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.inject.Instance;
import jakarta.inject.Inject;
@@ -20,6 +21,7 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
+import java.util.regex.Matcher;
/**
* Constructs the target resource ARN for a request so the policy evaluator
@@ -221,6 +223,10 @@ public class ResourceArnBuilder {
// ── Lambda ──────────────────────────────────────────────────────────────────
private String buildLambdaArn(String path, String region, String accountId) {
+ String executionArn = durableExecutionArn(path);
+ if (executionArn != null) {
+ return executionArn;
+ }
// path: /2015-03-31/functions/name or similar
String name = extractSegmentAfter(path, "functions");
if (name == null) return "*";
@@ -230,6 +236,17 @@ public class ResourceArnBuilder {
return AwsArnUtils.Arn.of("lambda", region, accountId, "function:" + name).toString();
}
+ /** Durable execution actions are authorized against the execution ARN, which carries the version. */
+ private static String durableExecutionArn(String path) {
+ String prefix = "/durable-executions/";
+ int executions = path.indexOf(prefix);
+ if (executions < 0) {
+ return null;
+ }
+ Matcher matcher = LambdaArnUtils.DURABLE_EXECUTION_ARN.matcher(path.substring(executions + prefix.length()));
+ return matcher.lookingAt() ? matcher.group() : null;
+ }
+
// ── SQS ─────────────────────────────────────────────────────────────────────
private String buildSqsArn(ContainerRequestContext ctx, String region, String accountId) {
String queueUrl = ctx.getUriInfo().getQueryParameters().getFirst("QueueUrl");
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaArnUtils.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaArnUtils.java
index 05ea35c069..dcd00ef712 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaArnUtils.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaArnUtils.java
@@ -29,6 +29,11 @@ public final class LambdaArnUtils {
private static final String EVENT_SOURCE_MAPPING_PREFIX = "event-source-mapping:";
+ /** The pattern AWS validates a durable execution ARN with, and quotes in its ValidationException. */
+ public static final Pattern DURABLE_EXECUTION_ARN = Pattern.compile(
+ "arn:([a-zA-Z0-9-]+):lambda:([a-zA-Z0-9-]+):(\\d{12}):function:([a-zA-Z0-9_-]+):"
+ + "(\\$LATEST(?:\\.PUBLISHED)?|[0-9]+)/durable-execution/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_-]+)");
+
private LambdaArnUtils() {}
/** {@code arn:<partition>:lambda:<region>:<account>:event-source-mapping:<uuid>}. */
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java
index a0c7deb147..b8998f16bd 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java
@@ -267,8 +267,10 @@ public class LambdaController {
String clientContext = type == InvocationType.RequestResponse
? decodeClientContext(headers.getHeaderString("X-Amz-Client-Context")) : null;
+ String durableExecutionName = headers.getHeaderString("X-Amz-Durable-Execution-Name");
- InvokeResult result = lambdaService.invoke(region, functionName, qualifier, payload, type, clientContext);
+ InvokeResult result = lambdaService.invoke(region, functionName, qualifier, payload, type, clientContext,
+ durableExecutionName);
if (type != InvocationType.Event
&& result.getPayload() != null
@@ -289,6 +291,9 @@ public class LambdaController {
}
builder.header("X-Amz-Executed-Version", result.getExecutedVersion());
builder.header("X-Amz-Request-Id", result.getRequestId());
+ if (result.getDurableExecutionArn() != null) {
+ builder.header("X-Amz-Durable-Execution-Arn", result.getDurableExecutionArn());
+ }
if (result.getPayload() != null && result.getPayload().length > 0) {
builder.entity(result.getPayload())
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
index c596986cfc..45722beddc 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
@@ -17,6 +17,11 @@ import io.github.hectorvent.floci.core.resource.ResourceProvider;
import io.github.hectorvent.floci.core.resource.SupportedResourceType;
import io.github.hectorvent.floci.services.ec2.Ec2Service;
import io.github.hectorvent.floci.services.ec2.model.Subnet;
+import io.github.hectorvent.floci.services.lambda.durable.DurableExecutionService;
+import io.github.hectorvent.floci.services.lambda.durable.DurableWire;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableErrorObject;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecution;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecutionStatus;
import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;
import io.github.hectorvent.floci.services.lambda.model.FunctionEventInvokeConfig;
import io.github.hectorvent.floci.services.lambda.model.InvocationType;
@@ -39,6 +44,7 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
@@ -55,6 +61,9 @@ import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.function.Consumer;
import java.util.regex.Pattern;
@@ -96,6 +105,8 @@ public class LambdaService implements ResourceProvider {
private static final int MAX_DURABLE_RETENTION_DAYS = 90;
private static final int DEFAULT_DURABLE_RETENTION_DAYS = 14;
private static final int MAX_FUNCTION_TIMEOUT_SECONDS = 900;
+ private static final int SYNC_DURABLE_GRACE_SECONDS = 5;
+ private static final Pattern DURABLE_EXECUTION_NAME_PATTERN = Pattern.compile("[a-zA-Z0-9-_]+");
private static final Pattern LAYER_VERSION_ARN_PATTERN = Pattern.compile(
"^((arn:(aws[a-zA-Z-]*)?:lambda:(eusc-)?[a-z]{2}((-gov)|(-iso([a-z]?)))?-[a-z]+-\\d{1}:\\d{12}:layer:[a-zA-Z0-9-_]+:[0-9]+)"
+ "|(arn:[a-zA-Z0-9-]+:lambda:::awslayer:[a-zA-Z0-9-_]+))$");
@@ -154,6 +165,7 @@ public class LambdaService implements ResourceProvider {
private final Ec2Service ec2Service;
/** Null in the constructors tests use, exactly as the other optional collaborators above are. */
private final CustomResourceLiveness customResourceLiveness;
+ private final DurableExecutionService durableExecutionService;
private final ObjectMapper objectMapper;
private Map<String, Integer> versionCounters = new ConcurrentHashMap<>();
private Map<String, FunctionEventInvokeConfig> eventInvokeConfigs = new ConcurrentHashMap<>();
@@ -227,6 +239,7 @@ public class LambdaService implements ResourceProvider {
this.layerService = null;
this.ec2Service = null;
this.customResourceLiveness = null;
+ this.durableExecutionService = null;
this.objectMapper = new ObjectMapper();
}
@@ -251,8 +264,10 @@ public class LambdaService implements ResourceProvider {
LambdaLayerService layerService,
Ec2Service ec2Service,
CustomResourceLiveness customResourceLiveness,
+ DurableExecutionService durableExecutionService,
ObjectMapper objectMapper) {
this.customResourceLiveness = customResourceLiveness;
+ this.durableExecutionService = durableExecutionService;
this.functionStore = functionStore;
this.executorService = executorService;
this.concurrencyLimiter = concurrencyLimiter;
@@ -1174,6 +1189,15 @@ public class LambdaService implements ResourceProvider {
*/
public InvokeResult invoke(String region, String functionName, String queryQualifier, byte[] payload,
InvocationType type, String clientContext) {
+ return invoke(region, functionName, queryQualifier, payload, type, clientContext, null);
+ }
+
+ /**
+ * Invokes a function. A durable function starts, or re-attaches to, the durable execution named
+ * by the {@code X-Amz-Durable-Execution-Name} header. A plain function ignores the name, as on AWS.
+ */
+ public InvokeResult invoke(String region, String functionName, String queryQualifier, byte[] payload,
+ InvocationType type, String clientContext, String durableExecutionName) {
validateInvokeQualifier(queryQualifier);
LambdaArnUtils.ResolvedFunctionRef ref = resolveWithRegion(region, functionName, queryQualifier);
String name = ref.name();
@@ -1186,12 +1210,94 @@ public class LambdaService implements ResourceProvider {
fn = targetResolver.resolveInvokeTarget(region, name, qualifier);
}
reportCustomResourceLiveness(payload);
+ InvokeResult durable = invokeIfDurable(fn, region, qualifier, durableExecutionName, payload, type);
+ if (durable != null) {
+ return durable;
+ }
InvokeResult result = executorService.invoke(fn, payload, type,
LambdaInvocationChain.currentDepth(), qualifier, clientContext);
result.setExecutedVersion(fn.getVersion());
return result;
}
+ /** Null for a plain function, and for a DryRun, which only checks the qualifier. */
+ private InvokeResult invokeIfDurable(LambdaFunction fn, String region, String qualifier,
+ String durableExecutionName, byte[] payload, InvocationType type) {
+ if (!fn.isDurable()) {
+ return null;
+ }
+ if (qualifier == null) {
+ throw new AwsException("InvalidParameterValueException",
+ "You cannot invoke a durable function using an unqualified ARN.", 400);
+ }
+ if (type == InvocationType.DryRun) {
+ return null;
+ }
+ InvokeResult result = invokeDurable(fn, region, durableExecutionName, payload, type);
+ result.setExecutedVersion(fn.getVersion());
+ return result;
+ }
+
+ /**
+ * RequestResponse waits for the whole execution, waits included, up to the 15 minute invocation
+ * limit. Event answers 202 as soon as the execution exists.
+ */
+ private InvokeResult invokeDurable(LambdaFunction fn, String region, String executionName, byte[] payload,
+ InvocationType type) {
+ boolean synchronous = type == InvocationType.RequestResponse;
+ if (synchronous && fn.getDurableExecutionTimeout() > MAX_FUNCTION_TIMEOUT_SECONDS) {
+ throw new AwsException("InvalidParameterValueException",
+ "You cannot synchronously invoke a durable function with an executionTimeout greater than "
+ + "15 minutes.", 400);
+ }
+ if (executionName != null) {
+ validateNonEmpty(executionName, "durableExecutionName", false);
+ validateMaxLength(executionName, "durableExecutionName", 64);
+ validatePattern(executionName, "durableExecutionName", DURABLE_EXECUTION_NAME_PATTERN);
+ }
+ String input = payload == null || payload.length == 0 ? "{}" : new String(payload, StandardCharsets.UTF_8);
+ DurableExecution execution = durableExecutionService.start(new DurableExecutionService.StartRequest(
+ fn.getAccountId(), region, fn.getFunctionName(), fn.getVersion(), executionName, input, synchronous));
+ String requestId = UUID.randomUUID().toString();
+ InvokeResult result;
+ if (!synchronous) {
+ result = new InvokeResult(202, null, new byte[0], null, requestId);
+ } else {
+ result = awaitDurableResult(execution.getExecutionArn(), requestId);
+ }
+ result.setDurableExecutionArn(execution.getExecutionArn());
+ return result;
+ }
+
+ private InvokeResult awaitDurableResult(String executionArn, String requestId) {
+ DurableExecution finished;
+ try {
+ // A little past the limit, so an execution that times out at 900 s reports its own error.
+ finished = durableExecutionService.awaitCompletion(executionArn)
+ .get(MAX_FUNCTION_TIMEOUT_SECONDS + SYNC_DURABLE_GRACE_SECONDS, TimeUnit.SECONDS);
+ } catch (TimeoutException e) {
+ return unhandledDurableResult(DurableErrorObject.of(
+ "Durable execution " + executionArn + " did not finish within 15 minutes",
+ "DurableExecution.InvocationTimedOut"), requestId);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return unhandledDurableResult(DurableErrorObject.of("Invocation interrupted", "Interrupted"), requestId);
+ } catch (ExecutionException e) {
+ return unhandledDurableResult(DurableErrorObject.of(
+ e.getCause() != null ? e.getCause().getMessage() : e.getMessage(), "InvocationError"), requestId);
+ }
+ if (finished.getStatus() == DurableExecutionStatus.SUCCEEDED) {
+ byte[] body = finished.getResult() == null ? new byte[0]
+ : finished.getResult().getBytes(StandardCharsets.UTF_8);
+ return new InvokeResult(200, null, body, null, requestId);
+ }
+ return unhandledDurableResult(finished.getError(), requestId);
+ }
+
+ private static InvokeResult unhandledDurableResult(DurableErrorObject error, String requestId) {
+ return new InvokeResult(200, "Unhandled", DurableWire.functionErrorPayload(error), null, requestId);
+ }
+
/**
* Invokes a Lambda target ARN using the account encoded in that ARN.
*
@@ -1216,6 +1322,10 @@ public class LambdaService implements ResourceProvider {
LambdaArnUtils.ResolvedFunctionRef ref = LambdaArnUtils.resolve(functionArn);
LambdaFunction fn = targetResolver.resolveInvokeTargetForAccount(
arn.accountId(), arn.region(), ref.name(), ref.qualifier());
+ InvokeResult durable = invokeIfDurable(fn, arn.region(), ref.qualifier(), null, payload, type);
+ if (durable != null) {
+ return durable;
+ }
InvokeResult result = executorService.invoke(fn, payload, type, chainDepth, ref.qualifier());
result.setExecutedVersion(fn.getVersion());
return result;
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java
new file mode 100644
index 0000000000..315754cfa7
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java
@@ -0,0 +1,411 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableErrorObject;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecution;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecutionStatus;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableHistoryEvent;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperation;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationAction;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationStatus;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationType;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationUpdate;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Consumer;
+import java.util.regex.Pattern;
+
+/**
+ * Applies one CheckpointDurableExecution batch. Validation and transitions run on a copy of the
+ * operations, so a rejected batch leaves the execution untouched, and timers that fell due are
+ * completed at the same time so a polling handler sees them in the response.
+ */
+final class DurableCheckpointApplier {
+
+ static final int MAX_OPERATION_PAYLOAD_BYTES = 256 * 1024;
+ static final int MAX_ERROR_BYTES = 32 * 1024;
+ private static final Pattern OPERATION_ID = Pattern.compile("[a-zA-Z0-9-_]{1,64}");
+
+ private DurableCheckpointApplier() {
+ }
+
+ record Outcome(boolean closed) {
+ }
+
+ static Outcome apply(DurableExecution execution, List<DurableOperationUpdate> updates, long now) {
+ validateBatch(updates);
+ Draft draft = new Draft(execution, now);
+ for (DurableOperationUpdate update : updates) {
+ draft.apply(update);
+ }
+ draft.commit();
+ return new Outcome(draft.closingStatus != null);
+ }
+
+ /** Completes waits and step retries whose time has come. The sweeper and every checkpoint call it. */
+ static boolean fireDueTimers(DurableExecution execution, long now) {
+ boolean changed = false;
+ for (DurableOperation operation : execution.getOperations().values()) {
+ if (operation.getType() == DurableOperationType.WAIT
+ && operation.getStatus() == DurableOperationStatus.STARTED
+ && operation.getScheduledEndTimestamp() != null
+ && operation.getScheduledEndTimestamp() <= now) {
+ operation.setStatus(DurableOperationStatus.SUCCEEDED);
+ operation.setEndTimestamp(now);
+ operation.setChangeSequence(execution.nextChangeSequence());
+ Map<String, Object> details = new LinkedHashMap<>();
+ details.put("Duration", waitSeconds(operation));
+ DurableHistory.operationEvent(execution, operation, "WaitSucceeded", now, details);
+ changed = true;
+ } else if (operation.getType() == DurableOperationType.STEP
+ && operation.getStatus() == DurableOperationStatus.PENDING
+ && operation.getNextAttemptTimestamp() != null
+ && operation.getNextAttemptTimestamp() <= now) {
+ operation.setStatus(DurableOperationStatus.READY);
+ operation.setNextAttemptTimestamp(null);
+ operation.setChangeSequence(execution.nextChangeSequence());
+ changed = true;
+ }
+ }
+ return changed;
+ }
+
+ private static void validateBatch(List<DurableOperationUpdate> updates) {
+ long executionUpdates = updates.stream().filter(update -> update.type() == DurableOperationType.EXECUTION).count();
+ if (executionUpdates > 1) {
+ throw invalid("Cannot checkpoint multiple EXECUTION updates.");
+ }
+ Map<String, DurableOperationUpdate> seen = new HashMap<>();
+ for (int i = 0; i < updates.size(); i++) {
+ DurableOperationUpdate update = updates.get(i);
+ if (update.type() == DurableOperationType.EXECUTION) {
+ if (i != updates.size() - 1) {
+ throw invalid("EXECUTION checkpoint must be the last update.");
+ }
+ continue;
+ }
+ DurableOperationUpdate previous = seen.put(update.id(), update);
+ if (previous != null && !(previous.action() == DurableOperationAction.START
+ && previous.type() == update.type() && closesInSameBatch(update))) {
+ throw invalid("Cannot checkpoint multiple operations with the same ID.");
+ }
+ }
+ }
+
+ /** A START may be followed by the action that finishes it in the same batch, once. */
+ private static boolean closesInSameBatch(DurableOperationUpdate update) {
+ return switch (update.type()) {
+ case STEP -> update.action() == DurableOperationAction.SUCCEED
+ || update.action() == DurableOperationAction.FAIL
+ || update.action() == DurableOperationAction.RETRY;
+ case CONTEXT -> update.action() == DurableOperationAction.SUCCEED
+ || update.action() == DurableOperationAction.FAIL;
+ default -> false;
+ };
+ }
+
+ static AwsException invalid(String message) {
+ return new AwsException("InvalidParameterValueException", message, 400);
+ }
+
+ private static int waitSeconds(DurableOperation operation) {
+ return (int) ((operation.getScheduledEndTimestamp() - operation.getStartTimestamp()) / 1000);
+ }
+
+ static int utf8Length(String value) {
+ return value == null ? 0 : value.getBytes(StandardCharsets.UTF_8).length;
+ }
+
+ /** The batch's copy of the operations, its queued history events and the closing outcome. */
+ private static final class Draft {
+
+ private final DurableExecution execution;
+ private final long now;
+ private final LinkedHashMap<String, DurableOperation> operations = new LinkedHashMap<>();
+ private final List<Consumer<DurableExecution>> events = new ArrayList<>();
+ private long changeSequence;
+ private DurableExecutionStatus closingStatus;
+ private String closingResult;
+ private DurableErrorObject closingError;
+
+ Draft(DurableExecution execution, long now) {
+ this.execution = execution;
+ this.now = now;
+ this.changeSequence = execution.getChangeSequence();
+ for (Map.Entry<String, DurableOperation> entry : execution.getOperations().entrySet()) {
+ operations.put(entry.getKey(), entry.getValue().copy());
+ }
+ }
+
+ void apply(DurableOperationUpdate update) {
+ if (update.type() == DurableOperationType.EXECUTION) {
+ applyExecution(update);
+ return;
+ }
+ if (update.id() == null || !OPERATION_ID.matcher(update.id()).matches()) {
+ throw invalid("Invalid operation id.");
+ }
+ DurableOperation existing = operations.get(update.id());
+ if (existing != null) {
+ requireConsistent(existing, update);
+ }
+ if (update.parentId() != null) {
+ DurableOperation parent = operations.get(update.parentId());
+ if (parent == null || parent.getType() != DurableOperationType.CONTEXT) {
+ throw invalid("Invalid parent operation id.");
+ }
+ }
+ requireErrorSize(update);
+ if (utf8Length(update.payload()) > MAX_OPERATION_PAYLOAD_BYTES) {
+ throw invalid(update.type() + " payload size must be less than or equal to "
+ + MAX_OPERATION_PAYLOAD_BYTES + " bytes.");
+ }
+ switch (update.type()) {
+ case CONTEXT -> applyContext(update, existing);
+ case STEP -> applyStep(update, existing);
+ case WAIT -> applyWait(update, existing);
+ case CALLBACK, CHAINED_INVOKE -> throw invalid(update.type() + " operations are not supported yet");
+ default -> throw invalid("Unknown operation type.");
+ }
+ }
+
+ void commit() {
+ execution.setOperations(operations);
+ execution.setChangeSequence(changeSequence);
+ for (Consumer<DurableExecution> event : events) {
+ event.accept(execution);
+ }
+ if (closingStatus != null) {
+ execution.setStatus(closingStatus);
+ execution.setResult(closingResult);
+ execution.setError(closingError);
+ execution.setEndTimestamp(now);
+ if (closingStatus == DurableExecutionStatus.SUCCEEDED) {
+ DurableHistory.executionSucceeded(execution, now);
+ } else {
+ DurableHistory.executionEnded(execution, "ExecutionFailed", now);
+ }
+ }
+ }
+
+ private void applyExecution(DurableOperationUpdate update) {
+ if (update.action() != DurableOperationAction.SUCCEED && update.action() != DurableOperationAction.FAIL) {
+ throw invalid("Invalid EXECUTION action.");
+ }
+ requireMatchingResult(update);
+ requireErrorSize(update);
+ if (utf8Length(update.payload()) > execution.getMaxResultBytes()) {
+ throw invalid("Execution output payload size must be less than or equal to "
+ + execution.getMaxResultBytes() + " bytes.");
+ }
+ boolean succeeded = update.action() == DurableOperationAction.SUCCEED;
+ closingStatus = succeeded ? DurableExecutionStatus.SUCCEEDED : DurableExecutionStatus.FAILED;
+ closingResult = succeeded ? update.payload() : null;
+ closingError = succeeded ? null : update.error();
+ DurableOperation operation = operations.get(execution.getExecutionId());
+ if (operation != null) {
+ operation.setStatus(succeeded ? DurableOperationStatus.SUCCEEDED : DurableOperationStatus.FAILED);
+ operation.setEndTimestamp(now);
+ touch(operation);
+ }
+ }
+
+ private void applyContext(DurableOperationUpdate update, DurableOperation existing) {
+ switch (update.action()) {
+ case START -> {
+ if (existing != null) {
+ throw invalid("Cannot start a CONTEXT that already exist.");
+ }
+ DurableOperation operation = create(update);
+ event(operation, "ContextStarted", new LinkedHashMap<>());
+ }
+ case SUCCEED, FAIL -> {
+ if (existing == null || existing.getStatus() != DurableOperationStatus.STARTED) {
+ throw invalid("Invalid current CONTEXT state to close.");
+ }
+ boolean succeeded = update.action() == DurableOperationAction.SUCCEED;
+ existing.setStatus(succeeded ? DurableOperationStatus.SUCCEEDED : DurableOperationStatus.FAILED);
+ existing.setResult(succeeded ? update.payload() : null);
+ existing.setError(succeeded ? null : update.error());
+ if (update.replayChildren() != null) {
+ existing.setReplayChildren(update.replayChildren());
+ }
+ existing.setEndTimestamp(now);
+ touch(existing);
+ event(existing, succeeded ? "ContextSucceeded" : "ContextFailed", resultOrError(existing));
+ }
+ default -> throw invalid("Invalid action for the given operation type.");
+ }
+ }
+
+ private void applyStep(DurableOperationUpdate update, DurableOperation existing) {
+ switch (update.action()) {
+ case START -> {
+ if (update.nextAttemptDelaySeconds() != null) {
+ throw invalid("Invalid StepOptions for the given action.");
+ }
+ if (existing == null) {
+ event(create(update), "StepStarted", new LinkedHashMap<>());
+ } else if (existing.getStatus() == DurableOperationStatus.READY) {
+ existing.setStatus(DurableOperationStatus.STARTED);
+ touch(existing);
+ event(existing, "StepStarted", new LinkedHashMap<>());
+ } else {
+ throw invalid("Invalid current STEP state to start.");
+ }
+ }
+ case SUCCEED, FAIL, RETRY -> closeStep(update, existing);
+ default -> throw invalid("Invalid action for the given operation type.");
+ }
+ }
+
+ private void closeStep(DurableOperationUpdate update, DurableOperation existing) {
+ DurableOperationAction action = update.action();
+ requireMatchingResult(update);
+ if (action == DurableOperationAction.RETRY && update.payload() != null && update.error() != null) {
+ throw invalid("Cannot provide both error and payload to RETRY a STEP.");
+ }
+ if ((action == DurableOperationAction.RETRY) != (update.nextAttemptDelaySeconds() != null)) {
+ throw invalid("Invalid StepOptions for the given action.");
+ }
+ DurableOperation operation = existing;
+ if (operation == null) {
+ operation = create(update);
+ } else if (operation.getStatus() != DurableOperationStatus.STARTED
+ && operation.getStatus() != DurableOperationStatus.READY) {
+ throw invalid(action == DurableOperationAction.RETRY
+ ? "Invalid current STEP state to re-attempt." : "Invalid current STEP state to close.");
+ }
+ int attempt = (operation.getAttempt() == null ? 0 : operation.getAttempt()) + 1;
+ operation.setAttempt(attempt);
+ operation.setResult(update.payload());
+ operation.setError(update.error());
+ Map<String, Object> details = resultOrError(operation);
+ Map<String, Object> retryDetails = new LinkedHashMap<>();
+ retryDetails.put("CurrentAttempt", attempt);
+ details.put("RetryDetails", retryDetails);
+ switch (action) {
+ case SUCCEED -> {
+ operation.setStatus(DurableOperationStatus.SUCCEEDED);
+ operation.setEndTimestamp(now);
+ event(operation, "StepSucceeded", details);
+ }
+ case FAIL -> {
+ operation.setStatus(DurableOperationStatus.FAILED);
+ operation.setEndTimestamp(now);
+ event(operation, "StepFailed", details);
+ }
+ default -> {
+ operation.setStatus(DurableOperationStatus.PENDING);
+ operation.setNextAttemptTimestamp(now + update.nextAttemptDelaySeconds() * 1000L);
+ retryDetails.put("NextAttemptDelaySeconds", update.nextAttemptDelaySeconds());
+ event(operation, update.error() != null ? "StepFailed" : "StepSucceeded", details);
+ }
+ }
+ touch(operation);
+ }
+
+ private void applyWait(DurableOperationUpdate update, DurableOperation existing) {
+ switch (update.action()) {
+ case START -> {
+ if (existing != null) {
+ throw invalid("Cannot start a WAIT that already exist.");
+ }
+ if (update.waitSeconds() == null) {
+ throw invalid("Update for WAIT operation requires WaitOptions.");
+ }
+ DurableOperation operation = create(update);
+ operation.setScheduledEndTimestamp(now + update.waitSeconds() * 1000L);
+ Map<String, Object> details = new LinkedHashMap<>();
+ details.put("Duration", update.waitSeconds());
+ details.put("ScheduledEndTimestamp", operation.getScheduledEndTimestamp());
+ event(operation, "WaitStarted", details);
+ }
+ case CANCEL -> {
+ if (existing == null || existing.getStatus() != DurableOperationStatus.STARTED) {
+ throw invalid("Cannot cancel a WAIT that does not exist or has already completed.");
+ }
+ existing.setStatus(DurableOperationStatus.CANCELLED);
+ existing.setEndTimestamp(now);
+ touch(existing);
+ event(existing, "WaitCancelled", new LinkedHashMap<>());
+ }
+ default -> throw invalid("Invalid action for the given operation type.");
+ }
+ }
+
+ private static void requireMatchingResult(DurableOperationUpdate update) {
+ if (update.action() == DurableOperationAction.SUCCEED && update.error() != null) {
+ throw invalid("Cannot provide an Error for SUCCEED action.");
+ }
+ if (update.action() == DurableOperationAction.FAIL && update.payload() != null) {
+ throw invalid("Cannot provide a Payload for FAIL action.");
+ }
+ }
+
+ private static void requireErrorSize(DurableOperationUpdate update) {
+ if (update.error() != null && utf8Length(errorJson(update.error())) >= MAX_ERROR_BYTES) {
+ throw invalid("Error object size must be less than " + MAX_ERROR_BYTES + " bytes.");
+ }
+ }
+
+ private DurableOperation create(DurableOperationUpdate update) {
+ DurableOperation operation = new DurableOperation();
+ operation.setId(update.id());
+ operation.setParentId(update.parentId());
+ operation.setName(update.name());
+ operation.setType(update.type());
+ operation.setSubType(update.subType());
+ operation.setStatus(DurableOperationStatus.STARTED);
+ operation.setStartTimestamp(now);
+ touch(operation);
+ operations.put(update.id(), operation);
+ return operation;
+ }
+
+ private void touch(DurableOperation operation) {
+ changeSequence++;
+ operation.setChangeSequence(changeSequence);
+ }
+
+ /** Events are queued against the operation copy and appended only once the whole batch is accepted. */
+ private void event(DurableOperation operation, String eventType, Map<String, Object> details) {
+ events.add(target -> DurableHistory.operationEvent(target, operation, eventType, now, details));
+ }
+
+ private static void requireConsistent(DurableOperation existing, DurableOperationUpdate update) {
+ if (existing.getType() != update.type()) {
+ throw invalid("Inconsistent operation type.");
+ }
+ if (update.subType() != null && !Objects.equals(existing.getSubType(), update.subType())) {
+ throw invalid("Inconsistent operation subtype.");
+ }
+ if (update.name() != null && !Objects.equals(existing.getName(), update.name())) {
+ throw invalid("Inconsistent operation name.");
+ }
+ if (!Objects.equals(existing.getParentId(), update.parentId())) {
+ throw invalid("Inconsistent parent operation id.");
+ }
+ }
+
+ private static Map<String, Object> resultOrError(DurableOperation operation) {
+ Map<String, Object> details = new LinkedHashMap<>();
+ if (operation.getError() != null) {
+ details.put("Error", DurableHistory.errorWrapper(operation.getError()));
+ } else if (operation.getResult() != null) {
+ details.put("Result", DurableHistory.payloadWrapper(operation.getResult()));
+ }
+ return details;
+ }
+
+ private static String errorJson(DurableErrorObject error) {
+ return DurableWire.error(error).toString();
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionController.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionController.java
new file mode 100644
index 0000000000..fb9cb0e408
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionController.java
@@ -0,0 +1,297 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.PaginatedResult;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.services.lambda.LambdaArnUtils;
+import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableErrorObject;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecution;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecutionStatus;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableHistoryEvent;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import jakarta.inject.Inject;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.HttpHeaders;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeParseException;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * The Lambda durable execution APIs under the {@code /2025-12-01} version prefix. The literal
+ * prefix keeps these routes off S3's bucket catch-all, as the other versioned Lambda controllers do.
+ *
+ * <p>An execution ARN holds {@code /}. The AWS SDKs send it percent-encoded as one path segment,
+ * and the CLI and curl send it raw, so the ARN templates match greedily.
+ */
+@Path("/2025-12-01")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.WILDCARD)
+public class DurableExecutionController {
+
+ /** Valid Statuses filters for a state Floci never puts an execution in. */
+ private static final Set<String> UNMODELLED_STATUSES = Set.of("PAUSED", "PAUSING", "DELETING");
+
+ private final DurableExecutionService service;
+ private final LambdaService lambdaService;
+ private final RegionResolver regionResolver;
+ private final ObjectMapper objectMapper;
+
+ @Inject
+ public DurableExecutionController(DurableExecutionService service, LambdaService lambdaService,
+ RegionResolver regionResolver, ObjectMapper objectMapper) {
+ this.service = service;
+ this.lambdaService = lambdaService;
+ this.regionResolver = regionResolver;
+ this.objectMapper = objectMapper;
+ }
+
+ @POST
+ @Path("/durable-executions/{arn: .+}/checkpoint")
+ public Response checkpoint(@Context HttpHeaders headers, @PathParam("arn") String arn, String body) {
+ Map<String, Object> request = readObject(body);
+ DurableExecutionService.CheckpointResult result = service.checkpoint(ownedArn(headers, arn),
+ stringMember(request, "CheckpointToken"), stringMember(request, "ClientToken"),
+ DurableWire.parseUpdates(request.get("Updates")));
+ ObjectNode response = objectMapper.createObjectNode();
+ if (result.checkpointToken() != null) {
+ response.put("CheckpointToken", result.checkpointToken());
+ }
+ response.set("NewExecutionState", DurableWire.operations(result.newExecutionState(), null));
+ return Response.ok(response).build();
+ }
+
+ @GET
+ @Path("/durable-executions/{arn: .+}/state")
+ public Response getState(@Context HttpHeaders headers, @PathParam("arn") String arn,
+ @QueryParam("CheckpointToken") String checkpointToken,
+ @QueryParam("Marker") String marker,
+ @QueryParam("MaxItems") String maxItems) {
+ Integer pageSize = parseMaxItems(maxItems);
+ requireValid(DurableExecutionService.maxItemsViolation(pageSize));
+ DurableExecutionService.StatePage page = service.getState(ownedArn(headers, arn), checkpointToken, marker,
+ pageSize);
+ return Response.ok(DurableWire.operations(page.operations(), page.nextMarker())).build();
+ }
+
+ @GET
+ @Path("/durable-executions/{arn: .+}/history")
+ public Response getHistory(@Context HttpHeaders headers, @PathParam("arn") String arn,
+ @QueryParam("IncludeExecutionData") String includeExecutionData,
+ @QueryParam("Marker") String marker,
+ @QueryParam("MaxItems") String maxItems,
+ @QueryParam("ReverseOrder") String reverseOrder) {
+ Integer pageSize = parseMaxItems(maxItems);
+ requireValid(DurableExecutionService.maxItemsViolation(pageSize));
+ PaginatedResult<DurableHistoryEvent> page = service.history(ownedArn(headers, arn), pageSize, marker,
+ Boolean.parseBoolean(reverseOrder));
+ boolean includeData = Boolean.parseBoolean(includeExecutionData);
+ ObjectNode response = objectMapper.createObjectNode();
+ ArrayNode events = response.putArray("Events");
+ for (DurableHistoryEvent event : page.items()) {
+ events.add(DurableWire.historyEvent(event, includeData));
+ }
+ if (page.nextToken() != null) {
+ response.put("NextMarker", page.nextToken());
+ }
+ return Response.ok(response).build();
+ }
+
+ @POST
+ @Path("/durable-executions/{arn: .+}/stop")
+ public Response stop(@Context HttpHeaders headers, @PathParam("arn") String arn, String body) {
+ DurableErrorObject error = body == null || body.isBlank() ? null : DurableWire.parseError(readObject(body));
+ if (error != null && error.getErrorMessage() == null && error.getErrorType() == null
+ && error.getErrorData() == null && error.getStackTrace() == null) {
+ error = null;
+ }
+ DurableExecution execution = service.stop(ownedArn(headers, arn), error);
+ ObjectNode response = objectMapper.createObjectNode();
+ response.put("StopTimestamp", execution.getEndTimestamp() / 1000.0);
+ return Response.ok(response).build();
+ }
+
+ /** IncludeExecutionData defaults to true here, unlike GetDurableExecutionHistory. */
+ @GET
+ @Path("/durable-executions/{arn: .+}")
+ public Response getExecution(@Context HttpHeaders headers, @PathParam("arn") String arn,
+ @QueryParam("IncludeExecutionData") String includeExecutionData) {
+ boolean includeData = includeExecutionData == null || Boolean.parseBoolean(includeExecutionData);
+ return Response.ok(DurableWire.execution(service.get(ownedArn(headers, arn)), includeData)).build();
+ }
+
+ @GET
+ @Path("/functions/{functionName}/durable-executions")
+ public Response listByFunction(@Context HttpHeaders headers,
+ @PathParam("functionName") String functionName,
+ @QueryParam("Qualifier") String qualifier,
+ @QueryParam("DurableExecutionName") String durableExecutionName,
+ @QueryParam("Statuses") List<String> statuses,
+ @QueryParam("StartedAfter") String startedAfter,
+ @QueryParam("StartedBefore") String startedBefore,
+ @QueryParam("ReverseOrder") String reverseOrder,
+ @QueryParam("Marker") String marker,
+ @QueryParam("MaxItems") String maxItems) {
+ String region = regionResolver.resolveRegion(headers);
+ // AWS converts types first, then checks values, then the single-status rule, then the function.
+ Long after = parseTimestamp(startedAfter);
+ Long before = parseTimestamp(startedBefore);
+ Integer pageSize = parseMaxItems(maxItems);
+ requireValid(DurableExecutionService.maxItemsViolation(pageSize), statusesViolation(statuses));
+ Set<DurableExecutionStatus> statusFilter = parseStatuses(statuses);
+ LambdaFunction fn = lambdaService.getFunction(region, functionName, qualifier);
+ boolean qualified = (qualifier != null && !qualifier.isBlank())
+ || LambdaArnUtils.resolve(functionName).qualifier() != null;
+ DurableExecutionService.ListRequest request = new DurableExecutionService.ListRequest(fn.getAccountId(),
+ region, fn.getFunctionName(), qualified ? fn.getVersion() : null, durableExecutionName, statusFilter,
+ after, before, Boolean.parseBoolean(reverseOrder), pageSize, marker);
+ PaginatedResult<DurableExecution> page = service.list(request);
+ ObjectNode response = objectMapper.createObjectNode();
+ ArrayNode executions = response.putArray("DurableExecutions");
+ for (DurableExecution execution : page.items()) {
+ executions.add(DurableWire.executionSummary(execution));
+ }
+ if (page.nextToken() != null) {
+ response.put("NextMarker", page.nextToken());
+ }
+ return Response.ok(response).build();
+ }
+
+ /**
+ * An execution of another account is not found. One of another region reads as "Function not found",
+ * as AWS answers for an execution ARN of another region.
+ */
+ private String ownedArn(HttpHeaders headers, String arn) {
+ DurableExecutionService.ArnParts parts = DurableExecutionService.parseArn(arn);
+ if (!parts.accountId().equals(regionResolver.getAccountId())) {
+ throw new AwsException("ResourceNotFoundException", DurableExecutionService.NOT_FOUND, 404);
+ }
+ if (!parts.region().equals(regionResolver.resolveRegion(headers))) {
+ throw new AwsException("ResourceNotFoundException", "Function not found", 404);
+ }
+ return arn;
+ }
+
+ private Map<String, Object> readObject(String body) {
+ if (body == null || body.isBlank()) {
+ return Map.of();
+ }
+ try {
+ Map<String, Object> request = objectMapper.readValue(body, new TypeReference<Map<String, Object>>() {});
+ return request == null ? Map.of() : request;
+ } catch (IOException e) {
+ throw new AwsException("SerializationException", "Request body must be a JSON object", 400);
+ }
+ }
+
+ private static String stringMember(Map<String, Object> request, String member) {
+ Object value = request.get(member);
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof String s) {
+ return s;
+ }
+ throw new AwsException("SerializationException", member + " must be a string", 400);
+ }
+
+ private static Integer parseMaxItems(String maxItems) {
+ if (maxItems == null || maxItems.isEmpty()) {
+ return null;
+ }
+ try {
+ return Integer.valueOf(maxItems);
+ } catch (NumberFormatException e) {
+ throw new AwsException("SerializationException", "'" + maxItems + "' can not be converted to Integer", 400);
+ }
+ }
+
+ private static void requireValid(String... violations) {
+ List<String> found = Arrays.stream(violations).filter(Objects::nonNull).toList();
+ if (!found.isEmpty()) {
+ throw DurableExecutionService.validationError(found);
+ }
+ }
+
+ private static String statusesViolation(List<String> statuses) {
+ if (statuses == null) {
+ return null;
+ }
+ for (String status : statuses) {
+ if (!UNMODELLED_STATUSES.contains(status) && !isExecutionStatus(status)) {
+ return "Value '" + statuses + "' at 'statuses' failed to satisfy constraint: Member must satisfy "
+ + "constraint: [Member must satisfy enum value set: [SUCCEEDED, TIMED_OUT, DELETING, STOPPED, "
+ + "PAUSED, PAUSING, FAILED, RUNNING], Member must not be null]";
+ }
+ }
+ return null;
+ }
+
+ private static boolean isExecutionStatus(String status) {
+ for (DurableExecutionStatus candidate : DurableExecutionStatus.values()) {
+ if (candidate.name().equals(status)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** Values are already valid. A status Floci never assigns matches nothing. */
+ private static Set<DurableExecutionStatus> parseStatuses(List<String> statuses) {
+ if (statuses == null || statuses.isEmpty()) {
+ return null;
+ }
+ if (statuses.size() > 1) {
+ throw new AwsException("InvalidParameterValueException", "Cannot filter by more than one status", 400);
+ }
+ String status = statuses.get(0);
+ return UNMODELLED_STATUSES.contains(status) ? EnumSet.noneOf(DurableExecutionStatus.class)
+ : EnumSet.of(DurableExecutionStatus.valueOf(status));
+ }
+
+ /** AWS takes ISO 8601, a plain date or epoch seconds. */
+ private static Long parseTimestamp(String value) {
+ if (value == null || value.isBlank()) {
+ return null;
+ }
+ try {
+ return Instant.parse(value).toEpochMilli();
+ } catch (DateTimeParseException notInstant) {
+ try {
+ return LocalDate.parse(value).atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli();
+ } catch (DateTimeParseException notDate) {
+ try {
+ double seconds = Double.parseDouble(value);
+ if (Double.isFinite(seconds)) {
+ return (long) (seconds * 1000);
+ }
+ } catch (NumberFormatException notNumber) {
+ // Falls through to the error AWS gives for every unreadable timestamp.
+ }
+ throw new AwsException("SerializationException", "'" + value + "' can not be converted to Date", 400);
+ }
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java
new file mode 100644
index 0000000000..2a7b6a7770
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java
@@ -0,0 +1,796 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.Pagination;
+import io.github.hectorvent.floci.core.common.PaginatedResult;
+import io.github.hectorvent.floci.core.common.Resettable;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend.AccountEntry;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.core.storage.WriteProfile;
+import io.github.hectorvent.floci.services.lambda.LambdaArnUtils;
+import io.github.hectorvent.floci.services.lambda.durable.DurableFunctionInvoker.DurableInvocationResult;
+import io.github.hectorvent.floci.services.lambda.durable.DurableFunctionInvoker.ResolvedDurableTarget;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableCheckpointReplay;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableErrorObject;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecution;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecutionStatus;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableHistoryEvent;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperation;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationStatus;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationType;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationUpdate;
+import jakarta.annotation.PreDestroy;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.regex.Matcher;
+
+/**
+ * The durable execution engine behind Lambda Durable Functions. It stores checkpointed operations,
+ * re-invokes the function when something it waited for is ready, and serves the execution APIs.
+ *
+ * <p>Timers are persisted deadlines fired by {@link DurableExecutionSweeper}, as SWF does, so a
+ * wait survives a restart. One execution runs one invocation at a time. Every change collects
+ * effects, such as launching an invocation or waking sync callers, that run after the
+ * per-execution lock is released. Lambda is never invoked while a lock is held.
+ */
+@ApplicationScoped
+public class DurableExecutionService implements Resettable {
+
+ private static final Logger LOG = Logger.getLogger(DurableExecutionService.class);
+
+ /**
+ * AWS retries a crashing invocation with growing delays for a few minutes, then fails the
+ * execution. Floci keeps that shape over a shorter run with 1, 2, 4 and 8 second delays.
+ */
+ static final int INVOCATION_RETRY_MAX_ATTEMPTS = 5;
+ static final Duration INVOCATION_RETRY_DELAY = Duration.ofSeconds(1);
+ static final int ASYNC_PAYLOAD_LIMIT = 1024 * 1024;
+ static final int SYNC_PAYLOAD_LIMIT = 6 * 1024 * 1024;
+ static final int DEFAULT_PAGE_SIZE = 100;
+ private static final int LOCK_STRIPES = 256;
+ static final int MAX_PAGE_SIZE = 1000;
+ static final String INVALID_TOKEN = "Invalid checkpoint token";
+ static final String TOKEN_FOR_OTHER_EXECUTION = "Checkpoint token is not valid for the durable execution ARN";
+ static final String NOT_FOUND = "Durable Execution does not exist";
+
+ private final AccountAwareStorageBackend<DurableExecution> store;
+ private final ObjectMapper objectMapper;
+ private final Clock clock;
+ private final DurableFunctionInvoker invoker;
+ private final Executor launchExecutor;
+ private final long invocationRetryDelayMillis;
+ /** Striped, so a lookup of an unknown ARN never grows a map. Two executions may share a lock, never nest one. */
+ private final Object[] executionLocks = newLocks(LOCK_STRIPES);
+ private final ConcurrentHashMap<String, Object> startLocks = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String, CompletableFuture<DurableExecution>> completions = new ConcurrentHashMap<>();
+ /** Moves on reset and shutdown so an invocation result from before cannot be written afterwards. */
+ private final AtomicLong generation = new AtomicLong();
+
+ @Inject
+ public DurableExecutionService(StorageFactory storageFactory, ObjectMapper objectMapper, Clock clock,
+ DurableFunctionInvoker invoker) {
+ this(storageFactory, objectMapper, clock, invoker,
+ Executors.newThreadPerTaskExecutor(Thread.ofVirtual().name("lambda-durable-", 0).factory()),
+ INVOCATION_RETRY_DELAY);
+ }
+
+ /** For tests. An inline executor and a zero delay make every invocation chain synchronous. */
+ DurableExecutionService(StorageFactory storageFactory, ObjectMapper objectMapper, Clock clock,
+ DurableFunctionInvoker invoker, Executor launchExecutor, Duration invocationRetryDelay) {
+ this.store = storageFactory.create("lambda", "lambda-durable-executions.json",
+ new TypeReference<Map<String, DurableExecution>>() {}, WriteProfile.APPEND_HEAVY);
+ this.objectMapper = objectMapper;
+ this.clock = clock;
+ this.invoker = invoker;
+ this.launchExecutor = launchExecutor;
+ this.invocationRetryDelayMillis = invocationRetryDelay.toMillis();
+ }
+
+ public record StartRequest(String accountId, String region, String functionName, String qualifier,
+ String executionName, String inputPayload, boolean synchronous) {
+ }
+
+ public record ListRequest(String accountId, String region, String functionName, String version,
+ String executionName, Set<DurableExecutionStatus> statuses, Long startedAfter,
+ Long startedBefore, boolean reverseOrder, Integer maxItems, String marker) {
+ }
+
+ /** The token is null when the batch closed the execution. The SDK reads that as "completed". */
+ public record CheckpointResult(String checkpointToken, List<DurableOperation> newExecutionState) {
+ }
+
+ public record StatePage(List<DurableOperation> operations, String nextMarker) {
+ }
+
+ public DurableExecution start(StartRequest request) {
+ ResolvedDurableTarget target = invoker.resolve(request.accountId(), request.region(),
+ request.functionName(), request.qualifier());
+ if (!target.durable()) {
+ throw new AwsException("InvalidParameterValueException",
+ "Function " + target.functionName() + " is not a durable function", 400);
+ }
+ String name = request.executionName() != null ? request.executionName() : UUID.randomUUID().toString();
+ List<Runnable> effects = new ArrayList<>();
+ DurableExecution execution;
+ synchronized (startLock(request.accountId(), request.region())) {
+ Optional<DurableExecution> existing = request.executionName() == null ? Optional.empty() : store
+ .scanForAccount(request.accountId(), key -> key.startsWith(request.region() + "/"))
+ .stream()
+ .filter(candidate -> name.equals(candidate.getName())
+ && target.functionName().equals(candidate.getFunctionName()))
+ .findFirst();
+ if (existing.isPresent()) {
+ // Names are unique per function across versions. Only the same version and payload re-attaches.
+ if (!existing.get().getVersion().equals(target.version())
+ || !Objects.equals(existing.get().getInputPayload(), request.inputPayload())) {
+ throw new AwsException("DurableExecutionAlreadyStartedException",
+ "Execution already started: " + existing.get().getExecutionArn(), 409);
+ }
+ return existing.get();
+ }
+ long now = clock.millis();
+ execution = newExecution(request, target, name, now);
+ trigger(execution, effects);
+ store.putForAccount(request.accountId(), storeKey(execution), execution);
+ }
+ runEffects(effects);
+ return execution;
+ }
+
+ public CompletableFuture<DurableExecution> awaitCompletion(String executionArn) {
+ ArnParts arn = parseArn(executionArn);
+ synchronized (lockFor(arn)) {
+ DurableExecution execution = load(arn);
+ if (execution.isClosed()) {
+ return CompletableFuture.completedFuture(execution);
+ }
+ return completions.computeIfAbsent(arn.lockKey(), ignored -> new CompletableFuture<>());
+ }
+ }
+
+ public DurableExecution get(String executionArn) {
+ ArnParts arn = parseArn(executionArn);
+ synchronized (lockFor(arn)) {
+ return load(arn);
+ }
+ }
+
+ /** A non-null but empty {@code statuses} matches nothing, and the page parameters are still checked. */
+ public PaginatedResult<DurableExecution> list(ListRequest request) {
+ // Pagination orders by the cursor string, so the cursor carries the direction. Newest first is the default.
+ boolean oldestFirst = request.reverseOrder();
+ List<Cursored<DurableExecution>> matching = new ArrayList<>();
+ boolean matchesNothing = request.statuses() != null && request.statuses().isEmpty();
+ List<DurableExecution> candidates = matchesNothing ? List.of() : store.scanForAccount(request.accountId(),
+ key -> key.startsWith(request.region() + "/"));
+ for (DurableExecution execution : candidates) {
+ if (!request.functionName().equals(execution.getFunctionName())) {
+ continue;
+ }
+ if (request.version() != null && !request.version().equals(execution.getVersion())) {
+ continue;
+ }
+ if (request.executionName() != null && !request.executionName().equals(execution.getName())) {
+ continue;
+ }
+ if (request.statuses() != null && !request.statuses().contains(execution.getStatus())) {
+ continue;
+ }
+ if (request.startedAfter() != null && execution.getStartTimestamp() <= request.startedAfter()) {
+ continue;
+ }
+ if (request.startedBefore() != null && request.startedBefore() <= execution.getStartTimestamp()) {
+ continue;
+ }
+ matching.add(new Cursored<>(orderedCursor(execution.getStartTimestamp(), oldestFirst) + ":"
+ + execution.getExecutionId(), execution));
+ }
+ return paginate(matching, pageSize(request.maxItems()), request.marker(),
+ oldestFirst ? "durable-executions-asc" : "durable-executions-desc");
+ }
+
+ public PaginatedResult<DurableHistoryEvent> history(String executionArn, Integer maxItems, String marker,
+ boolean reverseOrder) {
+ ArnParts arn = parseArn(executionArn);
+ List<Cursored<DurableHistoryEvent>> events = new ArrayList<>();
+ synchronized (lockFor(arn)) {
+ for (DurableHistoryEvent event : load(arn).getHistory()) {
+ events.add(new Cursored<>(orderedCursor(event.getEventId(), !reverseOrder), event));
+ }
+ }
+ return paginate(events, pageSize(maxItems), marker,
+ reverseOrder ? "durable-history-desc" : "durable-history-asc");
+ }
+
+ /** A fixed-width cursor that sorts ascending, or descending when the page runs newest first. */
+ private static String orderedCursor(long value, boolean ascending) {
+ return String.format("%019d", ascending ? value : Long.MAX_VALUE - value);
+ }
+
+ /** A retry of the last checkpoint with the same {@code clientToken} gets the same answer again. */
+ public CheckpointResult checkpoint(String executionArn, String checkpointToken, String clientToken,
+ List<DurableOperationUpdate> updates) {
+ ArnParts arn = parseArn(executionArn);
+ List<Runnable> effects = new ArrayList<>();
+ CheckpointResult result;
+ synchronized (lockFor(arn)) {
+ CheckpointResult replayed = replayLastCheckpoint(arn, checkpointToken, clientToken);
+ if (replayed != null) {
+ return replayed;
+ }
+ DurableExecution execution = loadWithCurrentToken(arn, checkpointToken);
+ long now = clock.millis();
+ DurableCheckpointApplier.Outcome outcome = DurableCheckpointApplier.apply(execution, updates, now);
+ if (!outcome.closed()) {
+ DurableCheckpointApplier.fireDueTimers(execution, now);
+ }
+ List<DurableOperation> changed = unseenOperations(execution);
+ String nextToken = null;
+ if (outcome.closed()) {
+ closeBookkeeping(execution, now, effects);
+ } else {
+ execution.setCheckpointSequence(execution.getCheckpointSequence() + 1);
+ nextToken = DurableTokens.checkpointToken(execution.getExecutionArn(),
+ execution.getCurrentInvocationId(), execution.getCheckpointSequence());
+ }
+ // AWS does not replay the checkpoint that closed the execution.
+ execution.setLastCheckpoint(clientToken == null || nextToken == null ? null
+ : new DurableCheckpointReplay(checkpointToken, clientToken, nextToken, changed));
+ execution.setSeenSequence(execution.getChangeSequence());
+ save(execution);
+ result = new CheckpointResult(nextToken, changed);
+ }
+ runEffects(effects);
+ return result;
+ }
+
+ public StatePage getState(String executionArn, String checkpointToken, String marker, Integer maxItems) {
+ ArnParts arn = parseArn(executionArn);
+ List<DurableOperation> operations;
+ synchronized (lockFor(arn)) {
+ DurableExecution execution = loadWithCurrentToken(arn, checkpointToken);
+ operations = new ArrayList<>(execution.getOperations().values());
+ }
+ List<Cursored<DurableOperation>> indexed = new ArrayList<>(operations.size());
+ for (int i = 0; i < operations.size(); i++) {
+ indexed.add(new Cursored<>(String.format("%010d", i), operations.get(i)));
+ }
+ PaginatedResult<DurableOperation> page = paginate(indexed, pageSize(maxItems), marker, "durable-state");
+ return new StatePage(page.items(), page.nextToken() != null ? page.nextToken() : "");
+ }
+
+ /** Stopping a closed execution changes nothing and reports the time it closed. */
+ public DurableExecution stop(String executionArn, DurableErrorObject error) {
+ ArnParts arn = parseArn(executionArn);
+ List<Runnable> effects = new ArrayList<>();
+ DurableExecution execution;
+ synchronized (lockFor(arn)) {
+ execution = load(arn);
+ if (!execution.isClosed()) {
+ // Without a body AWS records an empty error object, not a message.
+ DurableErrorObject stopError = error != null ? error : new DurableErrorObject();
+ close(execution, DurableExecutionStatus.STOPPED, null, stopError, clock.millis(), effects);
+ save(execution);
+ }
+ }
+ runEffects(effects);
+ return execution;
+ }
+
+ /** One tick of {@link DurableExecutionSweeper}. It fires due timers, times out and expires executions. */
+ public void sweep() {
+ long now = clock.millis();
+ for (AccountEntry<DurableExecution> entry : store.scanAllAccountEntries(key -> true)) {
+ List<Runnable> effects = new ArrayList<>();
+ synchronized (lockFor(entry.accountId(), entry.key())) {
+ DurableExecution execution = store.getForAccount(entry.accountId(), entry.key()).orElse(null);
+ if (execution == null) {
+ continue;
+ }
+ if (execution.isClosed()) {
+ if (execution.getRetentionDeadline() != null && execution.getRetentionDeadline() <= now) {
+ store.deleteForAccount(entry.accountId(), entry.key());
+ }
+ continue;
+ }
+ boolean changed = false;
+ if (execution.getExecutionDeadline() != null && execution.getExecutionDeadline() <= now) {
+ DurableErrorObject error = DurableErrorObject.of("Execution timed out after "
+ + execution.getExecutionTimeoutSeconds() + " seconds.", null);
+ close(execution, DurableExecutionStatus.TIMED_OUT, null, error, now, effects);
+ changed = true;
+ } else {
+ if (DurableCheckpointApplier.fireDueTimers(execution, now)) {
+ trigger(execution, effects);
+ changed = true;
+ }
+ if (execution.getCurrentInvocationId() == null && execution.getNextInvocationAttemptAt() != null
+ && execution.getNextInvocationAttemptAt() <= now) {
+ execution.setNextInvocationAttemptAt(null);
+ trigger(execution, effects);
+ changed = true;
+ }
+ }
+ if (changed) {
+ save(execution);
+ }
+ }
+ runEffects(effects);
+ }
+ }
+
+ /**
+ * Re-invokes every RUNNING execution after a restart. The invocation that was in flight is lost
+ * with the process, and the SDK replays from the checkpoints, so starting over is safe.
+ */
+ public void recoverAfterRestart() {
+ int recovered = 0;
+ for (AccountEntry<DurableExecution> entry : store.scanAllAccountEntries(key -> true)) {
+ List<Runnable> effects = new ArrayList<>();
+ synchronized (lockFor(entry.accountId(), entry.key())) {
+ DurableExecution execution = store.getForAccount(entry.accountId(), entry.key()).orElse(null);
+ if (execution == null || execution.isClosed()) {
+ continue;
+ }
+ execution.setCurrentInvocationId(null);
+ execution.setReinvokeRequested(false);
+ // A crash retry keeps its backoff, and the sweeper relaunches it when it is due.
+ if (execution.getNextInvocationAttemptAt() == null) {
+ trigger(execution, effects);
+ }
+ save(execution);
+ recovered++;
+ }
+ runEffects(effects);
+ }
+ if (recovered > 0) {
+ LOG.infov("Resumed {0} durable execution(s) after restart", recovered);
+ }
+ }
+
+ @Override
+ public void beforeReset() {
+ generation.incrementAndGet();
+ }
+
+ @Override
+ public void clear() {
+ generation.incrementAndGet();
+ startLocks.clear();
+ for (CompletableFuture<DurableExecution> waiter : completions.values()) {
+ waiter.completeExceptionally(new AwsException("ResourceNotFoundException", NOT_FOUND, 404));
+ }
+ completions.clear();
+ }
+
+ @PreDestroy
+ void shutdown() {
+ generation.incrementAndGet();
+ }
+
+ // ──────────────────────────── invocation lane ────────────────────────────
+
+ private void trigger(DurableExecution execution, List<Runnable> effects) {
+ if (execution.getCurrentInvocationId() != null) {
+ execution.setReinvokeRequested(true);
+ return;
+ }
+ String invocationId = UUID.randomUUID().toString();
+ execution.setCurrentInvocationId(invocationId);
+ execution.setNextInvocationAttemptAt(null);
+ String accountId = execution.getAccountId();
+ String key = storeKey(execution);
+ long launchGeneration = generation.get();
+ effects.add(() -> launchExecutor.execute(() -> runInvocation(accountId, key, invocationId, launchGeneration)));
+ }
+
+ private void runInvocation(String accountId, String key, String invocationId, long launchGeneration) {
+ ResolvedDurableTarget target;
+ byte[] payload;
+ long startedAt;
+ synchronized (lockFor(accountId, key)) {
+ DurableExecution execution = store.getForAccount(accountId, key).orElse(null);
+ if (execution == null || execution.isClosed() || launchGeneration != generation.get()
+ || !invocationId.equals(execution.getCurrentInvocationId())) {
+ return;
+ }
+ startedAt = clock.millis();
+ String token = DurableTokens.checkpointToken(execution.getExecutionArn(), invocationId,
+ execution.getCheckpointSequence());
+ List<String> updatedOperationIds = unseenOperations(execution).stream().map(DurableOperation::getId).toList();
+ try {
+ target = invoker.resolve(accountId, execution.getRegion(), execution.getFunctionName(),
+ execution.getVersion());
+ payload = objectMapper.writeValueAsBytes(
+ DurableWire.invocationEvent(execution, token, updatedOperationIds));
+ } catch (AwsException | IOException e) {
+ target = null;
+ payload = null;
+ LOG.warnv("Durable execution {0} cannot be invoked: {1}", execution.getExecutionArn(), e.getMessage());
+ }
+ // The event carries every operation, so the handler has now seen everything up to here.
+ execution.setSeenSequence(execution.getChangeSequence());
+ }
+ DurableInvocationResult result;
+ if (target == null) {
+ result = new DurableInvocationResult(null, null, "ResourceNotFoundException");
+ } else {
+ try {
+ result = invoker.invoke(target, payload);
+ } catch (AwsException e) {
+ result = new DurableInvocationResult(null, DurableWire.functionErrorPayload(
+ DurableErrorObject.of(e.getMessage(), e.getErrorCode())),
+ e.getErrorCode());
+ } catch (RuntimeException e) {
+ LOG.warnv("Durable invocation failed: {0}", e.getMessage());
+ result = new DurableInvocationResult(null, DurableWire.functionErrorPayload(
+ DurableErrorObject.of(e.getMessage(), "InvocationError")),
+ "InvocationError");
+ }
+ }
+ List<Runnable> effects = new ArrayList<>();
+ synchronized (lockFor(accountId, key)) {
+ DurableExecution execution = store.getForAccount(accountId, key).orElse(null);
+ if (execution == null || launchGeneration != generation.get()
+ || !invocationId.equals(execution.getCurrentInvocationId())) {
+ return;
+ }
+ long endedAt = clock.millis();
+ String requestId = result.requestId() != null ? result.requestId() : UUID.randomUUID().toString();
+ execution.setCurrentInvocationId(null);
+ if (execution.isClosed()) {
+ DurableHistory.invocationCompleted(execution, startedAt, endedAt, requestId, null);
+ } else if (result.functionError() != null) {
+ // The sweeper relaunches the invocation once its backoff has passed.
+ recordInvocationFailure(execution, result, startedAt, endedAt, requestId, effects);
+ } else {
+ recordHandlerResponse(execution, result.payload(), startedAt, endedAt, requestId, effects);
+ if (!execution.isClosed() && (execution.isReinvokeRequested() || execution.hasUnseenChanges())) {
+ execution.setReinvokeRequested(false);
+ trigger(execution, effects);
+ }
+ }
+ save(execution);
+ }
+ runEffects(effects);
+ }
+
+ private void recordInvocationFailure(DurableExecution execution, DurableInvocationResult result, long startedAt,
+ long endedAt, String requestId, List<Runnable> effects) {
+ DurableErrorObject error = errorFromLambdaPayload(result.payload(), result.functionError());
+ int failures = execution.getConsecutiveInvocationFailures() + 1;
+ execution.setConsecutiveInvocationFailures(failures);
+ DurableHistory.invocationCompleted(execution, startedAt, endedAt, requestId, error);
+ if (failures >= INVOCATION_RETRY_MAX_ATTEMPTS) {
+ close(execution, DurableExecutionStatus.FAILED, null, error, endedAt, effects);
+ } else {
+ execution.setNextInvocationAttemptAt(endedAt + (invocationRetryDelayMillis << (failures - 1)));
+ // The retry sees every change, so a wake-up that arrived during this invocation is spent.
+ execution.setReinvokeRequested(false);
+ }
+ }
+
+ private void recordHandlerResponse(DurableExecution execution, byte[] payload, long startedAt, long endedAt,
+ String requestId, List<Runnable> effects) {
+ execution.setConsecutiveInvocationFailures(0);
+ DurableWire.HandlerResponse response = parseHandlerResponse(payload);
+ String protocolError = protocolError(execution, response);
+ if (protocolError != null) {
+ DurableErrorObject error = DurableErrorObject.of(protocolError, "InvalidParameterValueException");
+ DurableHistory.invocationCompleted(execution, startedAt, endedAt, requestId, error);
+ close(execution, DurableExecutionStatus.FAILED, null, error, endedAt, effects);
+ return;
+ }
+ boolean failed = "FAILED".equals(response.status());
+ DurableHistory.invocationCompleted(execution, startedAt, endedAt, requestId, failed ? response.error() : null);
+ switch (response.status()) {
+ case "SUCCEEDED" -> close(execution, DurableExecutionStatus.SUCCEEDED, response.result(), null, endedAt,
+ effects);
+ case "FAILED" -> close(execution, DurableExecutionStatus.FAILED, null, response.error(), endedAt, effects);
+ default -> {
+ }
+ }
+ }
+
+ /** Null when the handler response is acceptable, else the error message AWS gives. */
+ private static String protocolError(DurableExecution execution, DurableWire.HandlerResponse response) {
+ if (response == null) {
+ return "Invalid Status in invocation output.";
+ }
+ return switch (response.status()) {
+ case "SUCCEEDED" -> response.error() != null ? "Cannot provide an Error for SUCCEEDED status."
+ : DurableCheckpointApplier.utf8Length(response.result()) > execution.getMaxResultBytes()
+ ? "Execution output payload size must be less than or equal to "
+ + execution.getMaxResultBytes() + " bytes." : null;
+ case "FAILED" -> response.result() != null ? "Cannot provide a Result for FAILED status." : null;
+ case "PENDING" -> execution.hasPendingOperations() || execution.isReinvokeRequested()
+ || execution.hasUnseenChanges() ? null
+ : "Cannot return PENDING status with no pending operations.";
+ default -> "Invalid Status in invocation output.";
+ };
+ }
+
+ private DurableWire.HandlerResponse parseHandlerResponse(byte[] payload) {
+ if (payload == null || payload.length == 0) {
+ return null;
+ }
+ try {
+ JsonNode body = objectMapper.readTree(payload);
+ return DurableWire.parseHandlerResponse(body);
+ } catch (IOException e) {
+ LOG.debugv("Durable handler returned a body that is not JSON: {0}", e.getMessage());
+ return null;
+ }
+ }
+
+ // ──────────────────────────── state changes ────────────────────────────
+
+ private DurableExecution newExecution(StartRequest request, ResolvedDurableTarget target, String name, long now) {
+ DurableExecution execution = new DurableExecution();
+ String executionId = UUID.randomUUID().toString();
+ execution.setExecutionId(executionId);
+ execution.setName(name);
+ execution.setAccountId(request.accountId());
+ execution.setRegion(request.region());
+ execution.setFunctionName(target.functionName());
+ execution.setFunctionArn(target.functionArn());
+ execution.setVersion(target.version());
+ execution.setExecutionArn(target.functionArn() + "/durable-execution/" + name + "/" + executionId);
+ execution.setStartTimestamp(now);
+ execution.setInputPayload(request.inputPayload());
+ execution.setExecutionTimeoutSeconds(target.executionTimeoutSeconds());
+ execution.setRetentionPeriodInDays(target.retentionPeriodInDays());
+ execution.setExecutionDeadline(now + target.executionTimeoutSeconds() * 1000L);
+ execution.setMaxResultBytes(request.synchronous() ? SYNC_PAYLOAD_LIMIT : ASYNC_PAYLOAD_LIMIT);
+
+ DurableOperation root = new DurableOperation();
+ root.setId(executionId);
+ root.setName(name);
+ root.setType(DurableOperationType.EXECUTION);
+ root.setStatus(DurableOperationStatus.STARTED);
+ root.setStartTimestamp(now);
+ root.setInputPayload(request.inputPayload());
+ root.setChangeSequence(execution.nextChangeSequence());
+ execution.getOperations().put(executionId, root);
+ DurableHistory.executionStarted(execution, now);
+ return execution;
+ }
+
+ private void close(DurableExecution execution, DurableExecutionStatus status, String result,
+ DurableErrorObject error, long now, List<Runnable> effects) {
+ execution.setStatus(status);
+ execution.setResult(result);
+ execution.setError(error);
+ execution.setEndTimestamp(now);
+ DurableOperation root = execution.getOperations().get(execution.getExecutionId());
+ if (root != null) {
+ root.setStatus(switch (status) {
+ case SUCCEEDED -> DurableOperationStatus.SUCCEEDED;
+ case TIMED_OUT -> DurableOperationStatus.TIMED_OUT;
+ case STOPPED -> DurableOperationStatus.STOPPED;
+ default -> DurableOperationStatus.FAILED;
+ });
+ root.setEndTimestamp(now);
+ root.setChangeSequence(execution.nextChangeSequence());
+ }
+ switch (status) {
+ case SUCCEEDED -> DurableHistory.executionSucceeded(execution, now);
+ case TIMED_OUT -> DurableHistory.executionTimedOut(execution, now);
+ case STOPPED -> DurableHistory.executionEnded(execution, "ExecutionStopped", now);
+ default -> DurableHistory.executionEnded(execution, "ExecutionFailed", now);
+ }
+ closeBookkeeping(execution, now, effects);
+ }
+
+ /** What every close shares, whether the applier or the service decided it. */
+ private void closeBookkeeping(DurableExecution execution, long now, List<Runnable> effects) {
+ execution.setRetentionDeadline(now + execution.getRetentionPeriodInDays() * 86_400_000L);
+ execution.setNextInvocationAttemptAt(null);
+ execution.setReinvokeRequested(false);
+ String lockKey = lockKey(execution.getAccountId(), storeKey(execution));
+ effects.add(() -> {
+ CompletableFuture<DurableExecution> waiter = completions.remove(lockKey);
+ if (waiter != null) {
+ waiter.complete(execution);
+ }
+ });
+ }
+
+ /** Null unless this is the last checkpoint again, with the same ClientToken, and its token is still current. */
+ private CheckpointResult replayLastCheckpoint(ArnParts arn, String checkpointToken, String clientToken) {
+ if (clientToken == null) {
+ return null;
+ }
+ DurableExecution execution = store.getForAccount(arn.accountId(), arn.storeKey())
+ .filter(candidate -> candidate.getExecutionArn().equals(arn.arn()))
+ .orElse(null);
+ DurableCheckpointReplay last = execution == null ? null : execution.getLastCheckpoint();
+ if (last == null || execution.isClosed() || execution.getCurrentInvocationId() == null
+ || !clientToken.equals(last.getClientToken()) || !last.getRequestToken().equals(checkpointToken)
+ || !last.getResponseToken().equals(DurableTokens.checkpointToken(execution.getExecutionArn(),
+ execution.getCurrentInvocationId(), execution.getCheckpointSequence()))) {
+ return null;
+ }
+ return new CheckpointResult(last.getResponseToken(), last.getOperations());
+ }
+
+ /** Checkpoint and state calls answer an unknown execution as a bad token, not as ResourceNotFoundException. */
+ private DurableExecution loadWithCurrentToken(ArnParts arn, String checkpointToken) {
+ Optional<DurableTokens.CheckpointToken> token = DurableTokens.parseCheckpointToken(checkpointToken);
+ if (token.isPresent() && !token.get().executionArn().equals(arn.arn())) {
+ throw new AwsException("InvalidParameterValueException", TOKEN_FOR_OTHER_EXECUTION, 400);
+ }
+ DurableExecution execution = store.getForAccount(arn.accountId(), arn.storeKey())
+ .filter(candidate -> candidate.getExecutionArn().equals(arn.arn()))
+ .orElse(null);
+ if (token.isEmpty() || execution == null || execution.isClosed() || execution.getCurrentInvocationId() == null
+ || !token.get().invocationId().equals(execution.getCurrentInvocationId())
+ || token.get().sequence() != execution.getCheckpointSequence()) {
+ throw new AwsException("InvalidParameterValueException", INVALID_TOKEN, 400);
+ }
+ return execution;
+ }
+
+ private void save(DurableExecution execution) {
+ store.putForAccount(execution.getAccountId(), storeKey(execution), execution);
+ }
+
+ private DurableExecution load(ArnParts arn) {
+ return store.getForAccount(arn.accountId(), arn.storeKey())
+ .filter(execution -> execution.getExecutionArn().equals(arn.arn()))
+ .orElseThrow(() -> new AwsException("ResourceNotFoundException", NOT_FOUND, 404));
+ }
+
+ private static List<DurableOperation> unseenOperations(DurableExecution execution) {
+ return execution.getOperations().values().stream()
+ .filter(operation -> operation.getChangeSequence() > execution.getSeenSequence())
+ .sorted(Comparator.comparingLong(DurableOperation::getChangeSequence))
+ .toList();
+ }
+
+ private static void runEffects(List<Runnable> effects) {
+ for (Runnable effect : effects) {
+ effect.run();
+ }
+ }
+
+ // ──────────────────────────── helpers ────────────────────────────
+
+ record ArnParts(String arn, String accountId, String region, String functionName, String version, String name,
+ String executionId) {
+ String storeKey() {
+ return region + "/" + executionId;
+ }
+
+ String lockKey() {
+ return DurableExecutionService.lockKey(accountId, storeKey());
+ }
+ }
+
+ static ArnParts parseArn(String executionArn) {
+ Matcher matcher = executionArn == null ? null : LambdaArnUtils.DURABLE_EXECUTION_ARN.matcher(executionArn);
+ if (matcher == null || !matcher.matches()) {
+ throw new AwsException("ValidationException",
+ "1 validation error detected: Value '" + executionArn + "' at 'durableExecutionArn' failed to "
+ + "satisfy constraint: Member must satisfy regular expression pattern: "
+ + LambdaArnUtils.DURABLE_EXECUTION_ARN.pattern(), 400);
+ }
+ return new ArnParts(executionArn, matcher.group(3), matcher.group(2), matcher.group(4), matcher.group(5),
+ matcher.group(6), matcher.group(7));
+ }
+
+ private static String storeKey(DurableExecution execution) {
+ return execution.getRegion() + "/" + execution.getExecutionId();
+ }
+
+ private static String lockKey(String accountId, String storeKey) {
+ return accountId + "/" + storeKey;
+ }
+
+ private Object lockFor(ArnParts arn) {
+ return lockFor(arn.accountId(), arn.storeKey());
+ }
+
+ private Object lockFor(String accountId, String storeKey) {
+ return executionLocks[Math.floorMod(lockKey(accountId, storeKey).hashCode(), executionLocks.length)];
+ }
+
+ private static Object[] newLocks(int count) {
+ Object[] locks = new Object[count];
+ for (int i = 0; i < count; i++) {
+ locks[i] = new Object();
+ }
+ return locks;
+ }
+
+ private Object startLock(String accountId, String region) {
+ return startLocks.computeIfAbsent(accountId + ":" + region, ignored -> new Object());
+ }
+
+ private static int pageSize(Integer maxItems) {
+ String violation = maxItemsViolation(maxItems);
+ if (violation != null) {
+ throw validationError(List.of(violation));
+ }
+ return maxItems == null || maxItems == 0 ? DEFAULT_PAGE_SIZE : maxItems;
+ }
+
+ /** Null when MaxItems is in range. Zero asks for the default page. */
+ static String maxItemsViolation(Integer maxItems) {
+ if (maxItems == null) {
+ return null;
+ }
+ String bound = maxItems < 0 ? "greater than or equal to 0"
+ : MAX_PAGE_SIZE < maxItems ? "less than or equal to " + MAX_PAGE_SIZE : null;
+ return bound == null ? null
+ : "Value '" + maxItems + "' at 'maxItems' failed to satisfy constraint: Member must have value " + bound;
+ }
+
+ /** AWS reports every invalid member of a request in one ValidationException. */
+ static AwsException validationError(List<String> violations) {
+ String count = violations.size() == 1 ? "1 validation error detected: "
+ : violations.size() + " validation errors detected: ";
+ return new AwsException("ValidationException", count + String.join("; ", violations), 400);
+ }
+
+ private static AwsException invalidMarker() {
+ return new AwsException("InvalidParameterValueException", "Invalid Marker", 400);
+ }
+
+ private DurableErrorObject errorFromLambdaPayload(byte[] payload, String functionError) {
+ if (payload != null && payload.length > 0) {
+ try {
+ JsonNode body = objectMapper.readTree(payload);
+ if (body.isObject() && body.hasNonNull("errorMessage")) {
+ DurableErrorObject error = DurableErrorObject.of(body.get("errorMessage").asText(),
+ body.path("errorType").isTextual() ? body.get("errorType").asText() : functionError);
+ JsonNode stackTrace = body.get("stackTrace");
+ if (stackTrace != null && stackTrace.isArray()) {
+ List<String> lines = new ArrayList<>();
+ stackTrace.forEach(line -> lines.add(line.isTextual() ? line.asText() : line.toString()));
+ error.setStackTrace(lines);
+ }
+ return error;
+ }
+ return DurableErrorObject.of(body.toString(), functionError);
+ } catch (IOException expected) {
+ // A runtime may report a plain-text error. Its text becomes the message.
+ return DurableErrorObject.of(new String(payload, StandardCharsets.UTF_8), functionError);
+ }
+ }
+ return DurableErrorObject.of("Function invocation failed", functionError);
+ }
+
+ private static <T> PaginatedResult<T> paginate(List<Cursored<T>> items, int pageSize, String marker,
+ String namespace) {
+ PaginatedResult<Cursored<T>> page = Pagination.paginate(items, Cursored::cursor, pageSize, marker, namespace,
+ ignored -> invalidMarker());
+ return new PaginatedResult<>(page.items().stream().map(Cursored::item).toList(), page.nextToken());
+ }
+
+ private record Cursored<T>(String cursor, T item) {
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionSweeper.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionSweeper.java
new file mode 100644
index 0000000000..df48fa389c
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionSweeper.java
@@ -0,0 +1,75 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.quarkus.runtime.ShutdownEvent;
+import io.quarkus.runtime.StartupEvent;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.event.Observes;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Drives durable execution timers from one background thread. Waits, step retries, execution
+ * timeouts and retention expiry are persisted deadlines. A fixed-interval sweep fires them without
+ * one timer per operation, so a restart has nothing to reconcile.
+ */
+@ApplicationScoped
+public class DurableExecutionSweeper {
+
+ private static final Logger LOG = Logger.getLogger(DurableExecutionSweeper.class);
+
+ static final long DEFAULT_TICK_INTERVAL_SECONDS = 1L;
+
+ private final DurableExecutionService service;
+ private final boolean enabled;
+ private final long tickIntervalSeconds;
+ private final ScheduledExecutorService executor;
+
+ @Inject
+ public DurableExecutionSweeper(DurableExecutionService service, EmulatorConfig config) {
+ this.service = service;
+ this.enabled = config.services().lambda().enabled() && config.services().lambda().durableSweepEnabled();
+ this.tickIntervalSeconds = resolveInterval(config.services().lambda().durableSweepIntervalSeconds());
+ this.executor = Executors.newSingleThreadScheduledExecutor(runnable -> {
+ Thread thread = new Thread(runnable, "lambda-durable-sweeper");
+ thread.setDaemon(true);
+ return thread;
+ });
+ }
+
+ /** A non-positive interval would throw from the startup observer and keep the emulator from becoming ready. */
+ private static long resolveInterval(long configured) {
+ if (configured > 0) {
+ return configured;
+ }
+ LOG.warnv("Ignoring Lambda durable-sweep interval {0}s: must be a positive number of seconds, "
+ + "falling back to {1}s", configured, DEFAULT_TICK_INTERVAL_SECONDS);
+ return DEFAULT_TICK_INTERVAL_SECONDS;
+ }
+
+ void onStart(@Observes StartupEvent event) {
+ if (!enabled) {
+ LOG.debug("Lambda durable execution sweeper disabled");
+ return;
+ }
+ executor.scheduleWithFixedDelay(this::tick, tickIntervalSeconds, tickIntervalSeconds, TimeUnit.SECONDS);
+ LOG.debugv("Lambda durable execution sweeper started, interval {0}s", tickIntervalSeconds);
+ }
+
+ void onStop(@Observes ShutdownEvent event) {
+ executor.shutdownNow();
+ }
+
+ private void tick() {
+ try {
+ service.sweep();
+ } catch (RuntimeException e) {
+ // A sweep failure must not kill the scheduled task. The next tick retries.
+ LOG.error("Lambda durable execution sweep failed", e);
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableFunctionInvoker.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableFunctionInvoker.java
new file mode 100644
index 0000000000..c0e1645de5
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableFunctionInvoker.java
@@ -0,0 +1,27 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+/**
+ * The slice of Lambda invocation the durable engine needs, so {@link DurableExecutionService}
+ * depends on a two-method seam rather than on the container stack. Tests supply a scripted
+ * stand-in for the function. At runtime {@link LambdaDurableFunctionInvoker} runs the real one.
+ */
+public interface DurableFunctionInvoker {
+
+ /** The function a durable execution runs on, or an AwsException when the name does not resolve. */
+ ResolvedDurableTarget resolve(String accountId, String region, String functionName, String qualifier);
+
+ DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload);
+
+ record ResolvedDurableTarget(String accountId,
+ String region,
+ String functionName,
+ String functionArn,
+ String version,
+ boolean durable,
+ int executionTimeoutSeconds,
+ int retentionPeriodInDays) {
+ }
+
+ record DurableInvocationResult(String requestId, byte[] payload, String functionError) {
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableHistory.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableHistory.java
new file mode 100644
index 0000000000..ce3b46e529
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableHistory.java
@@ -0,0 +1,113 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableErrorObject;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecution;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableHistoryEvent;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperation;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Builds GetDurableExecutionHistory events in their wire form. Event type values are PascalCase on
+ * the wire (ExecutionStarted, StepSucceeded), unlike the SCREAMING_SNAKE operation enums.
+ */
+final class DurableHistory {
+
+ static final String PAYLOAD = "Payload";
+ static final String TRUNCATED = "Truncated";
+
+ private DurableHistory() {
+ }
+
+ static void executionStarted(DurableExecution execution, long now) {
+ Map<String, Object> details = new LinkedHashMap<>();
+ details.put("Input", payloadWrapper(execution.getInputPayload()));
+ details.put("ExecutionTimeout", execution.getExecutionTimeoutSeconds());
+ executionEvent(execution, "ExecutionStarted", now, details);
+ }
+
+ static void executionSucceeded(DurableExecution execution, long now) {
+ Map<String, Object> details = new LinkedHashMap<>();
+ details.put("Result", payloadWrapper(execution.getResult()));
+ executionEvent(execution, "ExecutionSucceeded", now, details);
+ }
+
+ /** ExecutionFailed or ExecutionStopped, both of which carry the execution's error. */
+ static void executionEnded(DurableExecution execution, String eventType, long now) {
+ Map<String, Object> details = new LinkedHashMap<>();
+ details.put("Error", errorWrapper(execution.getError()));
+ executionEvent(execution, eventType, now, details);
+ }
+
+ /** AWS leaves ExecutionTimedOutDetails empty. The error is only on the execution itself. */
+ static void executionTimedOut(DurableExecution execution, long now) {
+ executionEvent(execution, "ExecutionTimedOut", now, new LinkedHashMap<>());
+ }
+
+ static void invocationCompleted(DurableExecution execution, long startedAt, long endedAt, String requestId,
+ DurableErrorObject error) {
+ Map<String, Object> details = new LinkedHashMap<>();
+ details.put("StartTimestamp", startedAt);
+ details.put("EndTimestamp", endedAt);
+ details.put("RequestId", requestId);
+ if (error != null) {
+ details.put("Error", errorWrapper(error));
+ }
+ DurableHistoryEvent event = execution.appendEvent("InvocationCompleted", endedAt);
+ event.setDetails(Map.of("InvocationCompletedDetails", details));
+ }
+
+ static void operationEvent(DurableExecution execution, DurableOperation operation, String eventType, long now,
+ Map<String, Object> details) {
+ DurableHistoryEvent event = execution.appendEvent(eventType, now);
+ event.setId(operation.getId());
+ event.setName(operation.getName());
+ event.setSubType(operation.getSubType());
+ event.setParentId(operation.getParentId());
+ event.setDetails(Map.of(eventType + "Details", details));
+ }
+
+ static Map<String, Object> payloadWrapper(String payload) {
+ Map<String, Object> wrapper = new LinkedHashMap<>();
+ if (payload != null) {
+ wrapper.put(PAYLOAD, payload);
+ }
+ wrapper.put(TRUNCATED, false);
+ return wrapper;
+ }
+
+ static Map<String, Object> errorWrapper(DurableErrorObject error) {
+ Map<String, Object> wrapper = new LinkedHashMap<>();
+ if (error != null) {
+ wrapper.put(PAYLOAD, errorMap(error));
+ }
+ wrapper.put(TRUNCATED, false);
+ return wrapper;
+ }
+
+ private static Map<String, Object> errorMap(DurableErrorObject error) {
+ Map<String, Object> map = new LinkedHashMap<>();
+ if (error.getErrorMessage() != null) {
+ map.put("ErrorMessage", error.getErrorMessage());
+ }
+ if (error.getErrorType() != null) {
+ map.put("ErrorType", error.getErrorType());
+ }
+ if (error.getErrorData() != null) {
+ map.put("ErrorData", error.getErrorData());
+ }
+ if (error.getStackTrace() != null) {
+ map.put("StackTrace", error.getStackTrace());
+ }
+ return map;
+ }
+
+ private static void executionEvent(DurableExecution execution, String eventType, long now,
+ Map<String, Object> details) {
+ DurableHistoryEvent event = execution.appendEvent(eventType, now);
+ event.setId(execution.getExecutionId());
+ event.setName(execution.getName());
+ event.setDetails(Map.of(eventType + "Details", details));
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableTokens.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableTokens.java
new file mode 100644
index 0000000000..df4fda7a3e
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableTokens.java
@@ -0,0 +1,59 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Optional;
+
+/**
+ * The opaque CheckpointToken of the durable execution protocol, standard base64 as AWS documents it.
+ */
+final class DurableTokens {
+
+ private static final String SEPARATOR = "|";
+
+ private DurableTokens() {
+ }
+
+ record CheckpointToken(String executionArn, String invocationId, long sequence) {
+ }
+
+ static String checkpointToken(String executionArn, String invocationId, long sequence) {
+ return encode(executionArn + SEPARATOR + invocationId + SEPARATOR + sequence);
+ }
+
+ static Optional<CheckpointToken> parseCheckpointToken(String token) {
+ String decoded = decode(token);
+ if (decoded == null) {
+ return Optional.empty();
+ }
+ int last = decoded.lastIndexOf(SEPARATOR);
+ int middle = last < 0 ? -1 : decoded.lastIndexOf(SEPARATOR, last - 1);
+ if (middle < 0) {
+ return Optional.empty();
+ }
+ try {
+ long sequence = Long.parseLong(decoded.substring(last + 1));
+ return Optional.of(new CheckpointToken(decoded.substring(0, middle),
+ decoded.substring(middle + 1, last), sequence));
+ } catch (NumberFormatException expected) {
+ // A token Floci did not mint. The caller answers "Invalid checkpoint token" as AWS does.
+ return Optional.empty();
+ }
+ }
+
+ private static String encode(String value) {
+ return Base64.getEncoder().encodeToString(value.getBytes(StandardCharsets.UTF_8));
+ }
+
+ private static String decode(String token) {
+ if (token == null || token.isEmpty()) {
+ return null;
+ }
+ try {
+ return new String(Base64.getDecoder().decode(token), StandardCharsets.UTF_8);
+ } catch (IllegalArgumentException expected) {
+ // Not base64, so not a token Floci minted. The caller rejects it.
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java
new file mode 100644
index 0000000000..e49b9218f1
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java
@@ -0,0 +1,406 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableErrorObject;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecution;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableHistoryEvent;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperation;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationAction;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationType;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationUpdate;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Maps the stored durable model to and from the PascalCase wire shapes. REST responses carry
+ * epoch-second timestamps. The invocation event handed to the function carries epoch millis,
+ * which is what the Durable Execution SDK parses.
+ */
+public final class DurableWire {
+
+ private static final JsonNodeFactory NODES = JsonNodeFactory.instance;
+ private static final long MAX_DELAY_SECONDS = 31_622_400L;
+
+ private DurableWire() {
+ }
+
+ public record HandlerResponse(String status, String result, DurableErrorObject error) {
+ }
+
+ /** Lambda's function-error body uses lower camel case, unlike the ErrorObject. */
+ public static byte[] functionErrorPayload(DurableErrorObject error) {
+ ObjectNode body = NODES.objectNode();
+ if (error != null) {
+ putIfPresent(body, "errorMessage", error.getErrorMessage());
+ putIfPresent(body, "errorType", error.getErrorType());
+ if (error.getStackTrace() != null) {
+ ArrayNode stackTrace = body.putArray("stackTrace");
+ error.getStackTrace().forEach(stackTrace::add);
+ }
+ }
+ return body.toString().getBytes(StandardCharsets.UTF_8);
+ }
+
+ /** {@code updatedOperationIds} names the operations that changed since the previous invocation saw them. */
+ public static ObjectNode invocationEvent(DurableExecution execution, String checkpointToken,
+ List<String> updatedOperationIds) {
+ ObjectNode event = NODES.objectNode();
+ event.put("DurableExecutionArn", execution.getExecutionArn());
+ event.put("CheckpointToken", checkpointToken);
+ ObjectNode state = event.putObject("InitialExecutionState");
+ ArrayNode operations = state.putArray("Operations");
+ for (DurableOperation operation : execution.getOperations().values()) {
+ operations.add(operation(operation, true));
+ }
+ state.put("NextMarker", "");
+ ArrayNode updated = event.putArray("UpdatedOperationIds");
+ updatedOperationIds.forEach(updated::add);
+ return event;
+ }
+
+ public static ObjectNode operations(Collection<DurableOperation> operations, String nextMarker) {
+ ObjectNode node = NODES.objectNode();
+ ArrayNode array = node.putArray("Operations");
+ for (DurableOperation operation : operations) {
+ array.add(operation(operation, false));
+ }
+ if (nextMarker != null) {
+ node.put("NextMarker", nextMarker);
+ }
+ return node;
+ }
+
+ public static ObjectNode operation(DurableOperation operation, boolean millis) {
+ ObjectNode node = NODES.objectNode();
+ node.put("Id", operation.getId());
+ putIfPresent(node, "ParentId", operation.getParentId());
+ putIfPresent(node, "Name", operation.getName());
+ node.put("Type", operation.getType().name());
+ putIfPresent(node, "SubType", operation.getSubType());
+ putTimestamp(node, "StartTimestamp", operation.getStartTimestamp(), millis);
+ if (operation.getEndTimestamp() != null) {
+ putTimestamp(node, "EndTimestamp", operation.getEndTimestamp(), millis);
+ }
+ node.put("Status", operation.getStatus().name());
+ switch (operation.getType()) {
+ case EXECUTION -> {
+ ObjectNode details = node.putObject("ExecutionDetails");
+ putIfPresent(details, "InputPayload", operation.getInputPayload());
+ }
+ case CONTEXT -> {
+ ObjectNode details = node.putObject("ContextDetails");
+ if (operation.getReplayChildren() != null) {
+ details.put("ReplayChildren", operation.getReplayChildren());
+ }
+ putResultAndError(details, operation);
+ }
+ case STEP -> {
+ ObjectNode details = node.putObject("StepDetails");
+ if (operation.getAttempt() != null) {
+ details.put("Attempt", operation.getAttempt());
+ }
+ if (operation.getNextAttemptTimestamp() != null) {
+ putTimestamp(details, "NextAttemptTimestamp", operation.getNextAttemptTimestamp(), millis);
+ }
+ putResultAndError(details, operation);
+ }
+ case WAIT -> {
+ ObjectNode details = node.putObject("WaitDetails");
+ if (operation.getScheduledEndTimestamp() != null) {
+ putTimestamp(details, "ScheduledEndTimestamp", operation.getScheduledEndTimestamp(), millis);
+ }
+ }
+ default -> {
+ }
+ }
+ return node;
+ }
+
+ public static ObjectNode execution(DurableExecution execution, boolean includeExecutionData) {
+ ObjectNode node = NODES.objectNode();
+ node.put("DurableExecutionArn", execution.getExecutionArn());
+ node.put("DurableExecutionName", execution.getName());
+ node.put("FunctionArn", execution.getFunctionArn());
+ node.put("Version", execution.getVersion());
+ node.put("Status", execution.getStatus().name());
+ putTimestamp(node, "StartTimestamp", execution.getStartTimestamp(), false);
+ if (execution.getEndTimestamp() != null) {
+ putTimestamp(node, "EndTimestamp", execution.getEndTimestamp(), false);
+ }
+ ObjectNode config = node.putObject("DurableConfig");
+ config.put("ExecutionTimeout", execution.getExecutionTimeoutSeconds());
+ config.put("RetentionPeriodInDays", execution.getRetentionPeriodInDays());
+ node.put("ExecutionDataIncluded", includeExecutionData);
+ if (includeExecutionData) {
+ putIfPresent(node, "InputPayload", execution.getInputPayload());
+ putIfPresent(node, "Result", execution.getResult());
+ if (execution.getError() != null) {
+ node.set("Error", error(execution.getError()));
+ }
+ }
+ return node;
+ }
+
+ public static ObjectNode executionSummary(DurableExecution execution) {
+ ObjectNode node = NODES.objectNode();
+ node.put("DurableExecutionArn", execution.getExecutionArn());
+ node.put("DurableExecutionName", execution.getName());
+ node.put("FunctionArn", execution.getFunctionArn());
+ node.put("Status", execution.getStatus().name());
+ putTimestamp(node, "StartTimestamp", execution.getStartTimestamp(), false);
+ if (execution.getEndTimestamp() != null) {
+ putTimestamp(node, "EndTimestamp", execution.getEndTimestamp(), false);
+ }
+ return node;
+ }
+
+ public static ObjectNode historyEvent(DurableHistoryEvent event, boolean includeExecutionData) {
+ ObjectNode node = NODES.objectNode();
+ node.put("EventType", event.getEventType());
+ node.put("EventId", event.getEventId());
+ putIfPresent(node, "Id", event.getId());
+ putIfPresent(node, "Name", event.getName());
+ putIfPresent(node, "SubType", event.getSubType());
+ putIfPresent(node, "ParentId", event.getParentId());
+ putTimestamp(node, "EventTimestamp", event.getEventTimestamp(), false);
+ if (event.getDetails() != null) {
+ for (Map.Entry<String, Object> detailsEntry : event.getDetails().entrySet()) {
+ node.set(detailsEntry.getKey(), detailsNode(detailsEntry.getValue(), includeExecutionData));
+ }
+ }
+ return node;
+ }
+
+ public static ObjectNode error(DurableErrorObject error) {
+ ObjectNode node = NODES.objectNode();
+ putIfPresent(node, "ErrorMessage", error.getErrorMessage());
+ putIfPresent(node, "ErrorType", error.getErrorType());
+ putIfPresent(node, "ErrorData", error.getErrorData());
+ if (error.getStackTrace() != null) {
+ ArrayNode stackTrace = node.putArray("StackTrace");
+ error.getStackTrace().forEach(stackTrace::add);
+ }
+ return node;
+ }
+
+ public static List<DurableOperationUpdate> parseUpdates(Object value) {
+ if (value == null) {
+ return List.of();
+ }
+ if (!(value instanceof List<?> list)) {
+ throw new AwsException("SerializationException", "Updates must be a JSON array or null", 400);
+ }
+ List<DurableOperationUpdate> updates = new ArrayList<>(list.size());
+ for (Object item : list) {
+ if (!(item instanceof Map<?, ?> map)) {
+ throw new AwsException("SerializationException", "Updates members must be JSON objects", 400);
+ }
+ updates.add(parseUpdate(map));
+ }
+ return updates;
+ }
+
+ public static DurableErrorObject parseError(Object value) {
+ if (value == null) {
+ return null;
+ }
+ if (!(value instanceof Map<?, ?> map)) {
+ throw new AwsException("SerializationException", "Error must be a JSON object or null", 400);
+ }
+ DurableErrorObject error = new DurableErrorObject();
+ error.setErrorMessage(string(map, "ErrorMessage"));
+ error.setErrorType(string(map, "ErrorType"));
+ error.setErrorData(string(map, "ErrorData"));
+ Object stackTrace = map.get("StackTrace");
+ if (stackTrace instanceof List<?> lines) {
+ error.setStackTrace(lines.stream().map(String::valueOf).toList());
+ }
+ return error;
+ }
+
+ /** Null when the body is not the {@code {Status, Result, Error}} object the protocol requires. */
+ public static HandlerResponse parseHandlerResponse(JsonNode body) {
+ if (body == null || !body.isObject() || !body.path("Status").isTextual()) {
+ return null;
+ }
+ JsonNode result = body.get("Result");
+ JsonNode errorNode = body.get("Error");
+ DurableErrorObject error = null;
+ if (errorNode != null && errorNode.isObject()) {
+ error = new DurableErrorObject();
+ error.setErrorMessage(text(errorNode, "ErrorMessage"));
+ error.setErrorType(text(errorNode, "ErrorType"));
+ error.setErrorData(text(errorNode, "ErrorData"));
+ JsonNode stackTrace = errorNode.get("StackTrace");
+ if (stackTrace != null && stackTrace.isArray()) {
+ List<String> lines = new ArrayList<>();
+ stackTrace.forEach(line -> lines.add(line.asText()));
+ error.setStackTrace(lines);
+ }
+ }
+ return new HandlerResponse(body.get("Status").asText(),
+ result != null && result.isTextual() ? result.asText() : null, error);
+ }
+
+ private static DurableOperationUpdate parseUpdate(Map<?, ?> map) {
+ DurableOperationType type = enumMember(map, "Type", DurableOperationType.class);
+ DurableOperationAction action = enumMember(map, "Action", DurableOperationAction.class);
+ Map<?, ?> stepOptions = structure(map, "StepOptions");
+ Map<?, ?> waitOptions = structure(map, "WaitOptions");
+ Map<?, ?> contextOptions = structure(map, "ContextOptions");
+ return new DurableOperationUpdate(
+ string(map, "Id"),
+ string(map, "ParentId"),
+ string(map, "Name"),
+ type,
+ string(map, "SubType"),
+ action,
+ string(map, "Payload"),
+ parseError(map.get("Error")),
+ stepOptions == null ? null : integer(stepOptions, "NextAttemptDelaySeconds", "StepOptions"),
+ waitOptions == null ? null : integer(waitOptions, "WaitSeconds", "WaitOptions"),
+ contextOptions == null ? null : bool(contextOptions, "ReplayChildren"));
+ }
+
+ private static <E extends Enum<E>> E enumMember(Map<?, ?> map, String member, Class<E> type) {
+ Object value = map.get(member);
+ if (value == null) {
+ throw new AwsException("InvalidParameterValueException", member + " is required", 400);
+ }
+ try {
+ return Enum.valueOf(type, String.valueOf(value));
+ } catch (IllegalArgumentException e) {
+ throw new AwsException("InvalidParameterValueException", "Unknown operation " + member.toLowerCase() + ".",
+ 400);
+ }
+ }
+
+ private static Map<?, ?> structure(Map<?, ?> map, String member) {
+ Object value = map.get(member);
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof Map<?, ?> structure) {
+ return structure;
+ }
+ throw new AwsException("SerializationException", member + " must be a JSON object or null", 400);
+ }
+
+ private static String string(Map<?, ?> map, String member) {
+ Object value = map.get(member);
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof String s) {
+ return s;
+ }
+ throw new AwsException("SerializationException", member + " must be a string", 400);
+ }
+
+ private static Integer integer(Map<?, ?> map, String member, String structure) {
+ Object value = map.get(member);
+ if (value == null) {
+ return null;
+ }
+ if (!(value instanceof Number number && (number instanceof Byte || number instanceof Short
+ || number instanceof Integer || number instanceof Long))) {
+ throw new AwsException("SerializationException", structure + "." + member + " must be an integer", 400);
+ }
+ long seconds = number.longValue();
+ if (seconds < 1 || MAX_DELAY_SECONDS < seconds) {
+ String field = Character.toLowerCase(structure.charAt(0)) + structure.substring(1) + "."
+ + Character.toLowerCase(member.charAt(0)) + member.substring(1);
+ String bound = seconds < 1 ? "greater than or equal to 1" : "less than or equal to " + MAX_DELAY_SECONDS;
+ throw new AwsException("ValidationException", "1 validation error detected: Value '" + seconds
+ + "' at '" + field + "' failed to satisfy constraint: Member must have value " + bound, 400);
+ }
+ return (int) seconds;
+ }
+
+ private static Boolean bool(Map<?, ?> map, String member) {
+ Object value = map.get(member);
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof Boolean b) {
+ return b;
+ }
+ throw new AwsException("SerializationException", member + " must be a boolean", 400);
+ }
+
+ private static String text(JsonNode node, String member) {
+ JsonNode value = node.get(member);
+ return value != null && value.isTextual() ? value.asText() : null;
+ }
+
+ private static void putResultAndError(ObjectNode details, DurableOperation operation) {
+ putIfPresent(details, "Result", operation.getResult());
+ if (operation.getError() != null) {
+ details.set("Error", error(operation.getError()));
+ }
+ }
+
+ private static void putIfPresent(ObjectNode node, String member, String value) {
+ if (value != null) {
+ node.put(member, value);
+ }
+ }
+
+ private static void putTimestamp(ObjectNode node, String member, long millis, boolean asMillis) {
+ if (asMillis) {
+ node.put(member, millis);
+ } else {
+ node.put(member, millis / 1000.0);
+ }
+ }
+
+ /** Stored details are already wire shaped. Only the payload wrappers change with IncludeExecutionData. */
+ private static JsonNode detailsNode(Object value, boolean includeExecutionData) {
+ if (value instanceof Map<?, ?> map) {
+ ObjectNode node = NODES.objectNode();
+ boolean wrapper = map.containsKey(DurableHistory.TRUNCATED);
+ for (Map.Entry<?, ?> entry : map.entrySet()) {
+ String key = String.valueOf(entry.getKey());
+ if (wrapper && !includeExecutionData) {
+ if (DurableHistory.PAYLOAD.equals(key)) {
+ continue;
+ }
+ if (DurableHistory.TRUNCATED.equals(key)) {
+ node.put(key, true);
+ continue;
+ }
+ }
+ node.set(key, detailsNode(entry.getValue(), includeExecutionData));
+ }
+ return node;
+ }
+ if (value instanceof List<?> list) {
+ ArrayNode array = NODES.arrayNode();
+ for (Object item : list) {
+ array.add(detailsNode(item, includeExecutionData));
+ }
+ return array;
+ }
+ if (value instanceof Long l) {
+ return NODES.numberNode(l / 1000.0);
+ }
+ if (value instanceof Integer i) {
+ return NODES.numberNode(i);
+ }
+ if (value instanceof Number n) {
+ return NODES.numberNode(n.doubleValue());
+ }
+ if (value instanceof Boolean b) {
+ return NODES.booleanNode(b);
+ }
+ return value == null ? NODES.nullNode() : NODES.textNode(String.valueOf(value));
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionInvoker.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionInvoker.java
new file mode 100644
index 0000000000..3fbaddc0c5
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionInvoker.java
@@ -0,0 +1,48 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.services.lambda.LambdaExecutorService;
+import io.github.hectorvent.floci.services.lambda.LambdaTargetResolver;
+import io.github.hectorvent.floci.services.lambda.model.InvocationType;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+
+/** Runs durable invocations on the real Lambda containers. */
+@ApplicationScoped
+public class LambdaDurableFunctionInvoker implements DurableFunctionInvoker {
+
+ private final LambdaTargetResolver targetResolver;
+ private final LambdaExecutorService executorService;
+
+ @Inject
+ public LambdaDurableFunctionInvoker(LambdaTargetResolver targetResolver, LambdaExecutorService executorService) {
+ this.targetResolver = targetResolver;
+ this.executorService = executorService;
+ }
+
+ @Override
+ public ResolvedDurableTarget resolve(String accountId, String region, String functionName, String qualifier) {
+ LambdaFunction fn = targetResolver.resolveInvokeTargetForAccount(accountId, region, functionName, qualifier);
+ String version = fn.getVersion();
+ String qualifiedArn = "$LATEST".equals(version) ? fn.getFunctionArn() + ":$LATEST" : fn.getFunctionArn();
+ return new ResolvedDurableTarget(accountId, region, fn.getFunctionName(), qualifiedArn, version,
+ fn.isDurable(),
+ fn.getDurableExecutionTimeout() != null ? fn.getDurableExecutionTimeout() : 0,
+ fn.getDurableRetentionPeriodInDays() != null ? fn.getDurableRetentionPeriodInDays() : 0);
+ }
+
+ /**
+ * Resolves the function again on every invocation. An execution is pinned to a version, and a
+ * {@code $LATEST} execution resumes on whatever code $LATEST holds now, as on AWS.
+ */
+ @Override
+ public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {
+ LambdaFunction fn = targetResolver.resolveInvokeTargetForAccount(target.accountId(), target.region(),
+ target.functionName(), target.version());
+ InvokeResult result = RequestScopes.callAs(target.accountId(),
+ () -> executorService.invoke(fn, payload, InvocationType.RequestResponse));
+ return new DurableInvocationResult(result.getRequestId(), result.getPayload(), result.getFunctionError());
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableCheckpointReplay.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableCheckpointReplay.java
new file mode 100644
index 0000000000..45e48b6ee7
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableCheckpointReplay.java
@@ -0,0 +1,44 @@
+package io.github.hectorvent.floci.services.lambda.durable.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** The last accepted checkpoint and its answer as sent, so a retry with the same ClientToken gets it again. */
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class DurableCheckpointReplay {
+
+ private String requestToken;
+ private String clientToken;
+ private String responseToken;
+ private List<DurableOperation> operations = new ArrayList<>();
+
+ public DurableCheckpointReplay() {
+ }
+
+ public DurableCheckpointReplay(String requestToken, String clientToken, String responseToken,
+ List<DurableOperation> operations) {
+ this.requestToken = requestToken;
+ this.clientToken = clientToken;
+ this.responseToken = responseToken;
+ this.operations = new ArrayList<>();
+ for (DurableOperation operation : operations) {
+ this.operations.add(operation.copy());
+ }
+ }
+
+ public String getRequestToken() { return requestToken; }
+ public void setRequestToken(String requestToken) { this.requestToken = requestToken; }
+
+ public String getClientToken() { return clientToken; }
+ public void setClientToken(String clientToken) { this.clientToken = clientToken; }
+
+ public String getResponseToken() { return responseToken; }
+ public void setResponseToken(String responseToken) { this.responseToken = responseToken; }
+
+ public List<DurableOperation> getOperations() { return operations; }
+ public void setOperations(List<DurableOperation> operations) { this.operations = operations; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableErrorObject.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableErrorObject.java
new file mode 100644
index 0000000000..7fd7881df9
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableErrorObject.java
@@ -0,0 +1,41 @@
+package io.github.hectorvent.floci.services.lambda.durable.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+/** The ErrorObject shape shared by checkpoint updates, execution results and history events. */
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class DurableErrorObject {
+
+ private String errorMessage;
+ private String errorType;
+ private String errorData;
+ private List<String> stackTrace;
+
+ public DurableErrorObject() {
+ }
+
+ public static DurableErrorObject of(String errorMessage, String errorType) {
+ DurableErrorObject error = new DurableErrorObject();
+ error.setErrorMessage(errorMessage);
+ error.setErrorType(errorType);
+ return error;
+ }
+
+ public String getErrorMessage() { return errorMessage; }
+ public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }
+
+ public String getErrorType() { return errorType; }
+ public void setErrorType(String errorType) { this.errorType = errorType; }
+
+ public String getErrorData() { return errorData; }
+ public void setErrorData(String errorData) { this.errorData = errorData; }
+
+ public List<String> getStackTrace() { return stackTrace; }
+ public void setStackTrace(List<String> stackTrace) { this.stackTrace = stackTrace; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java
new file mode 100644
index 0000000000..9ff7e2beb6
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java
@@ -0,0 +1,199 @@
+package io.github.hectorvent.floci.services.lambda.durable.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+/**
+ * A durable execution with its checkpointed operations and history. One stored object per
+ * execution, so a checkpoint persists operations, history and the invocation lane together.
+ * Timestamps are epoch millis.
+ */
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class DurableExecution {
+
+ private String executionArn;
+ /** Last ARN segment and the id of the EXECUTION operation. The SDK finds that operation by it. */
+ private String executionId;
+ private String name;
+ private String accountId;
+ private String region;
+ private String functionName;
+ /** Version qualified, as reported in FunctionArn. */
+ private String functionArn;
+ private String version;
+ private DurableExecutionStatus status = DurableExecutionStatus.RUNNING;
+ private long startTimestamp;
+ private Long endTimestamp;
+ private String inputPayload;
+ private String result;
+ private DurableErrorObject error;
+ private int executionTimeoutSeconds;
+ private int retentionPeriodInDays;
+ private Long executionDeadline;
+ private Long retentionDeadline;
+ private int maxResultBytes;
+ /** Creation order. The EXECUTION operation is first. */
+ private LinkedHashMap<String, DurableOperation> operations = new LinkedHashMap<>();
+ private List<DurableHistoryEvent> history = new ArrayList<>();
+ private long changeSequence;
+ /** Change position the handler has been shown. NewExecutionState carries everything newer. */
+ private long seenSequence;
+ private long checkpointSequence;
+ private String currentInvocationId;
+ private boolean reinvokeRequested;
+ private int consecutiveInvocationFailures;
+ private Long nextInvocationAttemptAt;
+ private DurableCheckpointReplay lastCheckpoint;
+
+ public DurableExecution() {
+ }
+
+ @JsonIgnore
+ public boolean isClosed() {
+ return status.isClosed();
+ }
+
+ @JsonIgnore
+ public boolean hasUnseenChanges() {
+ return changeSequence > seenSequence;
+ }
+
+ /** True while a wait or a step retry is still pending. */
+ @JsonIgnore
+ public boolean hasPendingOperations() {
+ for (DurableOperation operation : operations.values()) {
+ DurableOperationStatus operationStatus = operation.getStatus();
+ switch (operation.getType()) {
+ case WAIT -> {
+ if (operationStatus == DurableOperationStatus.STARTED) {
+ return true;
+ }
+ }
+ case STEP -> {
+ if (operationStatus == DurableOperationStatus.PENDING
+ || operationStatus == DurableOperationStatus.READY) {
+ return true;
+ }
+ }
+ default -> {
+ }
+ }
+ }
+ return false;
+ }
+
+ public long nextChangeSequence() {
+ changeSequence++;
+ return changeSequence;
+ }
+
+ public DurableHistoryEvent appendEvent(String eventType, long timestamp) {
+ DurableHistoryEvent event = new DurableHistoryEvent();
+ event.setEventId(history.size() + 1L);
+ event.setEventType(eventType);
+ event.setEventTimestamp(timestamp);
+ history.add(event);
+ return event;
+ }
+
+ public String getExecutionArn() { return executionArn; }
+ public void setExecutionArn(String executionArn) { this.executionArn = executionArn; }
+
+ public String getExecutionId() { return executionId; }
+ public void setExecutionId(String executionId) { this.executionId = executionId; }
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+
+ public String getAccountId() { return accountId; }
+ public void setAccountId(String accountId) { this.accountId = accountId; }
+
+ public String getRegion() { return region; }
+ public void setRegion(String region) { this.region = region; }
+
+ public String getFunctionName() { return functionName; }
+ public void setFunctionName(String functionName) { this.functionName = functionName; }
+
+ public String getFunctionArn() { return functionArn; }
+ public void setFunctionArn(String functionArn) { this.functionArn = functionArn; }
+
+ public String getVersion() { return version; }
+ public void setVersion(String version) { this.version = version; }
+
+ public DurableExecutionStatus getStatus() { return status; }
+ public void setStatus(DurableExecutionStatus status) { this.status = status; }
+
+ public long getStartTimestamp() { return startTimestamp; }
+ public void setStartTimestamp(long startTimestamp) { this.startTimestamp = startTimestamp; }
+
+ public Long getEndTimestamp() { return endTimestamp; }
+ public void setEndTimestamp(Long endTimestamp) { this.endTimestamp = endTimestamp; }
+
+ public String getInputPayload() { return inputPayload; }
+ public void setInputPayload(String inputPayload) { this.inputPayload = inputPayload; }
+
+ public String getResult() { return result; }
+ public void setResult(String result) { this.result = result; }
+
+ public DurableErrorObject getError() { return error; }
+ public void setError(DurableErrorObject error) { this.error = error; }
+
+ public int getExecutionTimeoutSeconds() { return executionTimeoutSeconds; }
+ public void setExecutionTimeoutSeconds(int executionTimeoutSeconds) {
+ this.executionTimeoutSeconds = executionTimeoutSeconds;
+ }
+
+ public int getRetentionPeriodInDays() { return retentionPeriodInDays; }
+ public void setRetentionPeriodInDays(int retentionPeriodInDays) { this.retentionPeriodInDays = retentionPeriodInDays; }
+
+ public Long getExecutionDeadline() { return executionDeadline; }
+ public void setExecutionDeadline(Long executionDeadline) { this.executionDeadline = executionDeadline; }
+
+ public Long getRetentionDeadline() { return retentionDeadline; }
+ public void setRetentionDeadline(Long retentionDeadline) { this.retentionDeadline = retentionDeadline; }
+
+ public int getMaxResultBytes() { return maxResultBytes; }
+ public void setMaxResultBytes(int maxResultBytes) { this.maxResultBytes = maxResultBytes; }
+
+ public LinkedHashMap<String, DurableOperation> getOperations() { return operations; }
+ public void setOperations(LinkedHashMap<String, DurableOperation> operations) { this.operations = operations; }
+
+ public List<DurableHistoryEvent> getHistory() { return history; }
+ public void setHistory(List<DurableHistoryEvent> history) { this.history = history; }
+
+ public long getChangeSequence() { return changeSequence; }
+ public void setChangeSequence(long changeSequence) { this.changeSequence = changeSequence; }
+
+ public long getSeenSequence() { return seenSequence; }
+ public void setSeenSequence(long seenSequence) { this.seenSequence = seenSequence; }
+
+ public long getCheckpointSequence() { return checkpointSequence; }
+ public void setCheckpointSequence(long checkpointSequence) { this.checkpointSequence = checkpointSequence; }
+
+ public String getCurrentInvocationId() { return currentInvocationId; }
+ public void setCurrentInvocationId(String currentInvocationId) { this.currentInvocationId = currentInvocationId; }
+
+ public boolean isReinvokeRequested() { return reinvokeRequested; }
+ public void setReinvokeRequested(boolean reinvokeRequested) { this.reinvokeRequested = reinvokeRequested; }
+
+ public int getConsecutiveInvocationFailures() { return consecutiveInvocationFailures; }
+ public void setConsecutiveInvocationFailures(int consecutiveInvocationFailures) {
+ this.consecutiveInvocationFailures = consecutiveInvocationFailures;
+ }
+
+ public Long getNextInvocationAttemptAt() { return nextInvocationAttemptAt; }
+ public void setNextInvocationAttemptAt(Long nextInvocationAttemptAt) {
+ this.nextInvocationAttemptAt = nextInvocationAttemptAt;
+ }
+
+ public DurableCheckpointReplay getLastCheckpoint() { return lastCheckpoint; }
+ public void setLastCheckpoint(DurableCheckpointReplay lastCheckpoint) { this.lastCheckpoint = lastCheckpoint; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecutionStatus.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecutionStatus.java
new file mode 100644
index 0000000000..4c224c3041
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecutionStatus.java
@@ -0,0 +1,12 @@
+package io.github.hectorvent.floci.services.lambda.durable.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public enum DurableExecutionStatus {
+ RUNNING, SUCCEEDED, FAILED, TIMED_OUT, STOPPED;
+
+ public boolean isClosed() {
+ return this != RUNNING;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableHistoryEvent.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableHistoryEvent.java
new file mode 100644
index 0000000000..af3a4c1a3e
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableHistoryEvent.java
@@ -0,0 +1,53 @@
+package io.github.hectorvent.floci.services.lambda.durable.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.Map;
+
+/**
+ * One GetDurableExecutionHistory event. {@code details} is stored in wire form already, keyed by
+ * the {@code <EventType>Details} member name, with payloads wrapped as {@code {Payload, Truncated}}.
+ */
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class DurableHistoryEvent {
+
+ private long eventId;
+ private long eventTimestamp;
+ private String eventType;
+ private String id;
+ private String name;
+ private String subType;
+ private String parentId;
+ private Map<String, Object> details;
+
+ public DurableHistoryEvent() {
+ }
+
+ public long getEventId() { return eventId; }
+ public void setEventId(long eventId) { this.eventId = eventId; }
+
+ public long getEventTimestamp() { return eventTimestamp; }
+ public void setEventTimestamp(long eventTimestamp) { this.eventTimestamp = eventTimestamp; }
+
+ public String getEventType() { return eventType; }
+ public void setEventType(String eventType) { this.eventType = eventType; }
+
+ public String getId() { return id; }
+ public void setId(String id) { this.id = id; }
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+
+ public String getSubType() { return subType; }
+ public void setSubType(String subType) { this.subType = subType; }
+
+ public String getParentId() { return parentId; }
+ public void setParentId(String parentId) { this.parentId = parentId; }
+
+ public Map<String, Object> getDetails() { return details; }
+ public void setDetails(Map<String, Object> details) { this.details = details; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java
new file mode 100644
index 0000000000..1ff6aa8f5c
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java
@@ -0,0 +1,106 @@
+package io.github.hectorvent.floci.services.lambda.durable.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+/**
+ * One checkpointed operation of a durable execution. A single flat class serves every operation
+ * type. The wire mapping emits only the details block that belongs to {@link #getType()}.
+ * Timestamps are epoch millis.
+ */
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class DurableOperation {
+
+ private String id;
+ private String parentId;
+ private String name;
+ private DurableOperationType type;
+ private String subType;
+ private DurableOperationStatus status;
+ private long startTimestamp;
+ private Long endTimestamp;
+ private String inputPayload;
+ private Boolean replayChildren;
+ private String result;
+ private DurableErrorObject error;
+ private Integer attempt;
+ private Long nextAttemptTimestamp;
+ private Long scheduledEndTimestamp;
+ /** Position in the execution's change log. The handler is only shown operations newer than it has seen. */
+ private long changeSequence;
+
+ public DurableOperation() {
+ }
+
+ public DurableOperation copy() {
+ DurableOperation copy = new DurableOperation();
+ copy.id = id;
+ copy.parentId = parentId;
+ copy.name = name;
+ copy.type = type;
+ copy.subType = subType;
+ copy.status = status;
+ copy.startTimestamp = startTimestamp;
+ copy.endTimestamp = endTimestamp;
+ copy.inputPayload = inputPayload;
+ copy.replayChildren = replayChildren;
+ copy.result = result;
+ copy.error = error;
+ copy.attempt = attempt;
+ copy.nextAttemptTimestamp = nextAttemptTimestamp;
+ copy.scheduledEndTimestamp = scheduledEndTimestamp;
+ copy.changeSequence = changeSequence;
+ return copy;
+ }
+
+ public String getId() { return id; }
+ public void setId(String id) { this.id = id; }
+
+ public String getParentId() { return parentId; }
+ public void setParentId(String parentId) { this.parentId = parentId; }
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+
+ public DurableOperationType getType() { return type; }
+ public void setType(DurableOperationType type) { this.type = type; }
+
+ public String getSubType() { return subType; }
+ public void setSubType(String subType) { this.subType = subType; }
+
+ public DurableOperationStatus getStatus() { return status; }
+ public void setStatus(DurableOperationStatus status) { this.status = status; }
+
+ public long getStartTimestamp() { return startTimestamp; }
+ public void setStartTimestamp(long startTimestamp) { this.startTimestamp = startTimestamp; }
+
+ public Long getEndTimestamp() { return endTimestamp; }
+ public void setEndTimestamp(Long endTimestamp) { this.endTimestamp = endTimestamp; }
+
+ public String getInputPayload() { return inputPayload; }
+ public void setInputPayload(String inputPayload) { this.inputPayload = inputPayload; }
+
+ public Boolean getReplayChildren() { return replayChildren; }
+ public void setReplayChildren(Boolean replayChildren) { this.replayChildren = replayChildren; }
+
+ public String getResult() { return result; }
+ public void setResult(String result) { this.result = result; }
+
+ public DurableErrorObject getError() { return error; }
+ public void setError(DurableErrorObject error) { this.error = error; }
+
+ public Integer getAttempt() { return attempt; }
+ public void setAttempt(Integer attempt) { this.attempt = attempt; }
+
+ public Long getNextAttemptTimestamp() { return nextAttemptTimestamp; }
+ public void setNextAttemptTimestamp(Long nextAttemptTimestamp) { this.nextAttemptTimestamp = nextAttemptTimestamp; }
+
+ public Long getScheduledEndTimestamp() { return scheduledEndTimestamp; }
+ public void setScheduledEndTimestamp(Long scheduledEndTimestamp) { this.scheduledEndTimestamp = scheduledEndTimestamp; }
+
+ public long getChangeSequence() { return changeSequence; }
+ public void setChangeSequence(long changeSequence) { this.changeSequence = changeSequence; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationAction.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationAction.java
new file mode 100644
index 0000000000..88b16318f4
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationAction.java
@@ -0,0 +1,8 @@
+package io.github.hectorvent.floci.services.lambda.durable.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public enum DurableOperationAction {
+ START, SUCCEED, FAIL, RETRY, CANCEL
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationStatus.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationStatus.java
new file mode 100644
index 0000000000..db44171a2b
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationStatus.java
@@ -0,0 +1,8 @@
+package io.github.hectorvent.floci.services.lambda.durable.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public enum DurableOperationStatus {
+ STARTED, PENDING, READY, SUCCEEDED, FAILED, CANCELLED, TIMED_OUT, STOPPED
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationType.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationType.java
new file mode 100644
index 0000000000..66f080e8f2
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationType.java
@@ -0,0 +1,8 @@
+package io.github.hectorvent.floci.services.lambda.durable.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public enum DurableOperationType {
+ EXECUTION, CONTEXT, STEP, WAIT, CALLBACK, CHAINED_INVOKE
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java
new file mode 100644
index 0000000000..ca9b275f4e
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java
@@ -0,0 +1,18 @@
+package io.github.hectorvent.floci.services.lambda.durable.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+/** One entry of a CheckpointDurableExecution {@code Updates} list, after parsing. */
+@RegisterForReflection
+public record DurableOperationUpdate(String id,
+ String parentId,
+ String name,
+ DurableOperationType type,
+ String subType,
+ DurableOperationAction action,
+ String payload,
+ DurableErrorObject error,
+ Integer nextAttemptDelaySeconds,
+ Integer waitSeconds,
+ Boolean replayChildren) {
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/model/InvokeResult.java b/src/main/java/io/github/hectorvent/floci/services/lambda/model/InvokeResult.java
index e155a4dfb9..0f88554421 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/model/InvokeResult.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/model/InvokeResult.java
@@ -11,6 +11,7 @@ public class InvokeResult {
private String logResult;
private String requestId;
private String executedVersion;
+ private String durableExecutionArn;
public InvokeResult() {
}
@@ -70,4 +71,12 @@ public class InvokeResult {
public void setExecutedVersion(String executedVersion) {
this.executedVersion = executedVersion;
}
+
+ public String getDurableExecutionArn() {
+ return durableExecutionArn;
+ }
+
+ public void setDurableExecutionArn(String durableExecutionArn) {
+ this.durableExecutionArn = durableExecutionArn;
+ }
}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 7dc9cb7a18..f89a74dc81 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -384,6 +384,8 @@ floci:
# container-name-prefix: floci # FLOCI_SERVICES_LAMBDA_CONTAINER_NAME_PREFIX — base name prefix for spawned containers and code volumes ([A-Za-z0-9][A-Za-z0-9_.-]*; invalid values fall back to "floci")
# code-volume-populate-concurrency: # FLOCI_SERVICES_LAMBDA_CODE_VOLUME_POPULATE_CONCURRENCY — max concurrent first-time code-volume populates (default max(2, cpus/2), which a CPU-constrained container collapses to 2)
executor: docker # docker | kubernetes (run Lambda environments as pods)
+ durable-sweep-enabled: true # FLOCI_SERVICES_LAMBDA_DURABLE_SWEEP_ENABLED - fire durable execution timers (waits, step retries, execution timeouts, retention)
+ durable-sweep-interval-seconds: 1 # FLOCI_SERVICES_LAMBDA_DURABLE_SWEEP_INTERVAL_SECONDS
# kubernetes:
# namespace: default # Namespace Lambda pods are created in
# labels: # Extra pod labels, key=value entries
diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
index 94c6fc2db2..1c61a00bed 100644
--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
@@ -24,6 +24,7 @@ import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.lambda.DynamoDbStreamsEventSourcePoller;
import io.github.hectorvent.floci.services.lambda.KinesisEventSourcePoller;
import io.github.hectorvent.floci.services.lambda.SqsEventSourcePoller;
+import io.github.hectorvent.floci.services.lambda.durable.DurableExecutionService;
import io.github.hectorvent.floci.services.neptune.container.NeptuneContainerManager;
import io.github.hectorvent.floci.services.neptune.proxy.NeptuneProxyManager;
import io.github.hectorvent.floci.services.pipes.PipesService;
@@ -73,6 +74,7 @@ class EmulatorLifecycleTest {
@Mock private EmulatorConfig config;
@Mock private EmulatorConfig.StorageConfig storageConfig;
@Mock private EmulatorConfig.ServicesConfig servicesConfig;
+ @Mock private EmulatorConfig.LambdaServiceConfig lambdaServiceConfig;
@Mock private EmulatorConfig.Ec2ServiceConfig ec2ServiceConfig;
@Mock private EmulatorConfig.EcsServiceConfig ecsServiceConfig;
@Mock private EmulatorConfig.EcsTaskRoleCredentialsConfig ecsTaskRoleCredentialsConfig;
@@ -116,6 +118,7 @@ class EmulatorLifecycleTest {
@Mock private EmulatorConfig.IamServiceConfig iamServiceConfig;
@Mock private SchemaCreationWorker schemaCreationWorker;
@Mock private StepFunctionsService stepFunctionsService;
+ @Mock private DurableExecutionService durableExecutionService;
@Mock private Instance<ContainerTeardown> containerTeardowns;
@Mock private DynamoDbRuntime dynamoDbRuntime;
@Mock private RedshiftDynamoDbZeroEtlConsumer redshiftZeroEtlConsumer;
@@ -137,6 +140,8 @@ class EmulatorLifecycleTest {
Mockito.lenient().when(elastiCacheServiceConfig.enabled()).thenReturn(false);
Mockito.lenient().when(servicesConfig.elb()).thenReturn(elbServiceConfig);
Mockito.lenient().when(elbServiceConfig.enabled()).thenReturn(false);
+ Mockito.lenient().when(servicesConfig.lambda()).thenReturn(lambdaServiceConfig);
+ Mockito.lenient().when(lambdaServiceConfig.enabled()).thenReturn(false);
Mockito.lenient().when(servicesConfig.eks()).thenReturn(eksServiceConfig);
Mockito.lenient().when(eksServiceConfig.enabled()).thenReturn(false);
Mockito.lenient().when(servicesConfig.timestreamInfluxdb()).thenReturn(timestreamInfluxDbServiceConfig);
@@ -158,7 +163,8 @@ class EmulatorLifecycleTest {
elbV2Service, elbClassicService, ecsService,
initializationHooksRunner, sqsPoller, kinesisPoller, dynamodbStreamsPoller,
pipesService, ec2MetadataServer, ecsTaskRoleCredentialsServer, ecrRegistryManager, flociUiManager, initLifecycleState,
- schemaCreationWorker, stepFunctionsService, containerTeardowns, persistentPathValidator,
+ schemaCreationWorker, stepFunctionsService, durableExecutionService, containerTeardowns,
+ persistentPathValidator,
dynamoDbRuntime, redshiftZeroEtlConsumer);
Mockito.lenient().when(containerTeardowns.iterator())
.thenReturn(java.util.Collections.emptyIterator());
diff --git a/src/test/java/io/github/hectorvent/floci/services/iam/IamActionRegistryTest.java b/src/test/java/io/github/hectorvent/floci/services/iam/IamActionRegistryTest.java
index 89a6b385d9..d9f8926544 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iam/IamActionRegistryTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iam/IamActionRegistryTest.java
@@ -210,6 +210,25 @@ class IamActionRegistryTest {
assertEquals("lambda:InvokeFunction", registry.resolveRoute("lambda", invoke));
}
+ @Test
+ void durableExecutionRoutesResolveToTheirActions() {
+ String arn = "arn%3Aaws%3Alambda%3Aus-east-1%3A000000000000%3Afunction%3Af%3A%24LATEST"
+ + "%2Fdurable-execution%2Fname%2Fid";
+ String base = "/2025-12-01/durable-executions/" + arn;
+ assertEquals("lambda:CheckpointDurableExecution", lambdaAction("POST", base + "/checkpoint"));
+ assertEquals("lambda:GetDurableExecutionState", lambdaAction("GET", base + "/state"));
+ assertEquals("lambda:GetDurableExecutionHistory", lambdaAction("GET", base + "/history"));
+ assertEquals("lambda:StopDurableExecution", lambdaAction("POST", base + "/stop"));
+ assertEquals("lambda:GetDurableExecution", lambdaAction("GET", base));
+ assertEquals("lambda:ListDurableExecutionsByFunction",
+ lambdaAction("GET", "/2025-12-01/functions/f/durable-executions"));
+ }
+
+ private String lambdaAction(String method, String path) {
+ return registry.resolveRoute("lambda",
+ mockCtx(method, path, new MultivaluedHashMap<>(), MediaType.APPLICATION_JSON_TYPE, "{}"));
+ }
+
@Test
void anEncodedSlashStaysInsideItsPathParameter() {
assertEquals("ses:GetSuppressedDestination", registry.resolveRoute("ses",
diff --git a/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java b/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java
index fcbdfa413c..42642f04d3 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java
@@ -595,4 +595,21 @@ class ResourceArnBuilderTest {
String arn = builder.build("lambda", ctx, "us-east-1", "000000000000");
assertEquals("arn:aws:lambda:us-east-1:000000000000:function:my-function", arn);
}
+
+ @Test
+ void lambdaDurableExecutionRoutesNameTheExecution() {
+ when(uriInfo.getPath()).thenReturn("/2025-12-01/durable-executions/arn:aws:lambda:us-east-1:000000000000:"
+ + "function:my-function:$LATEST/durable-execution/run-1/0b1c/stop");
+ String arn = builder.build("lambda", ctx, "us-east-1", "000000000000");
+ assertEquals("arn:aws:lambda:us-east-1:000000000000:function:my-function:$LATEST/durable-execution/run-1/0b1c",
+ arn);
+ }
+
+ @Test
+ void lambdaDurableExecutionGetNamesTheExecution() {
+ when(uriInfo.getPath()).thenReturn("/2025-12-01/durable-executions/arn:aws:lambda:us-east-1:000000000000:"
+ + "function:my-function:1/durable-execution/run-1/0b1c");
+ String arn = builder.build("lambda", ctx, "us-east-1", "000000000000");
+ assertEquals("arn:aws:lambda:us-east-1:000000000000:function:my-function:1/durable-execution/run-1/0b1c", arn);
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaArnInvocationAccountTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaArnInvocationAccountTest.java
index 68c7e23106..8a375b0f94 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaArnInvocationAccountTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaArnInvocationAccountTest.java
@@ -77,6 +77,7 @@ class LambdaArnInvocationAccountTest {
null,
null,
null,
+ null,
new ObjectMapper());
InvokeResult result = service.invokeArn(functionArn, "{}".getBytes(), InvocationType.Event);
@@ -140,6 +141,7 @@ class LambdaArnInvocationAccountTest {
null,
null,
null,
+ null,
new ObjectMapper());
InvokeResult versionResult = service.invokeArn(
@@ -313,6 +315,7 @@ class LambdaArnInvocationAccountTest {
null,
null,
null,
+ null,
new ObjectMapper());
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaInvokeClientContextIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaInvokeClientContextIntegrationTest.java
index 5468c6307d..c24a4a9af2 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaInvokeClientContextIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaInvokeClientContextIntegrationTest.java
@@ -31,7 +31,7 @@ class LambdaInvokeClientContextIntegrationTest {
}
private void stubInvoke() {
- when(lambdaService.invoke(any(), any(), any(), any(), any(), any()))
+ when(lambdaService.invoke(any(), any(), any(), any(), any(), any(), any()))
.thenReturn(new InvokeResult(200, null, "{}".getBytes(), null, "req"));
}
@@ -47,7 +47,7 @@ class LambdaInvokeClientContextIntegrationTest {
.then().statusCode(200);
verify(lambdaService).invoke(any(), eq("ctx-fn"), any(), any(),
- eq(InvocationType.RequestResponse), eq(CONTEXT_JSON));
+ eq(InvocationType.RequestResponse), eq(CONTEXT_JSON), isNull());
}
@Test
@@ -63,7 +63,7 @@ class LambdaInvokeClientContextIntegrationTest {
.then().statusCode(200);
verify(lambdaService).invoke(any(), eq("ctx-fn-event"), any(), any(),
- eq(InvocationType.Event), isNull());
+ eq(InvocationType.Event), isNull(), isNull());
}
@Test
@@ -80,6 +80,6 @@ class LambdaInvokeClientContextIntegrationTest {
.then().statusCode(200);
verify(lambdaService).invoke(any(), eq("ctx-fn-utf"), any(), any(),
- eq(InvocationType.RequestResponse), eq("{\"custom\":{\"name\":\"caf\\u00E9\"}}"));
+ eq(InvocationType.RequestResponse), eq("{\"custom\":{\"name\":\"caf\\u00E9\"}}"), isNull());
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java
new file mode 100644
index 0000000000..65963c5b9d
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java
@@ -0,0 +1,172 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableErrorObject;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecution;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperation;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationAction;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationStatus;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationType;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationUpdate;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/** The batch rules of CheckpointDurableExecution, and that a rejected batch changes nothing. */
+class DurableCheckpointApplierTest {
+
+ private static final long NOW = 1_700_000_000_000L;
+
+ @Test
+ void executionUpdateMustBeTheOnlyOneAndTheLast() {
+ assertRejected(List.of(executionSucceed(), executionSucceed()), "Cannot checkpoint multiple EXECUTION updates.");
+ assertRejected(List.of(executionSucceed(), stepStart("s1", null)), "EXECUTION checkpoint must be the last update.");
+ }
+
+ @Test
+ void anExecutionCloseCarriesOnlyItsOwnResultKind() {
+ DurableErrorObject error = DurableErrorObject.of("m", "T");
+ assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,
+ DurableOperationAction.SUCCEED, "\"p\"", error, null, null, null)),
+ "Cannot provide an Error for SUCCEED action.");
+ assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,
+ DurableOperationAction.FAIL, "\"p\"", error, null, null, null)),
+ "Cannot provide a Payload for FAIL action.");
+ }
+
+ @Test
+ void anExecutionErrorHasTheSameSizeLimitAsAnOperationError() {
+ DurableErrorObject large = DurableErrorObject.of("x".repeat(DurableCheckpointApplier.MAX_ERROR_BYTES), "T");
+ assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,
+ DurableOperationAction.FAIL, null, large, null, null, null)),
+ "Error object size must be less than 32768 bytes.");
+ }
+
+ @Test
+ void duplicateIdsAreRejectedUnlessAStartIsClosedInTheSameBatch() {
+ assertRejected(List.of(stepStart("s1", null), stepStart("s1", null)),
+ "Cannot checkpoint multiple operations with the same ID.");
+ assertRejected(List.of(waitStart("w1"), waitStart("w1")), "Cannot checkpoint multiple operations with the same ID.");
+ DurableExecution execution = execution();
+ DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null), update("s1", null,
+ DurableOperationType.STEP, DurableOperationAction.SUCCEED, "1")), NOW);
+ assertEquals(DurableOperationStatus.SUCCEEDED, execution.getOperations().get("s1").getStatus());
+ }
+
+ @Test
+ void parentMustBeAContextAndMetadataMustStayConsistent() {
+ assertRejected(List.of(stepStart("s1", "missing")), "Invalid parent operation id.");
+ DurableExecution execution = execution();
+ DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null)), NOW);
+ assertEquals("Inconsistent operation type.", reject(execution, List.of(waitStart("s1"))));
+ assertEquals("Inconsistent parent operation id.", reject(execution, List.of(stepStart("ctx", null),
+ update("s1", "ctx", DurableOperationType.STEP, DurableOperationAction.SUCCEED, "1"))));
+ }
+
+ @Test
+ void stepTransitionsAreGuarded() {
+ DurableExecution execution = execution();
+ DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null)), NOW);
+ assertEquals("Invalid current STEP state to start.", reject(execution, List.of(stepStart("s1", null))));
+ assertEquals("Cannot provide an Error for SUCCEED action.", reject(execution, List.of(
+ new DurableOperationUpdate("s1", null, null, DurableOperationType.STEP, null,
+ DurableOperationAction.SUCCEED, null, DurableErrorObject.of("x", "y"), null, null, null))));
+ assertEquals("Invalid StepOptions for the given action.", reject(execution, List.of(
+ update("s1", null, DurableOperationType.STEP, DurableOperationAction.RETRY, null))));
+ DurableCheckpointApplier.apply(execution, List.of(update("s1", null, DurableOperationType.STEP,
+ DurableOperationAction.FAIL, null)), NOW);
+ assertEquals("Invalid current STEP state to close.", reject(execution, List.of(
+ update("s1", null, DurableOperationType.STEP, DurableOperationAction.SUCCEED, "1"))));
+ }
+
+ @Test
+ void waitsNeedOptionsAndCancelOnlyWhileStarted() {
+ assertRejected(List.of(update("w1", null, DurableOperationType.WAIT, DurableOperationAction.START, null)),
+ "Update for WAIT operation requires WaitOptions.");
+ assertRejected(List.of(update("w1", null, DurableOperationType.WAIT, DurableOperationAction.CANCEL, null)),
+ "Cannot cancel a WAIT that does not exist or has already completed.");
+ DurableExecution execution = execution();
+ DurableCheckpointApplier.apply(execution, List.of(waitStart("w1")), NOW);
+ assertEquals("Cannot start a WAIT that already exist.", reject(execution, List.of(waitStart("w1"))));
+ DurableCheckpointApplier.apply(execution, List.of(update("w1", null, DurableOperationType.WAIT,
+ DurableOperationAction.CANCEL, null)), NOW);
+ assertEquals(DurableOperationStatus.CANCELLED, execution.getOperations().get("w1").getStatus());
+ }
+
+ @Test
+ void delaysOutsideTheirRangeAreValidationErrors() {
+ AwsException rejected = assertThrows(AwsException.class, () -> DurableWire.parseUpdates(
+ List.of(Map.of("Id", "w1", "Type", "WAIT", "Action", "START", "WaitOptions", Map.of("WaitSeconds", 0)))));
+ assertEquals("ValidationException", rejected.getErrorCode());
+ assertEquals("1 validation error detected: Value '0' at 'waitOptions.waitSeconds' failed to satisfy "
+ + "constraint: Member must have value greater than or equal to 1", rejected.getMessage());
+ }
+
+ @Test
+ void callbacksAndChainedInvokesAreNotSupportedYet() {
+ assertRejected(List.of(update("c1", null, DurableOperationType.CALLBACK, DurableOperationAction.START, null)),
+ "CALLBACK operations are not supported yet");
+ assertRejected(List.of(update("i1", null, DurableOperationType.CHAINED_INVOKE, DurableOperationAction.START,
+ null)), "CHAINED_INVOKE operations are not supported yet");
+ }
+
+ @Test
+ void aRejectedBatchLeavesTheExecutionUntouched() {
+ DurableExecution execution = execution();
+ long sequence = execution.getChangeSequence();
+ int events = execution.getHistory().size();
+ reject(execution, List.of(stepStart("s1", null), waitStart("w1"), waitStart("w1")));
+ assertEquals(1, execution.getOperations().size());
+ assertEquals(sequence, execution.getChangeSequence());
+ assertEquals(events, execution.getHistory().size());
+ }
+
+ private static void assertRejected(List<DurableOperationUpdate> updates, String message) {
+ assertEquals(message, reject(execution(), updates));
+ }
+
+ private static String reject(DurableExecution execution, List<DurableOperationUpdate> updates) {
+ AwsException rejected = assertThrows(AwsException.class,
+ () -> DurableCheckpointApplier.apply(execution, updates, NOW));
+ assertEquals(400, rejected.getHttpStatus());
+ assertEquals("InvalidParameterValueException", rejected.getErrorCode());
+ return rejected.getMessage();
+ }
+
+ private static DurableExecution execution() {
+ DurableExecution execution = new DurableExecution();
+ execution.setExecutionId("exec-id");
+ execution.setName("exec");
+ execution.setMaxResultBytes(DurableExecutionService.ASYNC_PAYLOAD_LIMIT);
+ DurableOperation root = new DurableOperation();
+ root.setId("exec-id");
+ root.setType(DurableOperationType.EXECUTION);
+ root.setStatus(DurableOperationStatus.STARTED);
+ root.setStartTimestamp(NOW);
+ root.setChangeSequence(execution.nextChangeSequence());
+ execution.getOperations().put("exec-id", root);
+ return execution;
+ }
+
+ private static DurableOperationUpdate stepStart(String id, String parentId) {
+ return update(id, parentId, DurableOperationType.STEP, DurableOperationAction.START, null);
+ }
+
+ private static DurableOperationUpdate waitStart(String id) {
+ return new DurableOperationUpdate(id, null, null, DurableOperationType.WAIT, null, DurableOperationAction.START,
+ null, null, null, 5, null);
+ }
+
+ private static DurableOperationUpdate executionSucceed() {
+ return update("result", null, DurableOperationType.EXECUTION, DurableOperationAction.SUCCEED, "{}");
+ }
+
+ private static DurableOperationUpdate update(String id, String parentId, DurableOperationType type,
+ DurableOperationAction action, String payload) {
+ return new DurableOperationUpdate(id, parentId, null, type, null, action, payload, null, null, null, null);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java
new file mode 100644
index 0000000000..b967e1ff07
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java
@@ -0,0 +1,196 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.PersistentStorage;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.core.storage.WriteProfile;
+import io.github.hectorvent.floci.services.lambda.durable.DurableExecutionService.StartRequest;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecution;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecutionStatus;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableHistoryEvent;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationAction;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationType;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationUpdate;
+import io.github.hectorvent.floci.testing.MutableClock;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * A restart reloads executions from the JSON file, not from shared objects, so this proves that
+ * operations, history, deadlines and the invocation lane survive serialization.
+ */
+class DurableExecutionServicePersistenceTest {
+
+ private static final String ACCOUNT = "000000000000";
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ @TempDir
+ Path directory;
+
+ private final MutableClock clock = new MutableClock();
+
+ @Test
+ void aRunningExecutionResumesFromDiskAfterARestart() {
+ ScriptedInvoker before = new ScriptedInvoker();
+ PersistentStorageFactory firstStorage = new PersistentStorageFactory(directory);
+ DurableExecutionService first = newService(firstStorage, before);
+ before.script((service, event) -> {
+ service.checkpoint(arn(event), token(event), null, List.of(
+ update("s1", DurableOperationType.STEP, DurableOperationAction.START, null, null),
+ update("s1", DurableOperationType.STEP, DurableOperationAction.SUCCEED, "\"saved\"", null),
+ update("w1", DurableOperationType.WAIT, DurableOperationAction.START, null, 5)));
+ return "{\"Status\":\"PENDING\"}";
+ });
+ String executionArn = first.start(new StartRequest(ACCOUNT, "us-east-1", "durable-fn", "1", "exec-1", "{\"a\":1}",
+ false)).getExecutionArn();
+ String oldToken = token(before.events.get(0));
+ firstStorage.flushAll();
+
+ ScriptedInvoker after = new ScriptedInvoker();
+ DurableExecutionService restarted = newService(new PersistentStorageFactory(directory), after);
+ after.script((service, event) -> {
+ assertEquals("SUCCEEDED", operation(event, "s1").get("Status").asText());
+ assertEquals("\"saved\"", operation(event, "s1").at("/StepDetails/Result").asText());
+ assertEquals("STARTED", operation(event, "w1").get("Status").asText());
+ return "{\"Status\":\"PENDING\"}";
+ });
+ after.script((service, event) -> {
+ assertEquals("SUCCEEDED", operation(event, "w1").get("Status").asText());
+ return "{\"Status\":\"SUCCEEDED\",\"Result\":\"\\\"done\\\"\"}";
+ });
+ restarted.recoverAfterRestart();
+
+ assertThrows(AwsException.class, () -> restarted.checkpoint(executionArn, oldToken, null, List.of()));
+ clock.advance(Duration.ofSeconds(5));
+ restarted.sweep();
+
+ DurableExecution execution = restarted.get(executionArn);
+ assertEquals(DurableExecutionStatus.SUCCEEDED, execution.getStatus());
+ assertEquals("\"done\"", execution.getResult());
+ assertEquals("{\"a\":1}", execution.getInputPayload());
+ assertEquals(List.of("ExecutionStarted", "StepStarted", "StepSucceeded", "WaitStarted", "InvocationCompleted",
+ "InvocationCompleted", "WaitSucceeded", "InvocationCompleted", "ExecutionSucceeded"),
+ execution.getHistory().stream().map(DurableHistoryEvent::getEventType).toList());
+ }
+
+ private DurableExecutionService newService(StorageFactory storage, ScriptedInvoker invoker) {
+ DurableExecutionService service = new DurableExecutionService(storage, MAPPER, clock, invoker, Runnable::run,
+ Duration.ZERO);
+ invoker.service = service;
+ return service;
+ }
+
+ private static DurableOperationUpdate update(String id, DurableOperationType type, DurableOperationAction action,
+ String payload, Integer waitSeconds) {
+ return new DurableOperationUpdate(id, null, null, type, null, action, payload, null, null, waitSeconds, null);
+ }
+
+ private static String arn(JsonNode event) {
+ return event.get("DurableExecutionArn").asText();
+ }
+
+ private static String token(JsonNode event) {
+ return event.get("CheckpointToken").asText();
+ }
+
+ private static JsonNode operation(JsonNode event, String id) {
+ for (JsonNode operation : event.at("/InitialExecutionState/Operations")) {
+ if (id.equals(operation.get("Id").asText())) {
+ return operation;
+ }
+ }
+ throw new AssertionError("no operation " + id);
+ }
+
+ private interface Script {
+ String run(DurableExecutionService service, JsonNode event);
+ }
+
+ private static final class ScriptedInvoker implements DurableFunctionInvoker {
+
+ final Deque<Script> scripts = new ArrayDeque<>();
+ final List<JsonNode> events = new ArrayList<>();
+ DurableExecutionService service;
+
+ void script(Script script) {
+ scripts.add(script);
+ }
+
+ @Override
+ public ResolvedDurableTarget resolve(String accountId, String region, String functionName, String qualifier) {
+ return new ResolvedDurableTarget(accountId, region, functionName,
+ "arn:aws:lambda:us-east-1:000000000000:function:" + functionName + ":1", "1", true, 3600, 7);
+ }
+
+ @Override
+ public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {
+ JsonNode event;
+ try {
+ event = MAPPER.readTree(payload);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ events.add(event);
+ Script script = scripts.poll();
+ String body = script == null ? "{\"Status\":\"FAILED\",\"Error\":{\"ErrorMessage\":\"no script\"}}"
+ : script.run(service, event);
+ return new DurableInvocationResult("req", body.getBytes(StandardCharsets.UTF_8), null);
+ }
+ }
+
+ /** Each instance reads the JSON file from disk, as a restarted Floci does. */
+ private static final class PersistentStorageFactory extends StorageFactory {
+
+ private final Path directory;
+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
+
+ PersistentStorageFactory(Path directory) {
+ super(null, null);
+ this.directory = directory;
+ }
+
+ @Override
+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return create(serviceName, fileName, typeReference, WriteProfile.DEFAULT);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+ TypeReference<Map<String, V>> typeReference,
+ WriteProfile profile) {
+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName, ignored -> {
+ PersistentStorage<String, V> storage = new PersistentStorage<>(directory.resolve(fileName),
+ typeReference);
+ storage.load();
+ return new AccountAwareStorageBackend<>(storage, null, ACCOUNT);
+ });
+ }
+
+ @Override
+ public void flushAll() {
+ stores.values().forEach(StorageBackend::flush);
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java
new file mode 100644
index 0000000000..10ef6e45a7
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java
@@ -0,0 +1,629 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.PaginatedResult;
+import io.github.hectorvent.floci.services.lambda.durable.DurableExecutionService.CheckpointResult;
+import io.github.hectorvent.floci.services.lambda.durable.DurableExecutionService.ListRequest;
+import io.github.hectorvent.floci.services.lambda.durable.DurableExecutionService.StartRequest;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableErrorObject;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecution;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableExecutionStatus;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableHistoryEvent;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperation;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationAction;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationStatus;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationType;
+import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationUpdate;
+import io.github.hectorvent.floci.testing.MutableClock;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Drives the engine with a scripted stand-in for the function. The script receives the invocation
+ * event, checkpoints through the service like the SDK would, and returns the handler response.
+ * An inline executor and a zero retry delay make every invocation chain run on the test thread.
+ */
+class DurableExecutionServiceTest {
+
+ private static final String ACCOUNT = "000000000000";
+ private static final String REGION = "us-east-1";
+ private static final String FUNCTION = "durable-fn";
+ private static final String FUNCTION_ARN = "arn:aws:lambda:us-east-1:000000000000:function:durable-fn:1";
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private final MutableClock clock = new MutableClock();
+ private InMemoryStorageFactory storage;
+ private ScriptedInvoker invoker;
+ private DurableExecutionService service;
+
+ @BeforeEach
+ void setUp() {
+ storage = new InMemoryStorageFactory(ACCOUNT);
+ invoker = new ScriptedInvoker();
+ service = newService(storage, invoker, clock);
+ }
+
+ @Test
+ void startInvokesTheFunctionWithTheExecutionOperationAndAToken() {
+ invoker.script(event -> succeeded("\"done\""));
+
+ DurableExecution execution = start("exec-1", "{\"a\":1}", true);
+
+ JsonNode event = invoker.events.get(0);
+ assertEquals(execution.getExecutionArn(), event.get("DurableExecutionArn").asText());
+ assertTrue(execution.getExecutionArn().startsWith(FUNCTION_ARN + "/durable-execution/exec-1/"));
+ assertFalse(event.get("CheckpointToken").asText().isEmpty());
+ assertEquals(execution.getExecutionId(), event.get("UpdatedOperationIds").get(0).asText());
+ JsonNode root = event.get("InitialExecutionState").get("Operations").get(0);
+ assertEquals(execution.getExecutionId(), root.get("Id").asText());
+ assertEquals("EXECUTION", root.get("Type").asText());
+ assertEquals("STARTED", root.get("Status").asText());
+ assertEquals("{\"a\":1}", root.get("ExecutionDetails").get("InputPayload").asText());
+ assertEquals("", event.get("InitialExecutionState").get("NextMarker").asText());
+ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(execution.getExecutionArn()).getStatus());
+ assertEquals("\"done\"", service.get(execution.getExecutionArn()).getResult());
+ }
+
+ @Test
+ void stepCheckpointIsReturnedAsNewStateAndTheResultClosesTheExecution() {
+ invoker.script(event -> {
+ CheckpointResult first = checkpoint(event, token(event),
+ List.of(step("s1", DurableOperationAction.START, null, null), step("s1",
+ DurableOperationAction.SUCCEED, "42", null)));
+ assertEquals(1, first.newExecutionState().size());
+ assertEquals(DurableOperationStatus.SUCCEEDED, first.newExecutionState().get(0).getStatus());
+ assertEquals(1, first.newExecutionState().get(0).getAttempt());
+ assertNotNull(first.checkpointToken());
+ CheckpointResult poll = checkpoint(event, first.checkpointToken(), List.of());
+ assertTrue(poll.newExecutionState().isEmpty(), "nothing changed since the last response");
+ return succeeded("\"ok\"");
+ });
+
+ DurableExecution execution = service.get(start("exec-1", "{}", true).getExecutionArn());
+
+ assertEquals(DurableExecutionStatus.SUCCEEDED, execution.getStatus());
+ assertEquals("\"ok\"", execution.getResult());
+ assertEquals(List.of("ExecutionStarted", "StepStarted", "StepSucceeded", "InvocationCompleted",
+ "ExecutionSucceeded"), eventTypes(execution));
+ assertNull(execution.getCurrentInvocationId());
+ }
+
+ @Test
+ void waitCompletesWhenTheSweepReachesItsDeadlineAndTheFunctionIsReinvoked() {
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(waitStart("w1", 5)));
+ return pending();
+ });
+ invoker.script(event -> {
+ JsonNode wait = operation(event, "w1");
+ assertEquals("SUCCEEDED", wait.get("Status").asText());
+ assertEquals("w1", event.get("UpdatedOperationIds").get(0).asText());
+ assertEquals(1, event.get("UpdatedOperationIds").size());
+ return succeeded("\"after wait\"");
+ });
+
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+ assertEquals(DurableExecutionStatus.RUNNING, service.get(arn).getStatus());
+ service.sweep();
+ assertEquals(1, invoker.events.size(), "the wait is not due yet");
+
+ clock.advance(Duration.ofSeconds(5));
+ service.sweep();
+
+ assertEquals(2, invoker.events.size());
+ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(arn).getStatus());
+ assertTrue(eventTypes(service.get(arn)).containsAll(List.of("WaitStarted", "WaitSucceeded")));
+ }
+
+ @Test
+ void stepRetryBecomesReadyAfterItsDelay() {
+ invoker.script(event -> {
+ CheckpointResult result = checkpoint(event, token(event), List.of(
+ step("s1", DurableOperationAction.START, null, null),
+ stepRetry("s1", 3, DurableErrorObject.of("boom", "Error"))));
+ assertEquals(DurableOperationStatus.PENDING, result.newExecutionState().get(0).getStatus());
+ assertNotNull(result.newExecutionState().get(0).getNextAttemptTimestamp());
+ return pending();
+ });
+ invoker.script(event -> {
+ assertEquals("READY", operation(event, "s1").get("Status").asText());
+ assertFalse(operation(event, "s1").get("StepDetails").has("NextAttemptTimestamp"));
+ checkpoint(event, token(event), List.of(step("s1", DurableOperationAction.START, null, null),
+ step("s1", DurableOperationAction.SUCCEED, "ok", null)));
+ return succeeded("\"done\"");
+ });
+
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+ clock.advance(Duration.ofSeconds(3));
+ service.sweep();
+
+ DurableExecution execution = service.get(arn);
+ assertEquals(DurableExecutionStatus.SUCCEEDED, execution.getStatus());
+ assertEquals(2, execution.getOperations().get("s1").getAttempt());
+ assertTrue(eventTypes(execution).contains("StepFailed"), "a RETRY with an error records StepFailed");
+ }
+
+ @Test
+ void emptyCheckpointCompletesADueWaitInsideTheInvocation() {
+ invoker.script(event -> {
+ String token = checkpoint(event, token(event), List.of(waitStart("w1", 1))).checkpointToken();
+ clock.advance(Duration.ofSeconds(1));
+ CheckpointResult poll = checkpoint(event, token, List.of());
+ assertEquals(1, poll.newExecutionState().size());
+ assertEquals(DurableOperationStatus.SUCCEEDED, poll.newExecutionState().get(0).getStatus());
+ return succeeded("\"x\"");
+ });
+
+ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(start("exec-1", "{}", true).getExecutionArn())
+ .getStatus());
+ }
+
+ @Test
+ void staleTokensAndTokensOfClosedExecutionsAreRejected() {
+ invoker.script(event -> {
+ String first = token(event);
+ String second = checkpoint(event, first, List.of()).checkpointToken();
+ AwsException stale = assertThrows(AwsException.class, () -> checkpoint(event, first, List.of()));
+ assertEquals("Invalid checkpoint token", stale.getMessage());
+ assertEquals(400, stale.getHttpStatus());
+ assertThrows(AwsException.class, () -> checkpoint(event, "QUJDRA==", List.of()));
+ String arn = event.get("DurableExecutionArn").asText();
+ String otherArn = arn.substring(0, arn.lastIndexOf('/') + 1) + "other-id";
+ AwsException other = assertThrows(AwsException.class,
+ () -> service.checkpoint(otherArn, second, null, List.of()));
+ assertEquals("Checkpoint token is not valid for the durable execution ARN", other.getMessage());
+ CheckpointResult closing = checkpoint(event, second, List.of(executionSucceed("\"r\"")));
+ assertNull(closing.checkpointToken(), "the closing checkpoint carries no token");
+ assertThrows(AwsException.class, () -> checkpoint(event, second, List.of()));
+ return succeeded("");
+ });
+
+ DurableExecution execution = service.get(start("exec-1", "{}", true).getExecutionArn());
+ assertEquals(DurableExecutionStatus.SUCCEEDED, execution.getStatus());
+ assertEquals("\"r\"", execution.getResult(), "the checkpointed result wins over the handler return");
+ }
+
+ @Test
+ void aRetriedCheckpointWithTheSameClientTokenGetsTheSameAnswer() {
+ invoker.script(event -> {
+ String arn = event.get("DurableExecutionArn").asText();
+ String first = token(event);
+ List<DurableOperationUpdate> updates = List.of(waitStart("w1", 1));
+ CheckpointResult accepted = service.checkpoint(arn, first, "ct-1", updates);
+ clock.advance(Duration.ofSeconds(2));
+ service.sweep();
+ CheckpointResult retried = service.checkpoint(arn, first, "ct-1", List.of());
+ assertEquals(accepted.checkpointToken(), retried.checkpointToken());
+ assertEquals(DurableOperationStatus.STARTED, retried.newExecutionState().get(0).getStatus(),
+ "the retry gets the answer as it was sent, not the fired wait");
+ assertThrows(AwsException.class, () -> service.checkpoint(arn, first, "ct-2", updates));
+ CheckpointResult next = service.checkpoint(arn, accepted.checkpointToken(), "ct-3", List.of());
+ assertEquals(DurableOperationStatus.SUCCEEDED, next.newExecutionState().get(0).getStatus());
+ assertThrows(AwsException.class, () -> service.checkpoint(arn, first, "ct-1", updates));
+ return succeeded("");
+ });
+
+ DurableExecution execution = service.get(start("exec-1", "{}", true).getExecutionArn());
+ assertEquals(List.of("ExecutionStarted", "WaitStarted", "WaitSucceeded", "InvocationCompleted",
+ "ExecutionSucceeded"), eventTypes(execution));
+ }
+
+ @Test
+ void sameNameIsIdempotentForTheSamePayloadAndRejectedForAnotherPayload() {
+ invoker.script(event -> succeeded("\"one\""));
+
+ DurableExecution first = start("same", "{\"a\":1}", true);
+ DurableExecution again = start("same", "{\"a\":1}", true);
+ assertEquals(first.getExecutionArn(), again.getExecutionArn());
+ assertEquals(1, invoker.events.size());
+
+ AwsException conflict = assertThrows(AwsException.class, () -> start("same", "{\"a\":2}", true));
+ assertEquals("DurableExecutionAlreadyStartedException", conflict.getErrorCode());
+ assertEquals("Execution already started: " + first.getExecutionArn(), conflict.getMessage());
+ assertEquals(409, conflict.getHttpStatus());
+
+ AwsException otherVersion = assertThrows(AwsException.class, () -> service.start(
+ new StartRequest(ACCOUNT, REGION, FUNCTION, "2", "same", "{\"a\":1}", true)));
+ assertEquals("Execution already started: " + first.getExecutionArn(), otherVersion.getMessage());
+ }
+
+ @Test
+ void handlerResponsesThatBreakTheProtocolFailTheExecution() {
+ invoker.script(event -> handlerResponse("{\"foo\":1}"));
+ DurableExecution invalid = service.get(start("invalid", "{}", true).getExecutionArn());
+ assertEquals(DurableExecutionStatus.FAILED, invalid.getStatus());
+ assertEquals("Invalid Status in invocation output.", invalid.getError().getErrorMessage());
+ assertEquals("InvalidParameterValueException", invalid.getError().getErrorType());
+ assertEquals(List.of("ExecutionStarted", "InvocationCompleted", "ExecutionFailed"), eventTypes(invalid));
+
+ invoker.script(event -> pending());
+ DurableExecution idle = service.get(start("idle", "{}", true).getExecutionArn());
+ assertEquals(DurableExecutionStatus.FAILED, idle.getStatus());
+ assertEquals("Cannot return PENDING status with no pending operations.", idle.getError().getErrorMessage());
+
+ invoker.script(event -> handlerResponse("{\"Status\":\"FAILED\",\"Error\":{\"ErrorMessage\":\"bad\","
+ + "\"ErrorType\":\"MyError\"}}"));
+ DurableExecution failed = service.get(start("failed", "{}", true).getExecutionArn());
+ assertEquals(DurableExecutionStatus.FAILED, failed.getStatus());
+ assertEquals("MyError", failed.getError().getErrorType());
+ }
+
+ @Test
+ void aFunctionErrorIsRetriedAndFailsTheExecutionAfterTheLastAttempt() {
+ for (int i = 0; i < DurableExecutionService.INVOCATION_RETRY_MAX_ATTEMPTS; i++) {
+ invoker.script(event -> functionError("{\"errorMessage\":\"crash\",\"errorType\":\"RuntimeError\"}"));
+ }
+
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+ for (int i = 1; i < DurableExecutionService.INVOCATION_RETRY_MAX_ATTEMPTS; i++) {
+ assertEquals(DurableExecutionStatus.RUNNING, service.get(arn).getStatus());
+ service.sweep();
+ }
+
+ DurableExecution execution = service.get(arn);
+ assertEquals(DurableExecutionStatus.FAILED, execution.getStatus());
+ assertEquals("crash", execution.getError().getErrorMessage());
+ assertEquals("RuntimeError", execution.getError().getErrorType());
+ assertEquals(DurableExecutionService.INVOCATION_RETRY_MAX_ATTEMPTS,
+ eventTypes(execution).stream().filter("InvocationCompleted"::equals).count());
+ }
+
+ @Test
+ void stopClosesTheExecutionAndIsIdempotent() {
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(waitStart("w1", 60)));
+ return pending();
+ });
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+ String token = token(invoker.events.get(0));
+
+ DurableExecution stopped = service.stop(arn, null);
+ assertEquals(DurableExecutionStatus.STOPPED, stopped.getStatus());
+ assertNotNull(stopped.getError(), "a stop without a body records an empty error object");
+ assertNull(stopped.getError().getErrorMessage());
+ assertEquals(stopped.getEndTimestamp(), service.stop(arn, DurableErrorObject.of("x", "y")).getEndTimestamp());
+ assertNull(service.get(arn).getError().getErrorType(), "a second stop does not overwrite the first");
+ assertThrows(AwsException.class, () -> service.checkpoint(arn, token, null, List.of()));
+ assertEquals("ExecutionStopped", eventTypes(service.get(arn)).getLast());
+
+ clock.advance(Duration.ofSeconds(60));
+ service.sweep();
+ assertEquals(1, invoker.events.size(), "a stopped execution is never re-invoked");
+ }
+
+ @Test
+ void executionTimeoutMarksTheExecutionTimedOut() {
+ invoker.executionTimeoutSeconds = 10;
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(waitStart("w1", 60)));
+ return pending();
+ });
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+
+ clock.advance(Duration.ofSeconds(10));
+ service.sweep();
+
+ DurableExecution execution = service.get(arn);
+ assertEquals(DurableExecutionStatus.TIMED_OUT, execution.getStatus());
+ assertEquals("Execution timed out after 10 seconds.", execution.getError().getErrorMessage());
+ assertEquals("ExecutionTimedOut", eventTypes(execution).getLast());
+ }
+
+ @Test
+ void listFiltersAndPagesNewestFirst() {
+ invoker.script(event -> succeeded("1"));
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(waitStart("w1", 60)));
+ return pending();
+ });
+ invoker.script(event -> succeeded("3"));
+ String first = start("a", "{}", true).getExecutionArn();
+ String second = start("b", "{}", false).getExecutionArn();
+ String third = start("c", "{}", true).getExecutionArn();
+
+ PaginatedResult<DurableExecution> all = service.list(listRequest(null, null, null, null));
+ assertEquals(List.of(third, second, first), all.items().stream().map(DurableExecution::getExecutionArn).toList());
+ assertNull(all.nextToken());
+
+ PaginatedResult<DurableExecution> running = service.list(listRequest(Set.of(DurableExecutionStatus.RUNNING),
+ null, null, null));
+ assertEquals(List.of(second), running.items().stream().map(DurableExecution::getExecutionArn).toList());
+
+ PaginatedResult<DurableExecution> byName = service.list(listRequest(null, "c", null, null));
+ assertEquals(List.of(third), byName.items().stream().map(DurableExecution::getExecutionArn).toList());
+
+ PaginatedResult<DurableExecution> page = service.list(listRequest(null, null, 2, null));
+ assertEquals(2, page.items().size());
+ assertNotNull(page.nextToken());
+ PaginatedResult<DurableExecution> rest = service.list(listRequest(null, null, 2, page.nextToken()));
+ assertEquals(List.of(first), rest.items().stream().map(DurableExecution::getExecutionArn).toList());
+ assertNull(rest.nextToken());
+ }
+
+ @Test
+ void anEmptyStatusFilterMatchesNothingButStillChecksTheMarker() {
+ invoker.script(event -> succeeded("1"));
+ start("a", "{}", true);
+
+ assertTrue(service.list(listRequest(Set.of(), null, null, null)).items().isEmpty());
+ assertThrows(AwsException.class, () -> service.list(listRequest(Set.of(), null, null, "bogus")));
+ }
+
+ @Test
+ void historyPagesInBothDirections() {
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(step("s1", DurableOperationAction.START, null, null),
+ step("s1", DurableOperationAction.SUCCEED, "1", null)));
+ return succeeded("\"x\"");
+ });
+ String arn = start("exec-1", "{}", true).getExecutionArn();
+
+ PaginatedResult<DurableHistoryEvent> firstPage = service.history(arn, 2, null, false);
+ assertEquals(List.of(1L, 2L), firstPage.items().stream().map(DurableHistoryEvent::getEventId).toList());
+ PaginatedResult<DurableHistoryEvent> secondPage = service.history(arn, 2, firstPage.nextToken(), false);
+ assertEquals(List.of(3L, 4L), secondPage.items().stream().map(DurableHistoryEvent::getEventId).toList());
+ PaginatedResult<DurableHistoryEvent> newestFirst = service.history(arn, 1, null, true);
+ assertEquals("ExecutionSucceeded", newestFirst.items().get(0).getEventType());
+ assertThrows(AwsException.class, () -> service.history(arn, 2, "bogus", false));
+ }
+
+ @Test
+ void aClosedExecutionIsDeletedWhenItsRetentionExpires() {
+ invoker.retentionPeriodInDays = 1;
+ invoker.script(event -> succeeded("1"));
+ String arn = start("exec-1", "{}", true).getExecutionArn();
+
+ clock.advance(Duration.ofHours(23));
+ service.sweep();
+ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(arn).getStatus());
+
+ clock.advance(Duration.ofHours(2));
+ service.sweep();
+ AwsException gone = assertThrows(AwsException.class, () -> service.get(arn));
+ assertEquals(404, gone.getHttpStatus());
+ assertEquals("Durable Execution does not exist", gone.getMessage());
+ }
+
+ @Test
+ void awaitCompletionResolvesWhenTheExecutionCloses() throws Exception {
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(waitStart("w1", 2)));
+ return pending();
+ });
+ invoker.script(event -> succeeded("\"late\""));
+ String arn = start("exec-1", "{}", true).getExecutionArn();
+ assertFalse(service.awaitCompletion(arn).isDone());
+
+ clock.advance(Duration.ofSeconds(2));
+ service.sweep();
+
+ assertEquals("\"late\"", service.awaitCompletion(arn).get(5, TimeUnit.SECONDS).getResult());
+ }
+
+ @Test
+ void aWaitThatFiresWhileTheHandlerRunsResumesInsteadOfFailing() {
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(waitStart("w1", 1)));
+ return pending();
+ });
+ invoker.script(event -> {
+ clock.advance(Duration.ofSeconds(1));
+ service.sweep();
+ return pending();
+ });
+ invoker.script(event -> {
+ assertEquals("SUCCEEDED", operation(event, "w1").get("Status").asText());
+ return succeeded("\"resumed\"");
+ });
+
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+ service.recoverAfterRestart();
+
+ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(arn).getStatus());
+ assertEquals(3, invoker.events.size());
+ }
+
+ @Test
+ void executionNamesAreScopedToTheFunction() {
+ invoker.script(event -> succeeded("\"a\""));
+ invoker.script(event -> succeeded("\"b\""));
+ DurableExecution first = start("shared", "{}", true);
+
+ invoker.functionName = "other-fn";
+ DurableExecution second = service.start(new StartRequest(ACCOUNT, REGION, "other-fn", "1", "shared", "{\"x\":1}",
+ true));
+
+ assertFalse(first.getExecutionArn().equals(second.getExecutionArn()));
+ assertEquals("\"b\"", service.get(second.getExecutionArn()).getResult());
+ }
+
+ @Test
+ void aWakeUpDuringAFailedInvocationDoesNotAddAnInvocationAfterTheRetry() {
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(waitStart("w1", 1)));
+ return pending();
+ });
+ invoker.script(event -> {
+ clock.advance(Duration.ofSeconds(1));
+ service.sweep();
+ return functionError("{\"errorMessage\":\"crash\",\"errorType\":\"E\"}");
+ });
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(waitStart("w2", 60)));
+ return pending();
+ });
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+ service.recoverAfterRestart();
+
+ service.sweep();
+
+ assertEquals(3, invoker.events.size(), "the retry is the only invocation after the crash");
+ assertEquals(DurableExecutionStatus.RUNNING, service.get(arn).getStatus());
+ }
+
+ @Test
+ void aRestartKeepsTheBackoffOfAPendingRetry() {
+ invoker.script(event -> functionError("{\"errorMessage\":\"crash\",\"errorType\":\"E\"}"));
+ invoker.script(event -> succeeded("\"ok\""));
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+
+ newService(storage, invoker, clock).recoverAfterRestart();
+ assertEquals(1, invoker.events.size(), "the retry waits for its backoff");
+
+ clock.advance(Duration.ofSeconds(1));
+ service.sweep();
+ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(arn).getStatus());
+ }
+
+ @Test
+ void aNewInvocationClearsAPendingRetry() {
+ invoker.script(event -> functionError("{\"errorMessage\":\"crash\",\"errorType\":\"E\"}"));
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(waitStart("w1", 60)));
+ return pending();
+ });
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+ assertNotNull(service.get(arn).getNextInvocationAttemptAt());
+
+ service.sweep();
+
+ assertNull(service.get(arn).getNextInvocationAttemptAt());
+ service.sweep();
+ assertEquals(2, invoker.events.size(), "no extra invocation after the retry ran");
+ }
+
+ // ──────────────────────────── helpers ────────────────────────────
+
+ private static DurableExecutionService newService(InMemoryStorageFactory storage, ScriptedInvoker invoker,
+ MutableClock clock) {
+ return new DurableExecutionService(storage, MAPPER, clock, invoker, Runnable::run, Duration.ZERO);
+ }
+
+ private DurableExecution start(String name, String payload, boolean synchronous) {
+ return service.start(new StartRequest(ACCOUNT, REGION, FUNCTION, "1", name, payload, synchronous));
+ }
+
+ private CheckpointResult checkpoint(JsonNode event, String token, List<DurableOperationUpdate> updates) {
+ return service.checkpoint(event.get("DurableExecutionArn").asText(), token, null, updates);
+ }
+
+ private static ListRequest listRequest(Set<DurableExecutionStatus> statuses, String name, Integer maxItems,
+ String marker) {
+ return new ListRequest(ACCOUNT, REGION, FUNCTION, null, name, statuses, null, null, false, maxItems, marker);
+ }
+
+ private static String token(JsonNode event) {
+ return event.get("CheckpointToken").asText();
+ }
+
+ private static JsonNode operation(JsonNode event, String id) {
+ for (JsonNode operation : event.get("InitialExecutionState").get("Operations")) {
+ if (id.equals(operation.get("Id").asText())) {
+ return operation;
+ }
+ }
+ throw new AssertionError("no operation " + id + " in " + event);
+ }
+
+ private static List<String> eventTypes(DurableExecution execution) {
+ return execution.getHistory().stream().map(DurableHistoryEvent::getEventType).toList();
+ }
+
+ private static DurableOperationUpdate step(String id, DurableOperationAction action, String payload,
+ DurableErrorObject error) {
+ return new DurableOperationUpdate(id, null, null, DurableOperationType.STEP, "Step", action, payload, error,
+ null, null, null);
+ }
+
+ private static DurableOperationUpdate stepRetry(String id, int delaySeconds, DurableErrorObject error) {
+ return new DurableOperationUpdate(id, null, null, DurableOperationType.STEP, "Step",
+ DurableOperationAction.RETRY, null, error, delaySeconds, null, null);
+ }
+
+ private static DurableOperationUpdate waitStart(String id, int seconds) {
+ return new DurableOperationUpdate(id, null, null, DurableOperationType.WAIT, "Wait",
+ DurableOperationAction.START, null, null, null, seconds, null);
+ }
+
+ private static DurableOperationUpdate executionSucceed(String payload) {
+ return new DurableOperationUpdate("execution-result", null, null, DurableOperationType.EXECUTION, null,
+ DurableOperationAction.SUCCEED, payload, null, null, null, null);
+ }
+
+ private static DurableFunctionInvoker.DurableInvocationResult succeeded(String result) {
+ return handlerResponse("{\"Status\":\"SUCCEEDED\",\"Result\":" + MAPPER.valueToTree(result) + "}");
+ }
+
+ private static DurableFunctionInvoker.DurableInvocationResult pending() {
+ return handlerResponse("{\"Status\":\"PENDING\"}");
+ }
+
+ private static DurableFunctionInvoker.DurableInvocationResult handlerResponse(String json) {
+ return new DurableFunctionInvoker.DurableInvocationResult("req", json.getBytes(StandardCharsets.UTF_8), null);
+ }
+
+ private static DurableFunctionInvoker.DurableInvocationResult functionError(String json) {
+ return new DurableFunctionInvoker.DurableInvocationResult("req", json.getBytes(StandardCharsets.UTF_8),
+ "Unhandled");
+ }
+
+ /** Plays one script per invocation; a missing script fails the execution so the test shows it. */
+ private static final class ScriptedInvoker implements DurableFunctionInvoker {
+
+ final Deque<Function<JsonNode, DurableInvocationResult>> scripts = new ArrayDeque<>();
+ final List<JsonNode> events = new ArrayList<>();
+ int executionTimeoutSeconds = 3600;
+ String functionName = FUNCTION;
+ int retentionPeriodInDays = 7;
+
+ void script(Function<JsonNode, DurableInvocationResult> script) {
+ scripts.add(script);
+ }
+
+ @Override
+ public ResolvedDurableTarget resolve(String accountId, String region, String functionName, String qualifier) {
+ String version = qualifier == null ? "1" : qualifier;
+ return new ResolvedDurableTarget(accountId, region, this.functionName,
+ "arn:aws:lambda:us-east-1:000000000000:function:" + this.functionName + ":" + version, version, true,
+ executionTimeoutSeconds, retentionPeriodInDays);
+ }
+
+ @Override
+ public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {
+ JsonNode event;
+ try {
+ event = MAPPER.readTree(payload);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ events.add(event);
+ Function<JsonNode, DurableInvocationResult> script = scripts.poll();
+ if (script == null) {
+ return handlerResponse("{\"Status\":\"FAILED\",\"Error\":{\"ErrorMessage\":\"no script for "
+ + "invocation " + events.size() + "\",\"ErrorType\":\"TestError\"}}");
+ }
+ return script.apply(event);
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionSweeperTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionSweeperTest.java
new file mode 100644
index 0000000000..3651bc8c5f
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionSweeperTest.java
@@ -0,0 +1,50 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.config.EmulatorConfig.LambdaServiceConfig;
+import io.github.hectorvent.floci.config.EmulatorConfig.ServicesConfig;
+import io.quarkus.runtime.ShutdownEvent;
+import io.quarkus.runtime.StartupEvent;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+/** A non-positive interval must not reach scheduleWithFixedDelay, which would abort startup. */
+class DurableExecutionSweeperTest {
+
+ @Test
+ void nonPositiveConfiguredIntervalDoesNotPreventStartup() {
+ for (long interval : new long[] {0L, -1L, Long.MIN_VALUE}) {
+ DurableExecutionSweeper sweeper = new DurableExecutionSweeper(null, configWithInterval(interval));
+ assertDoesNotThrow(() -> sweeper.onStart(new StartupEvent()), "interval " + interval + " must not abort startup");
+ sweeper.onStop(new ShutdownEvent());
+ }
+ }
+
+ @Test
+ void positiveConfiguredIntervalStartsNormally() {
+ DurableExecutionSweeper sweeper = new DurableExecutionSweeper(null, configWithInterval(30L));
+ assertDoesNotThrow(() -> sweeper.onStart(new StartupEvent()));
+ sweeper.onStop(new ShutdownEvent());
+ }
+
+ private static EmulatorConfig configWithInterval(long intervalSeconds) {
+ EmulatorConfig config = configView(EmulatorConfig.class);
+ ServicesConfig services = configView(ServicesConfig.class);
+ LambdaServiceConfig lambda = configView(LambdaServiceConfig.class);
+ doReturn(services).when(config).services();
+ doReturn(lambda).when(services).lambda();
+ doReturn(true).when(lambda).enabled();
+ doReturn(true).when(lambda).durableSweepEnabled();
+ doReturn(intervalSeconds).when(lambda).durableSweepIntervalSeconds();
+ return config;
+ }
+
+ private static <T> T configView(Class<T> configType) {
+ return mock(configType, invocation -> {
+ throw new AssertionError("Unexpected configuration access: " + invocation.getMethod());
+ });
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/InMemoryStorageFactory.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/InMemoryStorageFactory.java
new file mode 100644
index 0000000000..0dadfbb2a4
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/InMemoryStorageFactory.java
@@ -0,0 +1,37 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.core.storage.WriteProfile;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/** In-memory stores shared by every service built on the same factory, so a second service sees a "restart". */
+final class InMemoryStorageFactory extends StorageFactory {
+
+ private final String accountId;
+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
+
+ InMemoryStorageFactory(String accountId) {
+ super(null, null);
+ this.accountId = accountId;
+ }
+
+ @Override
+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return create(serviceName, fileName, typeReference, WriteProfile.DEFAULT);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+ TypeReference<Map<String, V>> typeReference,
+ WriteProfile profile) {
+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,
+ ignored -> AccountAwareStorageBackend.inMemory(accountId));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableExecutionApiIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableExecutionApiIntegrationTest.java
new file mode 100644
index 0000000000..6c2d3629f0
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableExecutionApiIntegrationTest.java
@@ -0,0 +1,247 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.lambda.model.InvocationType;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.specification.RequestSpecification;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.startsWith;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * The durable execution routes and the Invoke rules that need no running function: routing of
+ * raw and percent-encoded ARNs, request validation, and the errors AWS returns for them.
+ */
+@QuarkusTest
+class LambdaDurableExecutionApiIntegrationTest {
+
+ @Inject
+ LambdaService lambdaService;
+
+ private static final String LAMBDA = "/2015-03-31";
+ private static final String DURABLE = "/2025-12-01";
+ private static final String FUNCTION_ARN = "arn:aws:lambda:us-east-1:000000000000:function:api-durable-fn";
+ private static final String UNKNOWN_EXECUTION = FUNCTION_ARN
+ + ":1/durable-execution/missing/00000000-0000-0000-0000-000000000000";
+ private static final String ENCODED_UNKNOWN_EXECUTION = "arn%3Aaws%3Alambda%3Aus-east-1%3A000000000000%3A"
+ + "function%3Aapi-durable-fn%3A%24LATEST%2Fdurable-execution%2Fmissing%2F"
+ + "00000000-0000-0000-0000-000000000000";
+
+ @BeforeEach
+ void createFunctions() {
+ createFunction("api-durable-fn", ", \"DurableConfig\": {\"ExecutionTimeout\": 3600}");
+ createFunction("api-long-durable-fn", ", \"DurableConfig\": {\"ExecutionTimeout\": 901}");
+ createFunction("api-plain-fn", "");
+ }
+
+ /** Idempotent: the functions stay for the whole class, so a repeat create answers 409. */
+ private static void createFunction(String name, String members) {
+ given()
+ .contentType("application/json")
+ .body("""
+ {
+ "FunctionName": "%s",
+ "Runtime": "nodejs20.x",
+ "Role": "arn:aws:iam::000000000000:role/lambda-role",
+ "Handler": "index.handler"%s
+ }
+ """.formatted(name, members))
+ .when()
+ .post(LAMBDA + "/functions")
+ .then()
+ .statusCode(anyOf(is(201), is(409)));
+ }
+
+ /** RestAssured must not re-encode the path: both the raw and the SDK's percent-encoded form go on the wire as is. */
+ private static RequestSpecification raw() {
+ return given().urlEncodingEnabled(false);
+ }
+
+ @Test
+ void rawAndEncodedExecutionArnsReachTheSameRoute() {
+ raw().get(DURABLE + "/durable-executions/" + UNKNOWN_EXECUTION)
+ .then()
+ .statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"))
+ .body("message", equalTo("Durable Execution does not exist"));
+
+ raw().get(DURABLE + "/durable-executions/" + ENCODED_UNKNOWN_EXECUTION)
+ .then()
+ .statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"));
+
+ raw().get(DURABLE + "/durable-executions/" + ENCODED_UNKNOWN_EXECUTION + "/history")
+ .then()
+ .statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"));
+
+ raw().get(DURABLE + "/durable-executions/" + ENCODED_UNKNOWN_EXECUTION + "/history?MaxItems=-1")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", equalTo("1 validation error detected: Value '-1' at 'maxItems' failed to satisfy "
+ + "constraint: Member must have value greater than or equal to 0"));
+
+ raw().contentType("application/json").body("{\"CheckpointToken\": \"QUJD\", \"Updates\": []}")
+ .post(DURABLE + "/durable-executions/" + UNKNOWN_EXECUTION + "/checkpoint")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidParameterValueException"))
+ .body("message", equalTo("Invalid checkpoint token"));
+ }
+
+ @Test
+ void aMalformedExecutionArnIsAValidationError() {
+ raw().get(DURABLE + "/durable-executions/" + FUNCTION_ARN)
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", startsWith("1 validation error detected: Value '" + FUNCTION_ARN
+ + "' at 'durableExecutionArn' failed to satisfy constraint: Member must satisfy regular "
+ + "expression pattern: arn:([a-zA-Z0-9-]+):lambda:"));
+ }
+
+ @Test
+ void aDurableFunctionNeedsAQualifiedReference() {
+ for (String invocationType : new String[] {"RequestResponse", "DryRun"}) {
+ given()
+ .header("X-Amz-Invocation-Type", invocationType)
+ .body("{}")
+ .when()
+ .post(LAMBDA + "/functions/api-durable-fn/invocations")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidParameterValueException"))
+ .body("message", equalTo("You cannot invoke a durable function using an unqualified ARN."));
+ }
+
+ given()
+ .header("X-Amz-Invocation-Type", "DryRun")
+ .body("{}")
+ .when()
+ .post(LAMBDA + "/functions/api-durable-fn/invocations?Qualifier=$LATEST")
+ .then()
+ .statusCode(204);
+ }
+
+ @Test
+ void anArnInvocationOfADurableFunctionNeedsAQualifierToo() {
+ AwsException rejected = assertThrows(AwsException.class, () -> lambdaService.invokeArn(
+ FUNCTION_ARN, "{}".getBytes(StandardCharsets.UTF_8), InvocationType.Event));
+ assertEquals("InvalidParameterValueException", rejected.getErrorCode());
+ assertEquals("You cannot invoke a durable function using an unqualified ARN.", rejected.getMessage());
+ }
+
+ @Test
+ void aSynchronousInvokeNeedsAnExecutionTimeoutWithinTheInvocationLimit() {
+ given()
+ .body("{}")
+ .when()
+ .post(LAMBDA + "/functions/api-long-durable-fn:$LATEST/invocations")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidParameterValueException"))
+ .body("message", equalTo("You cannot synchronously invoke a durable function with an executionTimeout "
+ + "greater than 15 minutes."));
+ }
+
+ @Test
+ void anInvalidExecutionNameIsRejected() {
+ given()
+ .header("X-Amz-Invocation-Type", "Event")
+ .header("X-Amz-Durable-Execution-Name", "has space")
+ .body("{}")
+ .when()
+ .post(LAMBDA + "/functions/api-durable-fn:$LATEST/invocations")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", containsString("'durableExecutionName'"));
+ }
+
+ @Test
+ void listValidatesItsParametersAndTheFunction() {
+ given()
+ .when()
+ .get(DURABLE + "/functions/api-durable-fn/durable-executions")
+ .then()
+ .statusCode(200)
+ .body("DurableExecutions", empty());
+
+ given()
+ .when()
+ .get(DURABLE + "/functions/api-durable-fn/durable-executions?Statuses=PAUSED")
+ .then()
+ .statusCode(200)
+ .body("DurableExecutions", empty());
+
+ given()
+ .when()
+ .get(DURABLE + "/functions/api-durable-fn/durable-executions?Statuses=PAUSED&Marker=bogus")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidParameterValueException"))
+ .body("message", equalTo("Invalid Marker"));
+
+ given()
+ .when()
+ .get(DURABLE + "/functions/api-plain-fn/durable-executions?Statuses=BOGUS")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", startsWith("1 validation error detected: Value '[BOGUS]' at 'statuses'"));
+
+ given()
+ .when()
+ .get(DURABLE + "/functions/api-durable-fn/durable-executions?StartedAfter=NaN")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("SerializationException"))
+ .body("message", equalTo("'NaN' can not be converted to Date"));
+
+ given()
+ .when()
+ .get(DURABLE + "/functions/no-such-durable-fn/durable-executions?Statuses=RUNNING&Statuses=FAILED")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidParameterValueException"))
+ .body("message", equalTo("Cannot filter by more than one status"));
+
+ given()
+ .when()
+ .get(DURABLE + "/functions/no-such-durable-fn/durable-executions?Statuses=BOGUS&MaxItems=-1")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", startsWith("2 validation errors detected: Value '-1' at 'maxItems' failed to satisfy "
+ + "constraint: Member must have value greater than or equal to 0; Value '[BOGUS]' at 'statuses'"));
+
+ given()
+ .when()
+ .get(DURABLE + "/functions/api-durable-fn/durable-executions?MaxItems=x")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("SerializationException"))
+ .body("message", equalTo("'x' can not be converted to Integer"));
+
+ given()
+ .when()
+ .get(DURABLE + "/functions/no-such-durable-fn/durable-executions")
+ .then()
+ .statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionsDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionsDockerIntegrationTest.java
new file mode 100644
index 0000000000..cace454765
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableFunctionsDockerIntegrationTest.java
@@ -0,0 +1,111 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import com.github.dockerjava.api.DockerClient;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Runs a real Python function in a container. Its FAILED envelope closes the execution and the
+ * sync Invoke reports it as a function error. The compat suite covers the full checkpoint loop.
+ */
+@QuarkusTest
+class LambdaDurableFunctionsDockerIntegrationTest {
+
+ private static final String LAMBDA = "/2015-03-31";
+ private static final String DURABLE = "/2025-12-01";
+ private static final String FUNCTION = "durable-docker-fn";
+ private static final String ROLE = "arn:aws:iam::000000000000:role/lambda-role";
+
+ @Inject
+ DockerClient dockerClient;
+
+ @BeforeEach
+ void requireDockerAndFunction() throws Exception {
+ Assumptions.assumeTrue(isDockerAvailable(),
+ "Docker daemon must be available for Lambda durable function integration tests");
+ given()
+ .contentType("application/json")
+ .body("""
+ {
+ "FunctionName": "%s",
+ "Runtime": "python3.14",
+ "Role": "%s",
+ "Handler": "lambda_function.handler",
+ "Timeout": 30,
+ "Publish": true,
+ "DurableConfig": {"ExecutionTimeout": 120, "RetentionPeriodInDays": 1},
+ "Code": {"ZipFile": "%s"}
+ }
+ """.formatted(FUNCTION, ROLE, handlerZipBase64()))
+ .when()
+ .post(LAMBDA + "/functions")
+ .then()
+ .statusCode(anyOf(is(201), is(409)));
+ }
+
+ @Test
+ void aFailedExecutionIsReportedAsAFunctionError() {
+ Response response = given()
+ .header("X-Amz-Durable-Execution-Name", "docker-fail")
+ .body("{\"fail\": true}")
+ .post(LAMBDA + "/functions/" + FUNCTION + ":1/invocations");
+
+ assertEquals(200, response.statusCode(), response.asString());
+ assertEquals("Unhandled", response.getHeader("X-Amz-Function-Error"));
+ assertEquals("asked to fail", response.jsonPath().getString("errorMessage"));
+ assertEquals("TestFailure", response.jsonPath().getString("errorType"));
+
+ given()
+ .when()
+ .get(DURABLE + "/functions/" + FUNCTION + "/durable-executions?Qualifier=1&DurableExecutionName=docker-fail")
+ .then()
+ .statusCode(200)
+ .body("DurableExecutions", hasSize(1))
+ .body("DurableExecutions[0].Status", equalTo("FAILED"));
+ }
+
+ static String handlerSource() {
+ return """
+ def handler(event, context):
+ return {"Status": "FAILED",
+ "Error": {"ErrorMessage": "asked to fail", "ErrorType": "TestFailure"}}
+ """;
+ }
+
+ private static String handlerZipBase64() throws Exception {
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(bytes)) {
+ zip.putNextEntry(new ZipEntry("lambda_function.py"));
+ zip.write(handlerSource().getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ return Base64.getEncoder().encodeToString(bytes.toByteArray());
+ }
+
+ private boolean isDockerAvailable() {
+ try {
+ dockerClient.pingCmd().exec();
+ return true;
+ } catch (Exception expected) {
+ // Docker is unreachable, so requireDockerAndFunction() skips the test rather than failing it.
+ return false;
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java
new file mode 100644
index 0000000000..4ab96caaea
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java
@@ -0,0 +1,318 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.services.lambda.durable.DurableFunctionInvoker.DurableInvocationResult;
+import io.github.hectorvent.floci.services.lambda.durable.ScriptedDurableFunctionInvoker.Invocation;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.TestProfile;
+import io.restassured.response.Response;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Plays the Durable Execution SDK over the REST API while a scripted stand-in holds the function
+ * invocation open, so the checkpoint protocol is exercised end to end without a container.
+ */
+@QuarkusTest
+@TestProfile(LambdaDurableScriptedProfile.class)
+class LambdaDurableProtocolIntegrationTest {
+
+ private static final String LAMBDA = "/2015-03-31";
+ private static final String DURABLE = "/2025-12-01";
+ private static final String FUNCTION = "protocol-durable-fn";
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ @Inject
+ ScriptedDurableFunctionInvoker invoker;
+
+ @BeforeEach
+ void createFunctionAndDrainInvocations() {
+ given()
+ .contentType("application/json")
+ .body("""
+ {
+ "FunctionName": "%s",
+ "Runtime": "nodejs20.x",
+ "Role": "arn:aws:iam::000000000000:role/lambda-role",
+ "Handler": "index.handler",
+ "DurableConfig": {"ExecutionTimeout": 300, "RetentionPeriodInDays": 1}
+ }
+ """.formatted(FUNCTION))
+ .when()
+ .post(LAMBDA + "/functions")
+ .then()
+ .statusCode(anyOf(is(201), is(409)));
+ invoker.drain();
+ }
+
+ @Test
+ void aSynchronousInvokeReturnsTheResultOfACheckpointedExecution() throws Exception {
+ CompletableFuture<Response> invoke = CompletableFuture.supplyAsync(() -> given()
+ .header("X-Amz-Durable-Execution-Name", "sync-1")
+ .body("{\"order\": 1}")
+ .post(LAMBDA + "/functions/" + FUNCTION + ":$LATEST/invocations"));
+
+ Invocation invocation = invoker.awaitInvocation(Duration.ofSeconds(30));
+ JsonNode event = MAPPER.readTree(invocation.payload());
+ String arn = event.get("DurableExecutionArn").asText();
+ String token = event.get("CheckpointToken").asText();
+ assertEquals("{\"order\": 1}", event.at("/InitialExecutionState/Operations/0/ExecutionDetails/InputPayload")
+ .asText());
+
+ Response checkpoint = given()
+ .urlEncodingEnabled(false)
+ .contentType("application/json")
+ .body("""
+ {
+ "CheckpointToken": "%s",
+ "Updates": [
+ {"Id": "s1", "Name": "validate", "Type": "STEP", "SubType": "Step", "Action": "START"},
+ {"Id": "s1", "Name": "validate", "Type": "STEP", "SubType": "Step", "Action": "SUCCEED",
+ "Payload": "{\\"ok\\":true}"}
+ ]
+ }
+ """.formatted(token))
+ .post(DURABLE + "/durable-executions/" + arn + "/checkpoint");
+ checkpoint.then()
+ .statusCode(200)
+ .body("CheckpointToken", notNullValue())
+ .body("NewExecutionState.Operations", hasSize(1))
+ .body("NewExecutionState.Operations[0].Id", equalTo("s1"))
+ .body("NewExecutionState.Operations[0].Status", equalTo("SUCCEEDED"))
+ .body("NewExecutionState.Operations[0].StepDetails.Result", equalTo("{\"ok\":true}"))
+ .body("NewExecutionState.Operations[0].StepDetails.Attempt", equalTo(1));
+ String nextToken = checkpoint.jsonPath().getString("CheckpointToken");
+
+ given()
+ .urlEncodingEnabled(false)
+ .when()
+ .get(DURABLE + "/durable-executions/" + arn + "/state?CheckpointToken=" + nextToken)
+ .then()
+ .statusCode(200)
+ .body("Operations", hasSize(2))
+ .body("Operations[0].Type", equalTo("EXECUTION"))
+ .body("Operations[1].Id", equalTo("s1"))
+ .body("NextMarker", equalTo(""));
+
+ given()
+ .urlEncodingEnabled(false)
+ .contentType("application/json")
+ .body("{\"CheckpointToken\": \"" + token + "\", \"Updates\": []}")
+ .when()
+ .post(DURABLE + "/durable-executions/" + arn + "/checkpoint")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidParameterValueException"))
+ .body("message", equalTo("Invalid checkpoint token"));
+
+ invocation.response().complete(handlerResponse("{\"Status\": \"SUCCEEDED\", \"Result\": \"{\\\"total\\\":3}\"}"));
+
+ Response response = invoke.get(30, TimeUnit.SECONDS);
+ assertEquals(200, response.statusCode());
+ assertEquals(arn, response.getHeader("X-Amz-Durable-Execution-Arn"));
+ assertEquals("$LATEST", response.getHeader("X-Amz-Executed-Version"));
+ assertEquals("{\"total\":3}", response.asString());
+
+ given()
+ .urlEncodingEnabled(false)
+ .when()
+ .get(DURABLE + "/durable-executions/" + arn)
+ .then()
+ .statusCode(200)
+ .body("Status", equalTo("SUCCEEDED"))
+ .body("DurableExecutionName", equalTo("sync-1"))
+ .body("Version", equalTo("$LATEST"))
+ .body("InputPayload", equalTo("{\"order\": 1}"))
+ .body("Result", equalTo("{\"total\":3}"))
+ .body("ExecutionDataIncluded", equalTo(true))
+ .body("DurableConfig.ExecutionTimeout", equalTo(300))
+ .body("EndTimestamp", notNullValue());
+
+ given()
+ .urlEncodingEnabled(false)
+ .when()
+ .get(DURABLE + "/durable-executions/" + arn + "?IncludeExecutionData=false")
+ .then()
+ .statusCode(200)
+ .body("ExecutionDataIncluded", equalTo(false))
+ .body("$", not(hasKey("InputPayload")))
+ .body("$", not(hasKey("Result")));
+
+ given()
+ .urlEncodingEnabled(false)
+ .when()
+ .get(DURABLE + "/durable-executions/" + arn + "/history")
+ .then()
+ .statusCode(200)
+ .body("Events.EventType", equalTo(List.of("ExecutionStarted", "StepStarted", "StepSucceeded",
+ "InvocationCompleted", "ExecutionSucceeded")))
+ .body("Events[0].Id", notNullValue())
+ .body("Events[0].Name", equalTo("sync-1"))
+ .body("Events[0].ExecutionStartedDetails.Input.Truncated", equalTo(true))
+ .body("Events[0].ExecutionStartedDetails.Input", not(hasKey("Payload")))
+ .body("Events[2].Id", equalTo("s1"))
+ .body("Events[2].Name", equalTo("validate"))
+ .body("Events[2].SubType", equalTo("Step"))
+ .body("Events[3].InvocationCompletedDetails.RequestId", notNullValue());
+
+ given()
+ .urlEncodingEnabled(false)
+ .when()
+ .get(DURABLE + "/durable-executions/" + arn + "/history?IncludeExecutionData=true&ReverseOrder=true&MaxItems=2")
+ .then()
+ .statusCode(200)
+ .body("Events.EventType", equalTo(List.of("ExecutionSucceeded", "InvocationCompleted")))
+ .body("Events[0].ExecutionSucceededDetails.Result.Payload", equalTo("{\"total\":3}"))
+ .body("Events[0].ExecutionSucceededDetails.Result.Truncated", equalTo(false))
+ .body("NextMarker", notNullValue());
+
+ String encoded = arn.replace(":", "%3A").replace("/", "%2F").replace("$", "%24");
+ given()
+ .urlEncodingEnabled(false)
+ .when()
+ .get(DURABLE + "/durable-executions/" + encoded)
+ .then()
+ .statusCode(200)
+ .body("DurableExecutionArn", equalTo(arn));
+ }
+
+ @Test
+ void anAsynchronousInvokeIsIdempotentByNameAndCanBeStopped() throws Exception {
+ Response accepted = given()
+ .header("X-Amz-Invocation-Type", "Event")
+ .header("X-Amz-Durable-Execution-Name", "async-1")
+ .body("{\"job\": 1}")
+ .post(LAMBDA + "/functions/" + FUNCTION + ":$LATEST/invocations");
+ assertEquals(202, accepted.statusCode());
+ String arn = accepted.getHeader("X-Amz-Durable-Execution-Arn");
+ assertTrue(arn.contains("/durable-execution/async-1/"), arn);
+
+ Invocation invocation = invoker.awaitInvocation(Duration.ofSeconds(30));
+ JsonNode event = MAPPER.readTree(invocation.payload());
+ assertEquals(arn, event.get("DurableExecutionArn").asText());
+ given()
+ .urlEncodingEnabled(false)
+ .contentType("application/json")
+ .body("""
+ {
+ "CheckpointToken": "%s",
+ "Updates": [{"Id": "w1", "Type": "WAIT", "SubType": "Wait", "Action": "START",
+ "WaitOptions": {"WaitSeconds": 600}}]
+ }
+ """.formatted(event.get("CheckpointToken").asText()))
+ .when()
+ .post(DURABLE + "/durable-executions/" + arn + "/checkpoint")
+ .then()
+ .statusCode(200)
+ .body("NewExecutionState.Operations[0].Status", equalTo("STARTED"))
+ .body("NewExecutionState.Operations[0].WaitDetails.ScheduledEndTimestamp", notNullValue());
+ invocation.response().complete(handlerResponse("{\"Status\": \"PENDING\"}"));
+
+ given()
+ .header("X-Amz-Invocation-Type", "Event")
+ .header("X-Amz-Durable-Execution-Name", "async-1")
+ .body("{\"job\": 1}")
+ .when()
+ .post(LAMBDA + "/functions/" + FUNCTION + ":$LATEST/invocations")
+ .then()
+ .statusCode(202)
+ .header("X-Amz-Durable-Execution-Arn", equalTo(arn));
+
+ given()
+ .header("X-Amz-Invocation-Type", "Event")
+ .header("X-Amz-Durable-Execution-Name", "async-1")
+ .body("{\"job\": 2}")
+ .when()
+ .post(LAMBDA + "/functions/" + FUNCTION + ":$LATEST/invocations")
+ .then()
+ .statusCode(409)
+ .body("__type", equalTo("DurableExecutionAlreadyStartedException"))
+ .body("message", equalTo("Execution already started: " + arn));
+
+ given()
+ .when()
+ .get(DURABLE + "/functions/" + FUNCTION + "/durable-executions?Statuses=RUNNING&DurableExecutionName=async-1")
+ .then()
+ .statusCode(200)
+ .body("DurableExecutions", hasSize(1))
+ .body("DurableExecutions[0].DurableExecutionArn", equalTo(arn))
+ .body("DurableExecutions[0].Status", equalTo("RUNNING"))
+ .body("DurableExecutions[0]", not(hasKey("EndTimestamp")));
+
+ given()
+ .urlEncodingEnabled(false)
+ .header("Authorization", "AWS4-HMAC-SHA256 Credential=000000000000/20260101/us-west-2/lambda/"
+ + "aws4_request, SignedHeaders=host, Signature=abc")
+ .contentType("application/json")
+ .body("{}")
+ .when()
+ .post(DURABLE + "/durable-executions/" + arn + "/stop")
+ .then()
+ .statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"))
+ .body("message", equalTo("Function not found"));
+
+ given()
+ .urlEncodingEnabled(false)
+ .header("Authorization", "AWS4-HMAC-SHA256 Credential=111111111111/20260101/us-east-1/lambda/"
+ + "aws4_request, SignedHeaders=host, Signature=abc")
+ .contentType("application/json")
+ .body("{}")
+ .when()
+ .post(DURABLE + "/durable-executions/" + arn + "/stop")
+ .then()
+ .statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"));
+
+ given()
+ .urlEncodingEnabled(false)
+ .contentType("application/json")
+ .body("{\"ErrorType\": \"Cancelled\", \"ErrorMessage\": \"by test\"}")
+ .when()
+ .post(DURABLE + "/durable-executions/" + arn + "/stop")
+ .then()
+ .statusCode(200)
+ .body("StopTimestamp", notNullValue());
+
+ given()
+ .urlEncodingEnabled(false)
+ .when()
+ .get(DURABLE + "/durable-executions/" + arn)
+ .then()
+ .statusCode(200)
+ .body("Status", equalTo("STOPPED"))
+ .body("Error.ErrorType", equalTo("Cancelled"))
+ .body("Error.ErrorMessage", equalTo("by test"));
+
+ given()
+ .urlEncodingEnabled(false)
+ .when()
+ .get(DURABLE + "/durable-executions/" + arn + "/history?ReverseOrder=true&MaxItems=1")
+ .then()
+ .statusCode(200)
+ .body("Events[0].EventType", equalTo("ExecutionStopped"));
+ }
+
+ private static DurableInvocationResult handlerResponse(String json) {
+ return new DurableInvocationResult("req-" + System.nanoTime(), json.getBytes(StandardCharsets.UTF_8), null);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableScriptedProfile.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableScriptedProfile.java
new file mode 100644
index 0000000000..4989683dbf
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableScriptedProfile.java
@@ -0,0 +1,14 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import io.quarkus.test.junit.QuarkusTestProfile;
+
+import java.util.Set;
+
+/** Swaps the Lambda container stack for {@link ScriptedDurableFunctionInvoker}. */
+public class LambdaDurableScriptedProfile implements QuarkusTestProfile {
+
+ @Override
+ public Set<Class<?>> getEnabledAlternatives() {
+ return Set.of(ScriptedDurableFunctionInvoker.class);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/ScriptedDurableFunctionInvoker.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/ScriptedDurableFunctionInvoker.java
new file mode 100644
index 0000000000..268c67a6bc
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/ScriptedDurableFunctionInvoker.java
@@ -0,0 +1,66 @@
+package io.github.hectorvent.floci.services.lambda.durable;
+
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.inject.Alternative;
+import jakarta.inject.Inject;
+
+import java.time.Duration;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Stands in for the Lambda container in protocol tests: functions resolve for real, but each
+ * invocation is handed to the test, which plays the SDK over the REST API and then answers.
+ * Enabled per test class through {@link LambdaDurableScriptedProfile}.
+ */
+@Alternative
+@ApplicationScoped
+public class ScriptedDurableFunctionInvoker implements DurableFunctionInvoker {
+
+ public record Invocation(ResolvedDurableTarget target, byte[] payload,
+ CompletableFuture<DurableInvocationResult> response) {
+ }
+
+ private final LambdaDurableFunctionInvoker real;
+ private final BlockingQueue<Invocation> invocations = new LinkedBlockingQueue<>();
+
+ @Inject
+ public ScriptedDurableFunctionInvoker(LambdaDurableFunctionInvoker real) {
+ this.real = real;
+ }
+
+ @Override
+ public ResolvedDurableTarget resolve(String accountId, String region, String functionName, String qualifier) {
+ return real.resolve(accountId, region, functionName, qualifier);
+ }
+
+ @Override
+ public DurableInvocationResult invoke(ResolvedDurableTarget target, byte[] payload) {
+ Invocation invocation = new Invocation(target, payload, new CompletableFuture<>());
+ invocations.add(invocation);
+ try {
+ return invocation.response().get(30, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("interrupted while waiting for the scripted response", e);
+ } catch (ExecutionException | TimeoutException e) {
+ throw new IllegalStateException("no scripted response for the invocation", e);
+ }
+ }
+
+ public Invocation awaitInvocation(Duration timeout) throws InterruptedException {
+ Invocation invocation = invocations.poll(timeout.toMillis(), TimeUnit.MILLISECONDS);
+ if (invocation == null) {
+ throw new AssertionError("no durable invocation within " + timeout);
+ }
+ return invocation;
+ }
+
+ public void drain() {
+ invocations.clear();
+ }
+}
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index f91147e5c6..5b73768483 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -151,6 +151,8 @@ floci:
honour-architectures: false
zip-max-entries: 100000
async-retry-delay-seconds: 1
+ durable-sweep-enabled: true
+ durable-sweep-interval-seconds: 1
# aws-config-path:
hot-reload:
enabled: true